Skip to content

Fix long file names with special characters in containers #376

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
martenrebane wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix long file names with special characters in containers #376

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions commons-lib/src/androidTest/kotlin/ee/ria/DigiDoc/common/ConstantTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
package ee.ria.DigiDoc.common

import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
import java.time.ZoneOffset
Expand Down Expand Up @@ -70,8 +71,11 @@ class ConstantTest {

@Test
fun testFilenameRestrictions() {
assertTrue(Constant.RESTRICTED_FILENAME_CHARACTERS_AND_RTL_CHARACTERS_AS_STRING.contains('@'))
assertTrue(Constant.RESTRICTED_FILENAME_CHARACTERS_AND_RTL_CHARACTERS_AS_STRING.contains('\u202E'))
assertTrue(Constant.FORBIDDEN_FILENAME_CHARACTERS.contains('/'))
assertTrue(Constant.FORBIDDEN_FILENAME_CHARACTERS.contains('*'))
assertFalse(Constant.FORBIDDEN_FILENAME_CHARACTERS.contains('@'))
// The app adds " (99)", an extension and "-data-files" to a name later
assertTrue(Constant.MAX_FILENAME_BYTES + 25 <= 255)
}

@Test
Expand Down
12 changes: 8 additions & 4 deletions commons-lib/src/main/kotlin/ee/ria/DigiDoc/common/Constant.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,10 +115,14 @@ object Constant {

const val TSL_SEQUENCE_NUMBER_ELEMENT: String = "TSLSequenceNumber"
const val KEY_LOCALE = "locale"
private const val RESTRICTED_FILENAME_CHARACTERS_AS_STRING = "@%:^?[]\\'\"”’{}#&`\\\\~«»/´"
private const val RTL_CHARACTERS_AS_STRING = "" + '\u200E' + '\u200F' + '\u202E' + '\u202A' + '\u202B'
const val RESTRICTED_FILENAME_CHARACTERS_AND_RTL_CHARACTERS_AS_STRING =
RESTRICTED_FILENAME_CHARACTERS_AS_STRING + RTL_CHARACTERS_AS_STRING

const val FORBIDDEN_FILENAME_CHARACTERS = "/\\<>:\"|?*"

const val ZERO_WIDTH_JOINER_CODE = 0x200D

// 255 bytes is the file system limit. The rest is room for the " (99)",
// extension and "-data-files" that the app adds on top of this name.
const val MAX_FILENAME_BYTES = 230
const val DEFAULT_FILENAME = "newFile"
const val ALLOWED_URL_CHARACTERS =
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_,.:/%;+=@?&!()"
Expand Down
92 changes: 50 additions & 42 deletions crypto-lib/src/main/kotlin/ee/ria/DigiDoc/cryptolib/CryptoContainer.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import ee.ria.DigiDoc.utilsLib.container.ContainerUtil
import ee.ria.DigiDoc.utilsLib.extensions.isCryptoContainer
import ee.ria.DigiDoc.utilsLib.extensions.saveAs
import ee.ria.DigiDoc.utilsLib.file.FileUtil.sanitizeString
import ee.ria.DigiDoc.utilsLib.file.FileUtil.uniqueFileName
import ee.ria.DigiDoc.utilsLib.logging.LoggingUtil.Companion.debugLog
import ee.ria.DigiDoc.utilsLib.logging.LoggingUtil.Companion.errorLog
import ee.ria.cdoc.CDoc
Expand All @@ -65,7 +66,6 @@ import java.io.FileInputStream
import java.io.FileOutputStream
import java.io.IOException
import java.io.InputStream
import java.io.OutputStream
import java.util.Base64
import javax.inject.Inject
import javax.inject.Singleton
Expand Down Expand Up @@ -164,7 +164,9 @@ class CryptoContainer

companion object {
val logger = JavaLogger()
var loggingIsSet = false

private val logLevelLock = Any()
private val libcdocLogLevel = LogLevel.LEVEL_TRACE

@Throws(CryptoException::class)
private suspend fun open(
Expand Down Expand Up @@ -286,50 +288,59 @@ class CryptoContainer

val cdocReader = CDocReader.createReader(file.path, conf, token, network)
debugLog(LOG_TAG, "Reader created: (version ${cdocReader.version})")
val idx = cdocReader.getLockForCert(authCert)
try {
val idx = cdocReader.getLockForCert(authCert)

if (idx < 0) {
throw CryptoException("Failed to get lock for certificate")
}
if (idx < 0) {
throw CryptoException("Failed to get lock for certificate")
}

val fmk = cdocReader.getFMK(idx.toInt())
val fmk = cdocReader.getFMK(idx.toInt())

if (token.lastError != null) {
throw token.lastError as Throwable
}
if (token.lastError != null) {
throw token.lastError as Throwable
}

if (fmk.isEmpty()) {
throw CryptoException("Failed to get FMK")
}
if (fmk.isEmpty()) {
throw CryptoException("Failed to get FMK")
}

if (cdocReader.beginDecryption(fmk) != 0L) {
throw CryptoException("Failed to begin decryption")
}
if (cdocReader.beginDecryption(fmk) != 0L) {
throw CryptoException("Failed to begin decryption")
}

val fi = FileInfo()
var result: Long = cdocReader.nextFile(fi)
try {
while (result == CDoc.OK.toLong()) {
val ofile = File(fi.name)
val dir =
ContainerUtil.getContainerDataFilesDir(
context,
file,
)
val tmp = sanitizeString(ofile.name, "")
val fileToSave = File(dir, tmp)
val ofs: OutputStream = FileOutputStream(fileToSave)
cdocReader.readFile(ofs)
dataFiles.add(fileToSave)
ofs.close()
result = cdocReader.nextFile(fi)
val fi = FileInfo()
val savedNames = mutableSetOf<String>()
val dir = ContainerUtil.getContainerDataFilesDir(context, file)

synchronized(logLevelLock) {
// Some file names crash the app if libcdoc logs them while reading
logger.setMinLogLevel(LogLevel.LEVEL_INFO)
try {
var result: Long = cdocReader.nextFile(fi)
while (result == CDoc.OK.toLong()) {
val ofile = File(fi.name)
val tmp = uniqueFileName(sanitizeString(ofile.name, ""), savedNames)
savedNames.add(tmp)
val fileToSave = File(dir, tmp)
FileOutputStream(fileToSave).use { ofs ->
cdocReader.readFile(ofs)
}
dataFiles.add(fileToSave)
result = cdocReader.nextFile(fi)
}

if (cdocReader.finishDecryption() != 0L) {
throw CryptoException("Failed to finish decryption")
}
} finally {
logger.setMinLogLevel(libcdocLogLevel)
}
}
} catch (exc: IOException) {
throw CryptoException("IO Exception: ${exc.message}", exc)
}

if (cdocReader.finishDecryption() != 0L) {
throw CryptoException("Failed to finish decryption")
} finally {
cdocReader.delete()
}

return create(
Expand Down Expand Up @@ -499,11 +510,8 @@ class CryptoContainer

fun setLogging(isLoggingEnabled: Boolean) {
if (isLoggingEnabled) {
logger.setMinLogLevel(LogLevel.LEVEL_TRACE)
if (!loggingIsSet) {
CDoc.setLogger(logger)
loggingIsSet = true
}
logger.setMinLogLevel(libcdocLogLevel)
CDoc.setLogger(logger)
CDoc.log(LogLevel.LEVEL_DEBUG, "CryptoContainer", 450, "Set libcdoc logging: true")
}
}
Expand Down
3 changes: 3 additions & 0 deletions utils-lib/src/main/kotlin/ee/ria/DigiDoc/utilsLib/container/ContainerUtil.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,7 @@ object ContainerUtil {
if (i > 0) {
name.append(i)
}

dir = File(directory, name.toString())
if (dir.isDirectory || !dir.exists()) {
break
Expand All @@ -220,6 +221,8 @@ object ContainerUtil {
if (directory != null) {
debugLog(LOG_TAG, "Directories created or already exist for " + directory.path)
}
} else if (!dir.isDirectory) {
errorLog(LOG_TAG, "Unable to create data file directory, name is ${dir.name.toByteArray().size} bytes")
}

return dir
Expand Down
91 changes: 77 additions & 14 deletions utils-lib/src/main/kotlin/ee/ria/DigiDoc/utilsLib/file/FileUtil.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ import android.webkit.URLUtil
import androidx.core.net.toUri
import ee.ria.DigiDoc.common.Constant.ALLOWED_URL_CHARACTERS
import ee.ria.DigiDoc.common.Constant.DEFAULT_FILENAME
import ee.ria.DigiDoc.common.Constant.RESTRICTED_FILENAME_CHARACTERS_AND_RTL_CHARACTERS_AS_STRING
import ee.ria.DigiDoc.common.Constant.FORBIDDEN_FILENAME_CHARACTERS
import ee.ria.DigiDoc.common.Constant.MAX_FILENAME_BYTES
import ee.ria.DigiDoc.common.Constant.ZERO_WIDTH_JOINER_CODE
import ee.ria.DigiDoc.utilsLib.logging.LoggingUtil.Companion.errorLog
import ee.ria.DigiDoc.utilsLib.logging.LoggingUtil.Companion.infoLog
import kotlinx.coroutines.Dispatchers
Expand All @@ -55,6 +57,8 @@ import java.io.OutputStreamWriter
import java.nio.charset.Charset
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.text.BreakIterator
import java.text.Normalizer
import javax.xml.parsers.DocumentBuilderFactory
import javax.xml.parsers.SAXParserFactory

Expand Down Expand Up @@ -124,26 +128,85 @@ object FileUtil {
} else if (trimmed.startsWith(".")) {
trimmed = DEFAULT_FILENAME + trimmed
}
if (isRawUrl(trimmed)) {
return FilenameUtils.getName(FilenameUtils.normalize(trimmed)) ?: DEFAULT_FILENAME
}
if (URLUtil.isValidUrl(trimmed)) {
return FilenameUtils.getName(normalizeUri(trimmed.toUri()).toString()) ?: DEFAULT_FILENAME
}
val sb = StringBuilder(trimmed.length)
if (!URLUtil.isValidUrl(trimmed) && !isRawUrl(trimmed)) {
for (element in trimmed) {
if (RESTRICTED_FILENAME_CHARACTERS_AND_RTL_CHARACTERS_AS_STRING.indexOf(element) != -1) {
sb.append(replacement)
} else {
sb.append(element)
}
for (element in trimmed) {
if (isForbiddenInFileName(element)) {
sb.append(replacement)
} else {
sb.append(element)
}
} else if (!isRawUrl(trimmed)) {
return normalizeUri(trimmed.toUri()).toString()
}
return if (sb.toString().isNotEmpty()) {
val name: String =
FilenameUtils.getName(
FilenameUtils.normalize(
sb.toString(),
sb.toString().trim { it <= ' ' },
),
)
) ?: ""
return if (name.isEmpty() || name.all { it == '.' }) {
DEFAULT_FILENAME
} else {
FilenameUtils.normalize(trimmed)
truncateFileName(Normalizer.normalize(name, Normalizer.Form.NFC), MAX_FILENAME_BYTES)
}
}

private fun isForbiddenInFileName(character: Char): Boolean {
if (character.code == ZERO_WIDTH_JOINER_CODE) {
return false
}
return FORBIDDEN_FILENAME_CHARACTERS.indexOf(character) != -1 ||
character.category == CharCategory.CONTROL ||
character.category == CharCategory.FORMAT
}

fun truncateFileName(
fileName: String,
maxBytes: Int,
): String {
if (fileName.toByteArray().size <= maxBytes) {
return fileName
}
val extension = FilenameUtils.getExtension(fileName)
val suffix = if (extension.isEmpty()) "" else ".$extension"
val baseName = FilenameUtils.getBaseName(fileName)
val truncated = truncateToBytes(baseName, maxBytes - suffix.toByteArray().size)

val fitted = if (truncated.isEmpty()) truncateToBytes(fileName, maxBytes) else truncated + suffix

return fitted.ifEmpty { truncateToBytes(DEFAULT_FILENAME + suffix, maxBytes) }
}

private fun truncateToBytes(
text: String,
maxBytes: Int,
): String {
val characters = BreakIterator.getCharacterInstance()
characters.setText(text)
val cuts = generateSequence(characters.first()) { characters.next().takeIf { it != BreakIterator.DONE } }
return text.substring(0, cuts.lastOrNull { text.substring(0, it).toByteArray().size <= maxBytes } ?: 0)
}

fun uniqueFileName(
fileName: String,
taken: Set<String>,
): String {
if (!taken.contains(fileName)) {
return fileName
}
val baseName = FilenameUtils.getBaseName(fileName).ifEmpty { DEFAULT_FILENAME }
val extension = FilenameUtils.getExtension(fileName)
var counter = 1
while (true) {
val candidate = if (extension.isEmpty()) "$baseName ($counter)" else "$baseName ($counter).$extension"
if (!taken.contains(candidate)) {
return candidate
}
counter++
}
}

Expand Down
Loading
Loading