diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx index a0d9361810..57019d6259 100644 --- a/src/borg/compress.pyx +++ b/src/borg/compress.pyx @@ -23,6 +23,8 @@ import sys import threading import zlib +from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AsString + try: import lzma except ImportError: @@ -146,12 +148,11 @@ cdef class CompressorBase: def compress(self, meta, data): """ - Compress *data* (bytes) and return compression metadata and compressed bytes. + Compress *data* (bytes or memoryview) and return compression metadata and compressed data. """ - if not isinstance(data, bytes): - data = bytes(data) # code below does not work with memoryview if self.legacy_mode: - return None, bytes((self.ID, self.level)) + data + # the ID/level prefix concatenation needs bytes (bytes() is a no-op for bytes input) + return None, bytes((self.ID, self.level)) + bytes(data) else: meta["ctype"] = self.ID meta["clevel"] = self.level @@ -276,12 +277,15 @@ class LZ4(DecidingCompressor): *lz4_data* is the LZ4 result if *compressor* is LZ4 as well, otherwise it is None. """ - if not isinstance(idata, bytes): - idata = bytes(idata) # code below does not work with memoryview - cdef int isize = len(idata) + cdef const unsigned char[::1] iview = idata + cdef int isize = iview.shape[0] cdef int osize - cdef char *source = idata + cdef const char *source cdef char *dest + if isize == 0: + # empty input cannot shrink (and an empty view has no address to take below) + return NONE_COMPRESSOR, (meta, None) + source = &iview[0] osize = LZ4_compressBound(isize) buf = get_buffer().get(osize) dest = buf @@ -304,6 +308,21 @@ class LZ4(DecidingCompressor): cdef int rsize cdef char *source = idata cdef char *dest + size = None if meta is None else meta.get("size") + if size is not None and 0 <= size <= 2 ** 31 - 1: + # borg2 stores the exact plaintext size in the (authenticated) object metadata: + # decompress directly into the result bytes object - no scratch buffer, no copy, + # no output size guessing. rsize != size means corrupt (or misdescribed) data, + # this also covers everything check_fix_size would assert. + ret = PyBytes_FromStringAndSize(NULL, size) + dest = PyBytes_AsString(ret) + osize = size + with nogil: + rsize = LZ4_decompress_safe(source, dest, isize, osize) + if rsize != osize: + raise DecompressionError('lz4 decompress failed') + return meta, ret + # no size known up front (borg 1.x repos in legacy mode): guess and retry. # a bit more than 8MB is enough for the usual data sizes yielded by the chunker. # allocate more if isize * 3 is already bigger, to avoid having to resize often. osize = max(int(1.1 * 2**23), isize * 3) @@ -630,8 +649,11 @@ class ObfuscateSize(CompressorBase): addtl_size = self._obfuscate(compr_size) if meta["type"] == ROBJ_FILE_STREAM else 0 addtl_size = max(0, addtl_size) # we can only make it longer, not shorter! addtl_size = min(MAX_DATA_SIZE - 1024 - compr_size, addtl_size) # stay away from MAX_DATA_SIZE - trailer = bytes(addtl_size) - obfuscated_data = compressed_data + trailer + if addtl_size: + # join, not +: compressed_data may be a memoryview (CNONE passes the input through) + obfuscated_data = b"".join([compressed_data, bytes(addtl_size)]) + else: + obfuscated_data = compressed_data meta["csize"] = len(obfuscated_data) # csize is the overall output size of this "obfuscation compressor" meta["olevel"] = self.level # remember the obfuscation level, useful for repo-compress return meta, obfuscated_data # for borg2 it is enough that we have the payload size in meta["psize"] diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index 9c4d47b09d..af5c42128e 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -514,10 +514,11 @@ cdef class _AEAD_BASE: cdef int rc try: - odata = PyMem_Malloc(hlen + self.mac_len + - ilen + self.cipher_blk_len) - if not odata: - raise MemoryError + # Our AEAD ciphers (OCB, chacha20-poly1305) are padding-free: the ciphertext is + # exactly as long as the plaintext. Thus the result can be allocated up front + # and the cipher writes directly into it - no scratch buffer, no copy. + ret = PyBytes_FromStringAndSize(NULL, hlen + self.mac_len + ilen) + odata = PyBytes_AsString(ret) idata = ro_buffer(data) idata_acquired = True @@ -545,16 +546,19 @@ cdef class _AEAD_BASE: if not rc: raise CryptoError('EVP_EncryptUpdate failed') offset += olen + # Final can emit a buffered partial block (OCB does). Our AEAD modes are + # padding-free, so it never writes more than the space left in the + # exact-size result buffer. if not EVP_EncryptFinal_ex(self.ctx, odata+offset, &olen): raise CryptoError('EVP_EncryptFinal_ex failed') offset += olen if not EVP_CIPHER_CTX_ctrl(self.ctx, EVP_CTRL_AEAD_GET_TAG, self.mac_len, odata + hlen): raise CryptoError('EVP_CIPHER_CTX_ctrl GET TAG failed') + if offset != hlen + self.mac_len + ilen: + raise CryptoError('unexpected ciphertext length') self.blocks = block_count - return odata[:offset] + return ret finally: - if odata: - PyMem_Free(odata) if hdata_acquired: PyBuffer_Release(&hdata) if idata_acquired: @@ -591,9 +595,11 @@ cdef class _AEAD_BASE: cdef int rc try: - odata = PyMem_Malloc(ilen + self.cipher_blk_len) - if not odata: - raise MemoryError + # Our AEAD ciphers (OCB, chacha20-poly1305) are padding-free: the plaintext is + # exactly as long as the ciphertext. Thus the result can be allocated up front + # and the cipher writes directly into it - no scratch buffer, no copy. + ret = PyBytes_FromStringAndSize(NULL, ilen - hlen - self.mac_len) + odata = PyBytes_AsString(ret) idata = ro_buffer(envelope) idata_acquired = True @@ -619,15 +625,18 @@ cdef class _AEAD_BASE: offset += olen if not EVP_CIPHER_CTX_ctrl(self.ctx, EVP_CTRL_AEAD_SET_TAG, self.mac_len, idata.buf + hlen): raise CryptoError('EVP_CIPHER_CTX_ctrl SET TAG failed') + # Final can emit a buffered partial block (OCB does). Our AEAD modes are + # padding-free, so it never writes more than the space left in the + # exact-size result buffer. if not EVP_DecryptFinal_ex(self.ctx, odata+offset, &olen): # a failure here means corrupted or tampered tag (mac) or data. raise IntegrityError('Authentication / EVP_DecryptFinal_ex failed') offset += olen + if offset != ilen - hlen - self.mac_len: + raise CryptoError('unexpected plaintext length') self.blocks = self.block_count(offset) - return odata[:offset] + return ret finally: - if odata: - PyMem_Free(odata) if idata_acquired: PyBuffer_Release(&idata) if aadata_acquired: