From 488b411d8abf281744bfb11d2274910241e57eed Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 8 Aug 2026 23:05:32 +0200 Subject: [PATCH 1/3] compress: do not copy chunk data to bytes, use the buffer protocol The chunker hands chunk data to the compressors as a memoryview into its scan buffer, but LZ4._decide (the default compression) coerced it to bytes first, copying the full plaintext chunk once per chunk just to get a char* for the C call. Use a typed memoryview instead, so lz4 reads straight from the chunker's buffer. CompressorBase.compress (used by CNONE, i.e. also the fallback for incompressible chunks) did the same coercion, which is only needed in legacy mode for the ID/level prefix concatenation - keep it there and pass the buffer through otherwise. This removed a second full copy for incompressible chunks. ObfuscateSize.compress now may receive a memoryview from the inner compressor, so join instead of concatenating (and skip the copy completely when the added padding size is 0). Measured on a 5 GiB borg create (lz4, aes256-ocb, half incompressible / half compressible input): ~3% faster overall, compression-path time per stored chunk -23%. --- src/borg/compress.pyx | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx index a0d9361810..367196b373 100644 --- a/src/borg/compress.pyx +++ b/src/borg/compress.pyx @@ -146,12 +146,11 @@ cdef class CompressorBase: def compress(self, meta, data): """ - Compress *data* (bytes) and return compression metadata and compressed bytes. + Compress *data* (bytes or memoryview) and return compression metadata and compressed data. """ - if not isinstance(data, bytes): - data = bytes(data) # code below does not work with memoryview if self.legacy_mode: - return None, bytes((self.ID, self.level)) + data + # the ID/level prefix concatenation needs bytes (bytes() is a no-op for bytes input) + return None, bytes((self.ID, self.level)) + bytes(data) else: meta["ctype"] = self.ID meta["clevel"] = self.level @@ -276,12 +275,15 @@ class LZ4(DecidingCompressor): *lz4_data* is the LZ4 result if *compressor* is LZ4 as well, otherwise it is None. """ - if not isinstance(idata, bytes): - idata = bytes(idata) # code below does not work with memoryview - cdef int isize = len(idata) + cdef const unsigned char[::1] iview = idata + cdef int isize = iview.shape[0] cdef int osize - cdef char *source = idata + cdef const char *source cdef char *dest + if isize == 0: + # empty input cannot shrink (and an empty view has no address to take below) + return NONE_COMPRESSOR, (meta, None) + source = &iview[0] osize = LZ4_compressBound(isize) buf = get_buffer().get(osize) dest = buf @@ -630,8 +632,11 @@ class ObfuscateSize(CompressorBase): addtl_size = self._obfuscate(compr_size) if meta["type"] == ROBJ_FILE_STREAM else 0 addtl_size = max(0, addtl_size) # we can only make it longer, not shorter! addtl_size = min(MAX_DATA_SIZE - 1024 - compr_size, addtl_size) # stay away from MAX_DATA_SIZE - trailer = bytes(addtl_size) - obfuscated_data = compressed_data + trailer + if addtl_size: + # join, not +: compressed_data may be a memoryview (CNONE passes the input through) + obfuscated_data = b"".join([compressed_data, bytes(addtl_size)]) + else: + obfuscated_data = compressed_data meta["csize"] = len(obfuscated_data) # csize is the overall output size of this "obfuscation compressor" meta["olevel"] = self.level # remember the obfuscation level, useful for repo-compress return meta, obfuscated_data # for borg2 it is enough that we have the payload size in meta["psize"] From 29132090dabb0602a8dbb1988ea4351d404a993f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 9 Aug 2026 09:53:05 +0200 Subject: [PATCH 2/3] crypto: AEAD encrypt/decrypt directly into the result bytes object _AEAD_BASE.encrypt and .decrypt used a PyMem_Malloc'd scratch buffer (with a one-block safety margin) and created the returned bytes object by slicing it, copying the whole ciphertext / plaintext once more per chunk before freeing the scratch buffer. Our AEAD ciphers (AES-OCB, chacha20-poly1305) are padding-free, so the output size is known exactly up front: allocate the result via PyBytes_FromStringAndSize(NULL, n) and let OpenSSL write into it directly - no scratch buffer, no copy, no malloc/free per chunk. Caveat found by the tests: EVP_EncryptFinal_ex is not output-free for OCB - OpenSSL buffers the partial final block in EncryptUpdate and emits it at Final (the total is still exactly the input length). As our AEAD modes are padding-free, Final can never write more than the space left in the exact-size result buffer, so it writes directly into it; a length check afterwards verifies the total. This helps both directions: borg create saves one ciphertext-sized copy per chunk, and the read paths (extract, mount, check --verify-data) save a plaintext-sized copy per chunk. Measured on 5 GiB (lz4, aes256-ocb): create ~1.5% faster, extract ~6% faster. --- src/borg/crypto/low_level.pyx | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index 9c4d47b09d..af5c42128e 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -514,10 +514,11 @@ cdef class _AEAD_BASE: cdef int rc try: - odata = PyMem_Malloc(hlen + self.mac_len + - ilen + self.cipher_blk_len) - if not odata: - raise MemoryError + # Our AEAD ciphers (OCB, chacha20-poly1305) are padding-free: the ciphertext is + # exactly as long as the plaintext. Thus the result can be allocated up front + # and the cipher writes directly into it - no scratch buffer, no copy. + ret = PyBytes_FromStringAndSize(NULL, hlen + self.mac_len + ilen) + odata = PyBytes_AsString(ret) idata = ro_buffer(data) idata_acquired = True @@ -545,16 +546,19 @@ cdef class _AEAD_BASE: if not rc: raise CryptoError('EVP_EncryptUpdate failed') offset += olen + # Final can emit a buffered partial block (OCB does). Our AEAD modes are + # padding-free, so it never writes more than the space left in the + # exact-size result buffer. if not EVP_EncryptFinal_ex(self.ctx, odata+offset, &olen): raise CryptoError('EVP_EncryptFinal_ex failed') offset += olen if not EVP_CIPHER_CTX_ctrl(self.ctx, EVP_CTRL_AEAD_GET_TAG, self.mac_len, odata + hlen): raise CryptoError('EVP_CIPHER_CTX_ctrl GET TAG failed') + if offset != hlen + self.mac_len + ilen: + raise CryptoError('unexpected ciphertext length') self.blocks = block_count - return odata[:offset] + return ret finally: - if odata: - PyMem_Free(odata) if hdata_acquired: PyBuffer_Release(&hdata) if idata_acquired: @@ -591,9 +595,11 @@ cdef class _AEAD_BASE: cdef int rc try: - odata = PyMem_Malloc(ilen + self.cipher_blk_len) - if not odata: - raise MemoryError + # Our AEAD ciphers (OCB, chacha20-poly1305) are padding-free: the plaintext is + # exactly as long as the ciphertext. Thus the result can be allocated up front + # and the cipher writes directly into it - no scratch buffer, no copy. + ret = PyBytes_FromStringAndSize(NULL, ilen - hlen - self.mac_len) + odata = PyBytes_AsString(ret) idata = ro_buffer(envelope) idata_acquired = True @@ -619,15 +625,18 @@ cdef class _AEAD_BASE: offset += olen if not EVP_CIPHER_CTX_ctrl(self.ctx, EVP_CTRL_AEAD_SET_TAG, self.mac_len, idata.buf + hlen): raise CryptoError('EVP_CIPHER_CTX_ctrl SET TAG failed') + # Final can emit a buffered partial block (OCB does). Our AEAD modes are + # padding-free, so it never writes more than the space left in the + # exact-size result buffer. if not EVP_DecryptFinal_ex(self.ctx, odata+offset, &olen): # a failure here means corrupted or tampered tag (mac) or data. raise IntegrityError('Authentication / EVP_DecryptFinal_ex failed') offset += olen + if offset != ilen - hlen - self.mac_len: + raise CryptoError('unexpected plaintext length') self.blocks = self.block_count(offset) - return odata[:offset] + return ret finally: - if odata: - PyMem_Free(odata) if idata_acquired: PyBuffer_Release(&idata) if aadata_acquired: From 10fbb99bd79b4917ad593d9608ecc859f48103ad Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 9 Aug 2026 11:16:00 +0200 Subject: [PATCH 3/3] compress: lz4 decompresses directly into the result bytes object LZ4.decompress guessed the output size, decompressed into a thread-local scratch buffer (growing and retrying on overflow) and then copied the whole plaintext into the returned bytes object - one full plaintext-sized copy per chunk on every read (extract, mount, check --verify-data). borg2 stores the exact plaintext size in the (authenticated) object metadata, so when meta["size"] is known, allocate the result exactly and let LZ4_decompress_safe write straight into it: no scratch buffer, no guessing, no copy. A size mismatch raises DecompressionError (this also covers what check_fix_size asserted). The guess-and-retry loop remains as the fallback for borg 1.x data read in legacy mode (borg transfer), where the size is not known up front. Measured on a 5 GiB extract (lz4, aes256-ocb): ~4% faster. --- src/borg/compress.pyx | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx index 367196b373..57019d6259 100644 --- a/src/borg/compress.pyx +++ b/src/borg/compress.pyx @@ -23,6 +23,8 @@ import sys import threading import zlib +from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AsString + try: import lzma except ImportError: @@ -306,6 +308,21 @@ class LZ4(DecidingCompressor): cdef int rsize cdef char *source = idata cdef char *dest + size = None if meta is None else meta.get("size") + if size is not None and 0 <= size <= 2 ** 31 - 1: + # borg2 stores the exact plaintext size in the (authenticated) object metadata: + # decompress directly into the result bytes object - no scratch buffer, no copy, + # no output size guessing. rsize != size means corrupt (or misdescribed) data, + # this also covers everything check_fix_size would assert. + ret = PyBytes_FromStringAndSize(NULL, size) + dest = PyBytes_AsString(ret) + osize = size + with nogil: + rsize = LZ4_decompress_safe(source, dest, isize, osize) + if rsize != osize: + raise DecompressionError('lz4 decompress failed') + return meta, ret + # no size known up front (borg 1.x repos in legacy mode): guess and retry. # a bit more than 8MB is enough for the usual data sizes yielded by the chunker. # allocate more if isize * 3 is already bigger, to avoid having to resize often. osize = max(int(1.1 * 2**23), isize * 3)