From 9f35303beeb5ec120ce215c0a9274e495b21b5ce Mon Sep 17 00:00:00 2001 From: Vladimir Smitka Date: Sat, 12 Sep 2026 17:05:58 +0000 Subject: [PATCH 1/6] storage: map_file - read a file straight from memory-mapped flash storage.map_file(f) returns a tuple of read-only memoryviews over the flash bytes of an open file on the internal CIRCUITPY drive, one per contiguous cluster run, in file order and 0 copy. Anything that takes a buffer can then use the file without reading it into RAM: a synthio.MidiTrack, a RawSample, a wavetable, a ulab array, a bitmap. Assets stay ordinary files on the drive. Opening a file for reading already builds its FatFs cluster-link map, so the function only reads that map. The supervisor maps a FatFs sector to a flash address through a port hook that also reports how far the mapping stays contiguous, so a run is split where it is not. raspberrypi returns the execute-in-place address (the drive is XIP on every RP2 board); espressif esp_partition_mmap's each drive partition on first use and reports the seam of an extended drive. The function is always present; on a port whose drive is not mapped (CIRCUITPY_STORAGE_MAP_FILE off) it raises NotImplementedError. 600 B of text on pajenicko_picopad, 496 B on adafruit_feather_esp32s3_tft. --- ports/espressif/mpconfigport.mk | 3 + ports/espressif/supervisor/internal_flash.c | 37 ++++++++++ ports/raspberrypi/mpconfigport.mk | 2 + ports/raspberrypi/supervisor/internal_flash.c | 8 +++ py/circuitpy_mpconfig.mk | 5 ++ shared-bindings/storage/__init__.c | 24 +++++++ shared-bindings/storage/__init__.h | 2 + shared-module/storage/__init__.c | 68 +++++++++++++++++++ supervisor/flash.h | 6 ++ supervisor/shared/flash.c | 18 +++++ supervisor/shared/internal_flash.h | 6 ++ 11 files changed, 179 insertions(+) diff --git a/ports/espressif/mpconfigport.mk b/ports/espressif/mpconfigport.mk index 5c3c5c0bc74..447aa3da6aa 100644 --- a/ports/espressif/mpconfigport.mk +++ b/ports/espressif/mpconfigport.mk @@ -479,3 +479,6 @@ endif # Usually lots of flash space available CIRCUITPY_MESSAGE_COMPRESSION_LEVEL ?= 1 + +# The CIRCUITPY partition is mapped into the data address space on first use +CIRCUITPY_STORAGE_MAP_FILE ?= 1 diff --git a/ports/espressif/supervisor/internal_flash.c b/ports/espressif/supervisor/internal_flash.c index a18d3fe0a0f..cebeb4f89bb 100644 --- a/ports/espressif/supervisor/internal_flash.c +++ b/ports/espressif/supervisor/internal_flash.c @@ -80,6 +80,43 @@ uint32_t supervisor_flash_get_block_count(void) { void port_internal_flash_flush(void) { } +#if CIRCUITPY_STORAGE_MAP_FILE +// storage.map_file: each drive partition mapped into the data address space on first use. With +// extended storage the drive spans two partitions that need not be adjacent in the mapping, so +// *contiguous stops at the seam and the caller splits a cluster run there. +static const uint8_t *map_partition(size_t i) { + static const uint8_t *base[2]; + if (base[i] == NULL) { + const void *p; + esp_partition_mmap_handle_t handle; // stays mapped for the session + if (esp_partition_mmap(_partition[i], 0, _partition[i]->size, ESP_PARTITION_MMAP_DATA, + &p, &handle) != ESP_OK) { + return NULL; + } + base[i] = p; + } + return base[i]; +} + +const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous) { + size_t i = 0; + uint32_t blocks = _partition[0]->size / FILESYSTEM_BLOCK_SIZE; + #if CIRCUITPY_STORAGE_EXTEND + if (storage_extended && block >= blocks) { + block -= blocks; + blocks = _partition[1]->size / FILESYSTEM_BLOCK_SIZE; + i = 1; + } + #endif + const uint8_t *base = map_partition(i); + if (base == NULL) { + return NULL; + } + *contiguous = blocks - block; + return base + block * FILESYSTEM_BLOCK_SIZE; +} +#endif + static void single_partition_rw(const esp_partition_t *partition, uint8_t *data, const uint32_t offset, const uint32_t size_total, const bool op) { if (op == OP_READ) { diff --git a/ports/raspberrypi/mpconfigport.mk b/ports/raspberrypi/mpconfigport.mk index 551ab00ab47..851e2398b2b 100644 --- a/ports/raspberrypi/mpconfigport.mk +++ b/ports/raspberrypi/mpconfigport.mk @@ -26,6 +26,8 @@ CIRCUITPY_PWMIO ?= 1 CIRCUITPY_RGBMATRIX ?= $(CIRCUITPY_DISPLAYIO) CIRCUITPY_ROTARYIO ?= 1 CIRCUITPY_ROTARYIO_SOFTENCODER = 1 +# The CIRCUITPY drive is execute-in-place on every RP2 board +CIRCUITPY_STORAGE_MAP_FILE ?= 1 CIRCUITPY_SYNTHIO_MAX_CHANNELS = 24 CIRCUITPY_USB_HOST ?= 1 CIRCUITPY_USB_VIDEO ?= 1 diff --git a/ports/raspberrypi/supervisor/internal_flash.c b/ports/raspberrypi/supervisor/internal_flash.c index 07b15564c60..e9da8a39a14 100644 --- a/ports/raspberrypi/supervisor/internal_flash.c +++ b/ports/raspberrypi/supervisor/internal_flash.c @@ -149,3 +149,11 @@ mp_uint_t supervisor_flash_write_blocks(const uint8_t *src, uint32_t lba, uint32 void supervisor_flash_release_cache(void) { } + +#if CIRCUITPY_STORAGE_MAP_FILE +// storage.map_file: the CIRCUITPY drive is execute-in-place on every RP2 board. +const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous) { + *contiguous = supervisor_flash_get_block_count() - block; + return (const uint8_t *)(XIP_BASE + CIRCUITPY_CIRCUITPY_DRIVE_START_ADDR + block * FILESYSTEM_BLOCK_SIZE); +} +#endif diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk index b5b0921c4ac..68950c920e6 100755 --- a/py/circuitpy_mpconfig.mk +++ b/py/circuitpy_mpconfig.mk @@ -636,6 +636,11 @@ CFLAGS += -DCIRCUITPY_STORAGE=$(CIRCUITPY_STORAGE) CIRCUITPY_STORAGE_EXTEND ?= $(CIRCUITPY_DUALBANK) CFLAGS += -DCIRCUITPY_STORAGE_EXTEND=$(CIRCUITPY_STORAGE_EXTEND) +# storage.map_file(): read a file straight out of memory-mapped flash. Ports whose CIRCUITPY drive +# is memory-mapped implement port_internal_flash_xip_address() and turn it on. +CIRCUITPY_STORAGE_MAP_FILE ?= 0 +CFLAGS += -DCIRCUITPY_STORAGE_MAP_FILE=$(CIRCUITPY_STORAGE_MAP_FILE) + CIRCUITPY_STRUCT ?= 1 CFLAGS += -DCIRCUITPY_STRUCT=$(CIRCUITPY_STRUCT) diff --git a/shared-bindings/storage/__init__.c b/shared-bindings/storage/__init__.c index b5f19529be1..b0e36dc27f3 100644 --- a/shared-bindings/storage/__init__.c +++ b/shared-bindings/storage/__init__.c @@ -300,6 +300,29 @@ static mp_obj_t storage_enable_usb_drive(void) { } MP_DEFINE_CONST_FUN_OBJ_0(storage_enable_usb_drive_obj, storage_enable_usb_drive); + +//| def map_file(file: typing.BinaryIO) -> Tuple[memoryview, ...]: +//| """Map an open file on the internal CIRCUITPY drive straight from flash, without copying it +//| into RAM. Returns one read-only `memoryview` per contiguous run of the file's clusters, in +//| file order: a contiguous file gives a 1-tuple, an empty file an empty tuple. Run boundaries +//| fall on cluster boundaries. The views stay valid after the file is closed, and anything that +//| takes a buffer can use them: a `synthio.MidiTrack`, an `audiocore.RawSample`, a wavetable. +//| +//| Treat the file as read-only while a view is in use: rewriting it shows the new bytes, and +//| mid-write it shows a torn file. +//| +//| :param typing.BinaryIO file: A file on the CIRCUITPY drive open for binary reading (``"rb"``) +//| :raises OSError: ``EINVAL`` if the file is closed or not open for reading only, +//| ``EOPNOTSUPP`` if it is on another mount or this build cannot map the drive, +//| ``EIO`` if its cluster chain is corrupt +//| :raises ~builtins.MemoryError: if ``open()`` could not allocate the file's cluster map +//| :raises NotImplementedError: on a port whose drive is not memory-mapped""" +//| ... +static mp_obj_t storage_map_file(mp_obj_t file_in) { + return common_hal_storage_map_file(file_in); +} +MP_DEFINE_CONST_FUN_OBJ_1(storage_map_file_obj, storage_map_file); + static const mp_rom_map_elem_t storage_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_storage) }, @@ -307,6 +330,7 @@ static const mp_rom_map_elem_t storage_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&storage_umount_obj) }, { MP_ROM_QSTR(MP_QSTR_remount), MP_ROM_PTR(&storage_remount_obj) }, { MP_ROM_QSTR(MP_QSTR_getmount), MP_ROM_PTR(&storage_getmount_obj) }, + { MP_ROM_QSTR(MP_QSTR_map_file), MP_ROM_PTR(&storage_map_file_obj) }, { MP_ROM_QSTR(MP_QSTR_erase_filesystem), MP_ROM_PTR(&storage_erase_filesystem_obj) }, { MP_ROM_QSTR(MP_QSTR_disable_usb_drive), MP_ROM_PTR(&storage_disable_usb_drive_obj) }, { MP_ROM_QSTR(MP_QSTR_enable_usb_drive), MP_ROM_PTR(&storage_enable_usb_drive_obj) }, diff --git a/shared-bindings/storage/__init__.h b/shared-bindings/storage/__init__.h index 0e53c78b153..12d13c2ccfc 100644 --- a/shared-bindings/storage/__init__.h +++ b/shared-bindings/storage/__init__.h @@ -21,3 +21,5 @@ MP_NORETURN void common_hal_storage_erase_filesystem(bool extended); bool common_hal_storage_disable_usb_drive(void); bool common_hal_storage_unsafe_disable_usb_drive(void); bool common_hal_storage_enable_usb_drive(void); + +mp_obj_t common_hal_storage_map_file(mp_obj_t file); diff --git a/shared-module/storage/__init__.c b/shared-module/storage/__init__.c index 1522136332a..8fe286dfb34 100644 --- a/shared-module/storage/__init__.c +++ b/shared-module/storage/__init__.c @@ -18,6 +18,12 @@ #include "shared-bindings/storage/__init__.h" #include "supervisor/filesystem.h" #include "supervisor/flash.h" +#if CIRCUITPY_STORAGE_MAP_FILE +#include "extmod/vfs_fat.h" +#include "lib/oofatfs/ff.h" +#include "py/objarray.h" +#include "py/objlist.h" +#endif #if CIRCUITPY_USB_DEVICE #include "supervisor/usb.h" @@ -233,3 +239,65 @@ void common_hal_storage_erase_filesystem(bool extended) { common_hal_mcu_reset(); // We won't actually get here, since we're resetting. } + +mp_obj_t common_hal_storage_map_file(mp_obj_t file_in) { + #if CIRCUITPY_STORAGE_MAP_FILE + pyb_file_obj_t *file = MP_OBJ_TO_PTR(mp_arg_validate_type(file_in, &mp_type_vfs_fat_fileio, MP_QSTR_file)); + FATFS *fatfs = file->fp.obj.fs; + if (fatfs == NULL || (file->fp.flag & FA_WRITE)) { + mp_raise_OSError(MP_EINVAL); // closed, or not open for reading only + } + fs_user_mount_t *drive = filesystem_circuitpy(); + if (drive == NULL || fatfs != &drive->fatfs) { + mp_raise_OSError(MP_EOPNOTSUPP); // another mount (an SD card): not memory-mapped + } + if (file->fp.err) { + mp_raise_OSError(fresult_to_errno_table[file->fp.err]); // open() hit a bad cluster chain + } + DWORD *tbl = file->fp.cltbl; + if (tbl == NULL) { + mp_raise_type(&mp_type_MemoryError); // open() could not allocate the cluster map + } + FSIZE_t size = f_size(&file->fp); + if (size == 0) { + return mp_const_empty_tuple; + } + supervisor_flash_flush(); // write back any RAM sector cache + // Walk the cluster runs of the link map open() built and split a run wherever the port's + // mapping is not contiguous, e.g. at the seam of a drive that spans two flash partitions. + mp_obj_t views = mp_obj_new_list(0, NULL); + size_t nruns = (size_t)((tbl[0] - 2) / 2); + FSIZE_t left = size; + for (size_t i = 0; i < nruns && left > 0; i++) { + DWORD sector = fatfs->database + (tbl[2 + 2 * i] - 2) * fatfs->csize; + FSIZE_t span = (FSIZE_t)tbl[1 + 2 * i] * fatfs->csize * FF_MIN_SS; + if (span > left) { + span = left; // the last run is clipped to the dir-entry size + } + while (span > 0) { + uint32_t contiguous; + const uint8_t *addr = supervisor_flash_xip_address(sector, &contiguous); + if (addr == NULL) { + mp_raise_OSError(MP_EOPNOTSUPP); // this build cannot map the drive here + } + FSIZE_t piece = (FSIZE_t)contiguous * FF_MIN_SS; + if (piece > span) { + piece = span; + } + // read-only: a stray write raises instead of silently hitting the flash window + mp_obj_list_append(views, mp_obj_new_memoryview('B', (size_t)piece, (void *)addr)); + span -= piece; + left -= piece; + sector += contiguous; + } + } + if (left != 0) { + mp_raise_OSError(MP_EIO); // chain shorter than the dir-entry size: corrupt + } + mp_obj_list_t *list = MP_OBJ_TO_PTR(views); + return mp_obj_new_tuple(list->len, list->items); + #else + (void)file_in; + mp_raise_type(&mp_type_NotImplementedError); // this port does not map its drive + #endif +} diff --git a/supervisor/flash.h b/supervisor/flash.h index ca3e42735ff..fc37bf5f319 100644 --- a/supervisor/flash.h +++ b/supervisor/flash.h @@ -29,6 +29,12 @@ void supervisor_flash_init_vfs(struct _fs_user_mount_t *vfs); void supervisor_flash_flush(void); void supervisor_flash_release_cache(void); +#if CIRCUITPY_STORAGE_MAP_FILE +// storage.map_file: flash address of a CIRCUITPY FatFs sector, NULL if the port cannot map the +// drive there; *contiguous receives how many sectors from it are contiguous in the mapping. +const uint8_t *supervisor_flash_xip_address(uint32_t fatfs_sector, uint32_t *contiguous); +#endif + void supervisor_flash_set_extended(bool extended); bool supervisor_flash_get_extended(void); void supervisor_flash_update_extended(void); diff --git a/supervisor/shared/flash.c b/supervisor/shared/flash.c index ccb60efe321..135ae923c24 100644 --- a/supervisor/shared/flash.c +++ b/supervisor/shared/flash.c @@ -149,6 +149,24 @@ static mp_uint_t flash_read_blocks(mp_obj_t self_in, uint8_t *dest, uint32_t blo return supervisor_flash_read_blocks(dest, block_num, num_blocks); } +#if CIRCUITPY_STORAGE_MAP_FILE +MP_WEAK const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous) { + (void)block; + (void)contiguous; + return NULL; // this port does not map its drive +} + +// The sector -> block translation of flash_read_blocks(); FatFs has already validated the +// cluster chain, so every sector it hands over lies inside the volume. +const uint8_t *supervisor_flash_xip_address(uint32_t fatfs_sector, uint32_t *contiguous) { + uint32_t block = fatfs_sector - PART1_START_BLOCK; + #if CIRCUITPY_SAVES_PARTITION_SIZE > 0 + block += CIRCUITPY_SAVES_PARTITION_SIZE / FILESYSTEM_BLOCK_SIZE; + #endif + return port_internal_flash_xip_address(block, contiguous); +} +#endif + static volatile bool filesystem_dirty = false; static mp_uint_t flash_write_blocks(mp_obj_t self_in, const uint8_t *src, uint32_t block_num, uint32_t num_blocks) { diff --git a/supervisor/shared/internal_flash.h b/supervisor/shared/internal_flash.h index 139e86cf49b..dd18f0100f1 100644 --- a/supervisor/shared/internal_flash.h +++ b/supervisor/shared/internal_flash.h @@ -8,3 +8,9 @@ #include "supervisor/internal_flash.h" // This is per-port. void port_internal_flash_flush(void); + +#if CIRCUITPY_STORAGE_MAP_FILE +// The memory-mapped address of a drive block, NULL if the port cannot map it; *contiguous +// receives how many blocks from it are contiguous in the mapping. Weak NULL default. +const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous); +#endif From a34445b41f4e7f9b7ecb73c6fc77d93bdb0d1c59 Mon Sep 17 00:00:00 2001 From: Vladimir Smitka Date: Sat, 19 Sep 2026 22:09:02 +0000 Subject: [PATCH 2/6] storage: map_file - refuse writes to a mapped file, FAT only A view's consumers (audiocore.RawSample, synthio.MidiTrack) take the pointer through the buffer protocol once, so invalidating the view on a write would not protect them. Instead the file is protected on the write side: map_file records the file's start cluster, and open() for writing or os.remove() of that file raises OSError EACCES until the next reload. Rename is allowed, the data does not move. The record is a VM root pointer, cleared with the other per-run state in cleanup_after_vm. A non-FAT file (littlefs keeps its pointers inside the data blocks) raises OSError EOPNOTSUPP instead of TypeError, so one except clause covers the fallback to read(); the docstring shows it. +280 B on pajenicko_picopad, +196 B on adafruit_feather_esp32s3_tft; 0 B where CIRCUITPY_STORAGE_MAP_FILE is off. --- extmod/vfs_fat.c | 8 +++++ extmod/vfs_fat_file.c | 9 +++++- main.c | 8 +++++ shared-bindings/storage/__init__.c | 17 ++++++++-- shared-bindings/storage/__init__.h | 5 +++ shared-module/storage/__init__.c | 51 +++++++++++++++++++++++++++++- 6 files changed, 93 insertions(+), 5 deletions(-) diff --git a/extmod/vfs_fat.c b/extmod/vfs_fat.c index 95ed79eb18b..dae1e468bfd 100644 --- a/extmod/vfs_fat.c +++ b/extmod/vfs_fat.c @@ -47,6 +47,9 @@ #include "extmod/vfs_fat.h" #include "shared/timeutils/timeutils.h" #include "supervisor/filesystem.h" +#if CIRCUITPY_STORAGE_MAP_FILE +#include "shared-bindings/storage/__init__.h" +#endif #if FF_MAX_SS == FF_MIN_SS #define SECSIZE(fs) (FF_MIN_SS) @@ -243,6 +246,11 @@ static mp_obj_t fat_vfs_remove_internal(mp_obj_t vfs_in, mp_obj_t path_in, mp_in // check if path is a file or directory if ((fno.fattrib & AM_DIR) == attr) { + #if CIRCUITPY_STORAGE_MAP_FILE + if (attr == 0) { + storage_map_file_check_writable(self, path); + } + #endif res = f_unlink(&self->fatfs, path); if (res != FR_OK) { diff --git a/extmod/vfs_fat_file.c b/extmod/vfs_fat_file.c index aee6318cacf..2d23e4c9671 100644 --- a/extmod/vfs_fat_file.c +++ b/extmod/vfs_fat_file.c @@ -37,6 +37,9 @@ #include "lib/oofatfs/ff.h" #include "extmod/vfs_fat.h" #include "supervisor/filesystem.h" +#if CIRCUITPY_STORAGE_MAP_FILE +#include "shared-bindings/storage/__init__.h" +#endif // this table converts from FRESULT to POSIX errno const byte fresult_to_errno_table[20] = { @@ -249,7 +252,11 @@ static mp_obj_t fat_vfs_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_i if ((mode & FA_WRITE) != 0 && !filesystem_is_writable_by_python(self)) { mp_raise_OSError(MP_EROFS); } - + #if CIRCUITPY_STORAGE_MAP_FILE + if ((mode & FA_WRITE) != 0) { + storage_map_file_check_writable(self, mp_obj_str_get_str(path_in)); + } + #endif pyb_file_obj_t *o = mp_obj_malloc_with_finaliser(pyb_file_obj_t, type); diff --git a/main.c b/main.c index 8b1ebd4b9fe..f028f0e5aa7 100644 --- a/main.c +++ b/main.c @@ -84,6 +84,10 @@ #include "shared-module/keypad/__init__.h" #endif +#if CIRCUITPY_STORAGE_MAP_FILE +#include "shared-bindings/storage/__init__.h" +#endif + #if CIRCUITPY_AUDIOFILEWRITER #include "shared-module/audiofilewriter/AudioFileWriter.h" #endif @@ -396,6 +400,10 @@ static void cleanup_after_vm(mp_obj_t exception) { keypad_reset(); #endif + #if CIRCUITPY_STORAGE_MAP_FILE + storage_map_file_reset(); + #endif + #if CIRCUITPY_AUDIOFILEWRITER audiofilewriter_reset(); #endif diff --git a/shared-bindings/storage/__init__.c b/shared-bindings/storage/__init__.c index b0e36dc27f3..4a1fa12327c 100644 --- a/shared-bindings/storage/__init__.c +++ b/shared-bindings/storage/__init__.c @@ -308,12 +308,23 @@ MP_DEFINE_CONST_FUN_OBJ_0(storage_enable_usb_drive_obj, storage_enable_usb_drive //| fall on cluster boundaries. The views stay valid after the file is closed, and anything that //| takes a buffer can use them: a `synthio.MidiTrack`, an `audiocore.RawSample`, a wavetable. //| -//| Treat the file as read-only while a view is in use: rewriting it shows the new bytes, and -//| mid-write it shows a torn file. +//| A mapped file is read-only to Python until the next reload: opening it for writing or +//| removing it raises ``OSError`` ``EACCES``, so the flash bytes a view (or a `synthio.MidiTrack` +//| built from one) is using cannot change under it. Renaming it is fine. A USB host can still +//| rewrite the file; with auto-reload on, that restarts the code. +//| +//| Only a FAT drive can be mapped: littlefs keeps its own pointers inside the data blocks. To +//| run on any board, fall back to reading the file:: +//| +//| try: +//| views = storage.map_file(f) +//| except (OSError, NotImplementedError): +//| views = () +//| data = views[0] if len(views) == 1 else f.read() //| //| :param typing.BinaryIO file: A file on the CIRCUITPY drive open for binary reading (``"rb"``) //| :raises OSError: ``EINVAL`` if the file is closed or not open for reading only, -//| ``EOPNOTSUPP`` if it is on another mount or this build cannot map the drive, +//| ``EOPNOTSUPP`` if it is not on a FAT CIRCUITPY drive or this build cannot map the drive, //| ``EIO`` if its cluster chain is corrupt //| :raises ~builtins.MemoryError: if ``open()`` could not allocate the file's cluster map //| :raises NotImplementedError: on a port whose drive is not memory-mapped""" diff --git a/shared-bindings/storage/__init__.h b/shared-bindings/storage/__init__.h index 12d13c2ccfc..349306eb783 100644 --- a/shared-bindings/storage/__init__.h +++ b/shared-bindings/storage/__init__.h @@ -23,3 +23,8 @@ bool common_hal_storage_unsafe_disable_usb_drive(void); bool common_hal_storage_enable_usb_drive(void); mp_obj_t common_hal_storage_map_file(mp_obj_t file); +#if CIRCUITPY_STORAGE_MAP_FILE +// Raises OSError if path on vfs is a file mapped this run: a write would change bytes in use. +void storage_map_file_check_writable(struct _fs_user_mount_t *vfs, const char *path); +void storage_map_file_reset(void); +#endif diff --git a/shared-module/storage/__init__.c b/shared-module/storage/__init__.c index 8fe286dfb34..bad1d8875e3 100644 --- a/shared-module/storage/__init__.c +++ b/shared-module/storage/__init__.c @@ -23,6 +23,7 @@ #include "lib/oofatfs/ff.h" #include "py/objarray.h" #include "py/objlist.h" +#include "py/stream.h" #endif #if CIRCUITPY_USB_DEVICE @@ -240,9 +241,56 @@ void common_hal_storage_erase_filesystem(bool extended) { // We won't actually get here, since we're resetting. } +#if CIRCUITPY_STORAGE_MAP_FILE +// Start clusters of the files mapped this run; a write to one of them is refused until reload. +MP_REGISTER_ROOT_POINTER(mp_obj_t storage_mapped_files); + +void storage_map_file_reset(void) { + MP_STATE_VM(storage_mapped_files) = MP_OBJ_NULL; +} + +static bool mapped_files_contain(DWORD sclust) { + mp_obj_list_t *mapped = MP_OBJ_TO_PTR(MP_STATE_VM(storage_mapped_files)); + for (size_t i = 0; i < mapped->len; i++) { + if (MP_OBJ_SMALL_INT_VALUE(mapped->items[i]) == (mp_int_t)sclust) { + return true; + } + } + return false; +} + +static void mapped_files_add(DWORD sclust) { + if (MP_STATE_VM(storage_mapped_files) == MP_OBJ_NULL) { + MP_STATE_VM(storage_mapped_files) = mp_obj_new_list(0, NULL); + } + if (!mapped_files_contain(sclust)) { + mp_obj_list_append(MP_STATE_VM(storage_mapped_files), MP_OBJ_NEW_SMALL_INT(sclust)); + } +} + +void storage_map_file_check_writable(fs_user_mount_t *vfs, const char *path) { + if (MP_STATE_VM(storage_mapped_files) == MP_OBJ_NULL || vfs != filesystem_circuitpy()) { + return; + } + FIL fp; + if (f_open(&vfs->fatfs, &fp, path, FA_READ) != FR_OK) { + return; // no such file yet + } + DWORD sclust = fp.obj.sclust; + f_close(&fp); + if (mapped_files_contain(sclust)) { + mp_raise_OSError(MP_EACCES); // mapped: its flash bytes are in use + } +} +#endif + mp_obj_t common_hal_storage_map_file(mp_obj_t file_in) { #if CIRCUITPY_STORAGE_MAP_FILE - pyb_file_obj_t *file = MP_OBJ_TO_PTR(mp_arg_validate_type(file_in, &mp_type_vfs_fat_fileio, MP_QSTR_file)); + mp_get_stream_raise(file_in, MP_STREAM_OP_READ); + if (!mp_obj_is_type(file_in, &mp_type_vfs_fat_fileio)) { + mp_raise_OSError(MP_EOPNOTSUPP); // only a FAT volume stores a file as flash bytes + } + pyb_file_obj_t *file = MP_OBJ_TO_PTR(file_in); FATFS *fatfs = file->fp.obj.fs; if (fatfs == NULL || (file->fp.flag & FA_WRITE)) { mp_raise_OSError(MP_EINVAL); // closed, or not open for reading only @@ -294,6 +342,7 @@ mp_obj_t common_hal_storage_map_file(mp_obj_t file_in) { if (left != 0) { mp_raise_OSError(MP_EIO); // chain shorter than the dir-entry size: corrupt } + mapped_files_add(file->fp.obj.sclust); mp_obj_list_t *list = MP_OBJ_TO_PTR(views); return mp_obj_new_tuple(list->len, list->items); #else From 59b61b1699bf2936723153fc3ebf36174c0b05d6 Mon Sep 17 00:00:00 2001 From: Vladimir Smitka Date: Sun, 20 Sep 2026 08:21:18 +0000 Subject: [PATCH 3/6] storage: map_file - shorten the docstring --- shared-bindings/storage/__init__.c | 33 +++++++++--------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/shared-bindings/storage/__init__.c b/shared-bindings/storage/__init__.c index 4a1fa12327c..6576b9a4543 100644 --- a/shared-bindings/storage/__init__.c +++ b/shared-bindings/storage/__init__.c @@ -302,31 +302,18 @@ MP_DEFINE_CONST_FUN_OBJ_0(storage_enable_usb_drive_obj, storage_enable_usb_drive //| def map_file(file: typing.BinaryIO) -> Tuple[memoryview, ...]: -//| """Map an open file on the internal CIRCUITPY drive straight from flash, without copying it -//| into RAM. Returns one read-only `memoryview` per contiguous run of the file's clusters, in -//| file order: a contiguous file gives a 1-tuple, an empty file an empty tuple. Run boundaries -//| fall on cluster boundaries. The views stay valid after the file is closed, and anything that -//| takes a buffer can use them: a `synthio.MidiTrack`, an `audiocore.RawSample`, a wavetable. -//| -//| A mapped file is read-only to Python until the next reload: opening it for writing or -//| removing it raises ``OSError`` ``EACCES``, so the flash bytes a view (or a `synthio.MidiTrack` -//| built from one) is using cannot change under it. Renaming it is fine. A USB host can still -//| rewrite the file; with auto-reload on, that restarts the code. -//| -//| Only a FAT drive can be mapped: littlefs keeps its own pointers inside the data blocks. To -//| run on any board, fall back to reading the file:: -//| -//| try: -//| views = storage.map_file(f) -//| except (OSError, NotImplementedError): -//| views = () -//| data = views[0] if len(views) == 1 else f.read() -//| -//| :param typing.BinaryIO file: A file on the CIRCUITPY drive open for binary reading (``"rb"``) +//| """Map a file on the CIRCUITPY drive straight from flash, without copying it into RAM. +//| Returns one read-only `memoryview` per contiguous run of the file's clusters, in file +//| order. The views stay valid after the file is closed. +//| +//| Until the next reload, opening the file for writing or removing it raises ``OSError`` +//| ``EACCES``. A USB host can still rewrite it. +//| +//| :param typing.BinaryIO file: A file on the CIRCUITPY drive open with ``"rb"`` //| :raises OSError: ``EINVAL`` if the file is closed or not open for reading only, -//| ``EOPNOTSUPP`` if it is not on a FAT CIRCUITPY drive or this build cannot map the drive, +//| ``EOPNOTSUPP`` if it is not on a FAT CIRCUITPY drive or the drive is not memory-mapped, //| ``EIO`` if its cluster chain is corrupt -//| :raises ~builtins.MemoryError: if ``open()`` could not allocate the file's cluster map +//| :raises ~builtins.MemoryError: if the file's cluster map could not be allocated //| :raises NotImplementedError: on a port whose drive is not memory-mapped""" //| ... static mp_obj_t storage_map_file(mp_obj_t file_in) { From a9bff16a7f84b43b01bc32efc79e5c1210ed55d9 Mon Sep 17 00:00:00 2001 From: Vladimir Smitka Date: Sun, 20 Sep 2026 08:40:58 +0000 Subject: [PATCH 4/6] storage: map_file - drop two redundant checks The type check already rejects a non-stream, and a cluster chain open() could not walk shows up as a run total shorter than the file (EIO), so the separate fp.err check was never reached. -48 B on pajenicko_picopad. --- shared-module/storage/__init__.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/shared-module/storage/__init__.c b/shared-module/storage/__init__.c index bad1d8875e3..730a9844827 100644 --- a/shared-module/storage/__init__.c +++ b/shared-module/storage/__init__.c @@ -23,7 +23,6 @@ #include "lib/oofatfs/ff.h" #include "py/objarray.h" #include "py/objlist.h" -#include "py/stream.h" #endif #if CIRCUITPY_USB_DEVICE @@ -286,7 +285,6 @@ void storage_map_file_check_writable(fs_user_mount_t *vfs, const char *path) { mp_obj_t common_hal_storage_map_file(mp_obj_t file_in) { #if CIRCUITPY_STORAGE_MAP_FILE - mp_get_stream_raise(file_in, MP_STREAM_OP_READ); if (!mp_obj_is_type(file_in, &mp_type_vfs_fat_fileio)) { mp_raise_OSError(MP_EOPNOTSUPP); // only a FAT volume stores a file as flash bytes } @@ -299,9 +297,6 @@ mp_obj_t common_hal_storage_map_file(mp_obj_t file_in) { if (drive == NULL || fatfs != &drive->fatfs) { mp_raise_OSError(MP_EOPNOTSUPP); // another mount (an SD card): not memory-mapped } - if (file->fp.err) { - mp_raise_OSError(fresult_to_errno_table[file->fp.err]); // open() hit a bad cluster chain - } DWORD *tbl = file->fp.cltbl; if (tbl == NULL) { mp_raise_type(&mp_type_MemoryError); // open() could not allocate the cluster map From b4fda7cd8af0f8acfdd731f5a1420660cf473721 Mon Sep 17 00:00:00 2001 From: Vladimir Smitka Date: Sun, 20 Sep 2026 10:01:27 +0000 Subject: [PATCH 5/6] storage: map_file - forward-declare fs_user_mount_t in the header Boards without USB MSC (esp32, esp32c3, esp32c6) do not include extmod/vfs_fat.h before this header, so the struct in the prototype was scoped to the parameter list and the definition failed with conflicting types under -Werror. --- shared-bindings/storage/__init__.h | 1 + 1 file changed, 1 insertion(+) diff --git a/shared-bindings/storage/__init__.h b/shared-bindings/storage/__init__.h index 349306eb783..f97c99a3356 100644 --- a/shared-bindings/storage/__init__.h +++ b/shared-bindings/storage/__init__.h @@ -24,6 +24,7 @@ bool common_hal_storage_enable_usb_drive(void); mp_obj_t common_hal_storage_map_file(mp_obj_t file); #if CIRCUITPY_STORAGE_MAP_FILE +struct _fs_user_mount_t; // Raises OSError if path on vfs is a file mapped this run: a write would change bytes in use. void storage_map_file_check_writable(struct _fs_user_mount_t *vfs, const char *path); void storage_map_file_reset(void); From cdc6966e955270de501444ccc10cfdcaac3ff31f Mon Sep 17 00:00:00 2001 From: Vladimir Smitka Date: Mon, 21 Sep 2026 20:10:36 +0000 Subject: [PATCH 6/6] storage: map_file - require the port hook instead of a weak default A port that sets CIRCUITPY_STORAGE_MAP_FILE now has to define port_internal_flash_xip_address(); there is no weak fallback returning NULL. Both ports that set it - raspberrypi and espressif - already define it, and espressif still returns NULL for a partition it could not map, so the caller's NULL path keeps its meaning. Setting the flag without the function is a link error rather than a module that builds and then refuses every call at runtime. --- py/circuitpy_mpconfig.mk | 5 +++-- supervisor/shared/flash.c | 6 ------ supervisor/shared/internal_flash.h | 3 ++- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk index 68950c920e6..1aeb4c1b761 100755 --- a/py/circuitpy_mpconfig.mk +++ b/py/circuitpy_mpconfig.mk @@ -636,8 +636,9 @@ CFLAGS += -DCIRCUITPY_STORAGE=$(CIRCUITPY_STORAGE) CIRCUITPY_STORAGE_EXTEND ?= $(CIRCUITPY_DUALBANK) CFLAGS += -DCIRCUITPY_STORAGE_EXTEND=$(CIRCUITPY_STORAGE_EXTEND) -# storage.map_file(): read a file straight out of memory-mapped flash. Ports whose CIRCUITPY drive -# is memory-mapped implement port_internal_flash_xip_address() and turn it on. +# storage.map_file(): read a file straight out of memory-mapped flash. A port whose CIRCUITPY drive +# is memory-mapped implements port_internal_flash_xip_address() and turns this on; turning it on +# without that function is a link error, not a silent no-op. CIRCUITPY_STORAGE_MAP_FILE ?= 0 CFLAGS += -DCIRCUITPY_STORAGE_MAP_FILE=$(CIRCUITPY_STORAGE_MAP_FILE) diff --git a/supervisor/shared/flash.c b/supervisor/shared/flash.c index 135ae923c24..f0d47a1cc37 100644 --- a/supervisor/shared/flash.c +++ b/supervisor/shared/flash.c @@ -150,12 +150,6 @@ static mp_uint_t flash_read_blocks(mp_obj_t self_in, uint8_t *dest, uint32_t blo } #if CIRCUITPY_STORAGE_MAP_FILE -MP_WEAK const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous) { - (void)block; - (void)contiguous; - return NULL; // this port does not map its drive -} - // The sector -> block translation of flash_read_blocks(); FatFs has already validated the // cluster chain, so every sector it hands over lies inside the volume. const uint8_t *supervisor_flash_xip_address(uint32_t fatfs_sector, uint32_t *contiguous) { diff --git a/supervisor/shared/internal_flash.h b/supervisor/shared/internal_flash.h index dd18f0100f1..df3d2e9f6dd 100644 --- a/supervisor/shared/internal_flash.h +++ b/supervisor/shared/internal_flash.h @@ -11,6 +11,7 @@ void port_internal_flash_flush(void); #if CIRCUITPY_STORAGE_MAP_FILE // The memory-mapped address of a drive block, NULL if the port cannot map it; *contiguous -// receives how many blocks from it are contiguous in the mapping. Weak NULL default. +// receives how many blocks from it are contiguous in the mapping. A port that sets +// CIRCUITPY_STORAGE_MAP_FILE implements this. const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous); #endif