diff --git a/extmod/vfs_fat.c b/extmod/vfs_fat.c index 95ed79eb18b..dae1e468bfd 100644 --- a/extmod/vfs_fat.c +++ b/extmod/vfs_fat.c @@ -47,6 +47,9 @@ #include "extmod/vfs_fat.h" #include "shared/timeutils/timeutils.h" #include "supervisor/filesystem.h" +#if CIRCUITPY_STORAGE_MAP_FILE +#include "shared-bindings/storage/__init__.h" +#endif #if FF_MAX_SS == FF_MIN_SS #define SECSIZE(fs) (FF_MIN_SS) @@ -243,6 +246,11 @@ static mp_obj_t fat_vfs_remove_internal(mp_obj_t vfs_in, mp_obj_t path_in, mp_in // check if path is a file or directory if ((fno.fattrib & AM_DIR) == attr) { + #if CIRCUITPY_STORAGE_MAP_FILE + if (attr == 0) { + storage_map_file_check_writable(self, path); + } + #endif res = f_unlink(&self->fatfs, path); if (res != FR_OK) { diff --git a/extmod/vfs_fat_file.c b/extmod/vfs_fat_file.c index aee6318cacf..2d23e4c9671 100644 --- a/extmod/vfs_fat_file.c +++ b/extmod/vfs_fat_file.c @@ -37,6 +37,9 @@ #include "lib/oofatfs/ff.h" #include "extmod/vfs_fat.h" #include "supervisor/filesystem.h" +#if CIRCUITPY_STORAGE_MAP_FILE +#include "shared-bindings/storage/__init__.h" +#endif // this table converts from FRESULT to POSIX errno const byte fresult_to_errno_table[20] = { @@ -249,7 +252,11 @@ static mp_obj_t fat_vfs_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_i if ((mode & FA_WRITE) != 0 && !filesystem_is_writable_by_python(self)) { mp_raise_OSError(MP_EROFS); } - + #if CIRCUITPY_STORAGE_MAP_FILE + if ((mode & FA_WRITE) != 0) { + storage_map_file_check_writable(self, mp_obj_str_get_str(path_in)); + } + #endif pyb_file_obj_t *o = mp_obj_malloc_with_finaliser(pyb_file_obj_t, type); diff --git a/main.c b/main.c index 8b1ebd4b9fe..f028f0e5aa7 100644 --- a/main.c +++ b/main.c @@ -84,6 +84,10 @@ #include "shared-module/keypad/__init__.h" #endif +#if CIRCUITPY_STORAGE_MAP_FILE +#include "shared-bindings/storage/__init__.h" +#endif + #if CIRCUITPY_AUDIOFILEWRITER #include "shared-module/audiofilewriter/AudioFileWriter.h" #endif @@ -396,6 +400,10 @@ static void cleanup_after_vm(mp_obj_t exception) { keypad_reset(); #endif + #if CIRCUITPY_STORAGE_MAP_FILE + storage_map_file_reset(); + #endif + #if CIRCUITPY_AUDIOFILEWRITER audiofilewriter_reset(); #endif diff --git a/ports/espressif/mpconfigport.mk b/ports/espressif/mpconfigport.mk index 5c3c5c0bc74..447aa3da6aa 100644 --- a/ports/espressif/mpconfigport.mk +++ b/ports/espressif/mpconfigport.mk @@ -479,3 +479,6 @@ endif # Usually lots of flash space available CIRCUITPY_MESSAGE_COMPRESSION_LEVEL ?= 1 + +# The CIRCUITPY partition is mapped into the data address space on first use +CIRCUITPY_STORAGE_MAP_FILE ?= 1 diff --git a/ports/espressif/supervisor/internal_flash.c b/ports/espressif/supervisor/internal_flash.c index a18d3fe0a0f..cebeb4f89bb 100644 --- a/ports/espressif/supervisor/internal_flash.c +++ b/ports/espressif/supervisor/internal_flash.c @@ -80,6 +80,43 @@ uint32_t supervisor_flash_get_block_count(void) { void port_internal_flash_flush(void) { } +#if CIRCUITPY_STORAGE_MAP_FILE +// storage.map_file: each drive partition mapped into the data address space on first use. With +// extended storage the drive spans two partitions that need not be adjacent in the mapping, so +// *contiguous stops at the seam and the caller splits a cluster run there. +static const uint8_t *map_partition(size_t i) { + static const uint8_t *base[2]; + if (base[i] == NULL) { + const void *p; + esp_partition_mmap_handle_t handle; // stays mapped for the session + if (esp_partition_mmap(_partition[i], 0, _partition[i]->size, ESP_PARTITION_MMAP_DATA, + &p, &handle) != ESP_OK) { + return NULL; + } + base[i] = p; + } + return base[i]; +} + +const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous) { + size_t i = 0; + uint32_t blocks = _partition[0]->size / FILESYSTEM_BLOCK_SIZE; + #if CIRCUITPY_STORAGE_EXTEND + if (storage_extended && block >= blocks) { + block -= blocks; + blocks = _partition[1]->size / FILESYSTEM_BLOCK_SIZE; + i = 1; + } + #endif + const uint8_t *base = map_partition(i); + if (base == NULL) { + return NULL; + } + *contiguous = blocks - block; + return base + block * FILESYSTEM_BLOCK_SIZE; +} +#endif + static void single_partition_rw(const esp_partition_t *partition, uint8_t *data, const uint32_t offset, const uint32_t size_total, const bool op) { if (op == OP_READ) { diff --git a/ports/raspberrypi/mpconfigport.mk b/ports/raspberrypi/mpconfigport.mk index 551ab00ab47..851e2398b2b 100644 --- a/ports/raspberrypi/mpconfigport.mk +++ b/ports/raspberrypi/mpconfigport.mk @@ -26,6 +26,8 @@ CIRCUITPY_PWMIO ?= 1 CIRCUITPY_RGBMATRIX ?= $(CIRCUITPY_DISPLAYIO) CIRCUITPY_ROTARYIO ?= 1 CIRCUITPY_ROTARYIO_SOFTENCODER = 1 +# The CIRCUITPY drive is execute-in-place on every RP2 board +CIRCUITPY_STORAGE_MAP_FILE ?= 1 CIRCUITPY_SYNTHIO_MAX_CHANNELS = 24 CIRCUITPY_USB_HOST ?= 1 CIRCUITPY_USB_VIDEO ?= 1 diff --git a/ports/raspberrypi/supervisor/internal_flash.c b/ports/raspberrypi/supervisor/internal_flash.c index 07b15564c60..e9da8a39a14 100644 --- a/ports/raspberrypi/supervisor/internal_flash.c +++ b/ports/raspberrypi/supervisor/internal_flash.c @@ -149,3 +149,11 @@ mp_uint_t supervisor_flash_write_blocks(const uint8_t *src, uint32_t lba, uint32 void supervisor_flash_release_cache(void) { } + +#if CIRCUITPY_STORAGE_MAP_FILE +// storage.map_file: the CIRCUITPY drive is execute-in-place on every RP2 board. +const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous) { + *contiguous = supervisor_flash_get_block_count() - block; + return (const uint8_t *)(XIP_BASE + CIRCUITPY_CIRCUITPY_DRIVE_START_ADDR + block * FILESYSTEM_BLOCK_SIZE); +} +#endif diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk index b5b0921c4ac..1aeb4c1b761 100755 --- a/py/circuitpy_mpconfig.mk +++ b/py/circuitpy_mpconfig.mk @@ -636,6 +636,12 @@ CFLAGS += -DCIRCUITPY_STORAGE=$(CIRCUITPY_STORAGE) CIRCUITPY_STORAGE_EXTEND ?= $(CIRCUITPY_DUALBANK) CFLAGS += -DCIRCUITPY_STORAGE_EXTEND=$(CIRCUITPY_STORAGE_EXTEND) +# storage.map_file(): read a file straight out of memory-mapped flash. A port whose CIRCUITPY drive +# is memory-mapped implements port_internal_flash_xip_address() and turns this on; turning it on +# without that function is a link error, not a silent no-op. +CIRCUITPY_STORAGE_MAP_FILE ?= 0 +CFLAGS += -DCIRCUITPY_STORAGE_MAP_FILE=$(CIRCUITPY_STORAGE_MAP_FILE) + CIRCUITPY_STRUCT ?= 1 CFLAGS += -DCIRCUITPY_STRUCT=$(CIRCUITPY_STRUCT) diff --git a/shared-bindings/storage/__init__.c b/shared-bindings/storage/__init__.c index b5f19529be1..6576b9a4543 100644 --- a/shared-bindings/storage/__init__.c +++ b/shared-bindings/storage/__init__.c @@ -300,6 +300,27 @@ static mp_obj_t storage_enable_usb_drive(void) { } MP_DEFINE_CONST_FUN_OBJ_0(storage_enable_usb_drive_obj, storage_enable_usb_drive); + +//| def map_file(file: typing.BinaryIO) -> Tuple[memoryview, ...]: +//| """Map a file on the CIRCUITPY drive straight from flash, without copying it into RAM. +//| Returns one read-only `memoryview` per contiguous run of the file's clusters, in file +//| order. The views stay valid after the file is closed. +//| +//| Until the next reload, opening the file for writing or removing it raises ``OSError`` +//| ``EACCES``. A USB host can still rewrite it. +//| +//| :param typing.BinaryIO file: A file on the CIRCUITPY drive open with ``"rb"`` +//| :raises OSError: ``EINVAL`` if the file is closed or not open for reading only, +//| ``EOPNOTSUPP`` if it is not on a FAT CIRCUITPY drive or the drive is not memory-mapped, +//| ``EIO`` if its cluster chain is corrupt +//| :raises ~builtins.MemoryError: if the file's cluster map could not be allocated +//| :raises NotImplementedError: on a port whose drive is not memory-mapped""" +//| ... +static mp_obj_t storage_map_file(mp_obj_t file_in) { + return common_hal_storage_map_file(file_in); +} +MP_DEFINE_CONST_FUN_OBJ_1(storage_map_file_obj, storage_map_file); + static const mp_rom_map_elem_t storage_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_storage) }, @@ -307,6 +328,7 @@ static const mp_rom_map_elem_t storage_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&storage_umount_obj) }, { MP_ROM_QSTR(MP_QSTR_remount), MP_ROM_PTR(&storage_remount_obj) }, { MP_ROM_QSTR(MP_QSTR_getmount), MP_ROM_PTR(&storage_getmount_obj) }, + { MP_ROM_QSTR(MP_QSTR_map_file), MP_ROM_PTR(&storage_map_file_obj) }, { MP_ROM_QSTR(MP_QSTR_erase_filesystem), MP_ROM_PTR(&storage_erase_filesystem_obj) }, { MP_ROM_QSTR(MP_QSTR_disable_usb_drive), MP_ROM_PTR(&storage_disable_usb_drive_obj) }, { MP_ROM_QSTR(MP_QSTR_enable_usb_drive), MP_ROM_PTR(&storage_enable_usb_drive_obj) }, diff --git a/shared-bindings/storage/__init__.h b/shared-bindings/storage/__init__.h index 0e53c78b153..f97c99a3356 100644 --- a/shared-bindings/storage/__init__.h +++ b/shared-bindings/storage/__init__.h @@ -21,3 +21,11 @@ MP_NORETURN void common_hal_storage_erase_filesystem(bool extended); bool common_hal_storage_disable_usb_drive(void); bool common_hal_storage_unsafe_disable_usb_drive(void); bool common_hal_storage_enable_usb_drive(void); + +mp_obj_t common_hal_storage_map_file(mp_obj_t file); +#if CIRCUITPY_STORAGE_MAP_FILE +struct _fs_user_mount_t; +// Raises OSError if path on vfs is a file mapped this run: a write would change bytes in use. +void storage_map_file_check_writable(struct _fs_user_mount_t *vfs, const char *path); +void storage_map_file_reset(void); +#endif diff --git a/shared-module/storage/__init__.c b/shared-module/storage/__init__.c index 1522136332a..730a9844827 100644 --- a/shared-module/storage/__init__.c +++ b/shared-module/storage/__init__.c @@ -18,6 +18,12 @@ #include "shared-bindings/storage/__init__.h" #include "supervisor/filesystem.h" #include "supervisor/flash.h" +#if CIRCUITPY_STORAGE_MAP_FILE +#include "extmod/vfs_fat.h" +#include "lib/oofatfs/ff.h" +#include "py/objarray.h" +#include "py/objlist.h" +#endif #if CIRCUITPY_USB_DEVICE #include "supervisor/usb.h" @@ -233,3 +239,109 @@ void common_hal_storage_erase_filesystem(bool extended) { common_hal_mcu_reset(); // We won't actually get here, since we're resetting. } + +#if CIRCUITPY_STORAGE_MAP_FILE +// Start clusters of the files mapped this run; a write to one of them is refused until reload. +MP_REGISTER_ROOT_POINTER(mp_obj_t storage_mapped_files); + +void storage_map_file_reset(void) { + MP_STATE_VM(storage_mapped_files) = MP_OBJ_NULL; +} + +static bool mapped_files_contain(DWORD sclust) { + mp_obj_list_t *mapped = MP_OBJ_TO_PTR(MP_STATE_VM(storage_mapped_files)); + for (size_t i = 0; i < mapped->len; i++) { + if (MP_OBJ_SMALL_INT_VALUE(mapped->items[i]) == (mp_int_t)sclust) { + return true; + } + } + return false; +} + +static void mapped_files_add(DWORD sclust) { + if (MP_STATE_VM(storage_mapped_files) == MP_OBJ_NULL) { + MP_STATE_VM(storage_mapped_files) = mp_obj_new_list(0, NULL); + } + if (!mapped_files_contain(sclust)) { + mp_obj_list_append(MP_STATE_VM(storage_mapped_files), MP_OBJ_NEW_SMALL_INT(sclust)); + } +} + +void storage_map_file_check_writable(fs_user_mount_t *vfs, const char *path) { + if (MP_STATE_VM(storage_mapped_files) == MP_OBJ_NULL || vfs != filesystem_circuitpy()) { + return; + } + FIL fp; + if (f_open(&vfs->fatfs, &fp, path, FA_READ) != FR_OK) { + return; // no such file yet + } + DWORD sclust = fp.obj.sclust; + f_close(&fp); + if (mapped_files_contain(sclust)) { + mp_raise_OSError(MP_EACCES); // mapped: its flash bytes are in use + } +} +#endif + +mp_obj_t common_hal_storage_map_file(mp_obj_t file_in) { + #if CIRCUITPY_STORAGE_MAP_FILE + if (!mp_obj_is_type(file_in, &mp_type_vfs_fat_fileio)) { + mp_raise_OSError(MP_EOPNOTSUPP); // only a FAT volume stores a file as flash bytes + } + pyb_file_obj_t *file = MP_OBJ_TO_PTR(file_in); + FATFS *fatfs = file->fp.obj.fs; + if (fatfs == NULL || (file->fp.flag & FA_WRITE)) { + mp_raise_OSError(MP_EINVAL); // closed, or not open for reading only + } + fs_user_mount_t *drive = filesystem_circuitpy(); + if (drive == NULL || fatfs != &drive->fatfs) { + mp_raise_OSError(MP_EOPNOTSUPP); // another mount (an SD card): not memory-mapped + } + DWORD *tbl = file->fp.cltbl; + if (tbl == NULL) { + mp_raise_type(&mp_type_MemoryError); // open() could not allocate the cluster map + } + FSIZE_t size = f_size(&file->fp); + if (size == 0) { + return mp_const_empty_tuple; + } + supervisor_flash_flush(); // write back any RAM sector cache + // Walk the cluster runs of the link map open() built and split a run wherever the port's + // mapping is not contiguous, e.g. at the seam of a drive that spans two flash partitions. + mp_obj_t views = mp_obj_new_list(0, NULL); + size_t nruns = (size_t)((tbl[0] - 2) / 2); + FSIZE_t left = size; + for (size_t i = 0; i < nruns && left > 0; i++) { + DWORD sector = fatfs->database + (tbl[2 + 2 * i] - 2) * fatfs->csize; + FSIZE_t span = (FSIZE_t)tbl[1 + 2 * i] * fatfs->csize * FF_MIN_SS; + if (span > left) { + span = left; // the last run is clipped to the dir-entry size + } + while (span > 0) { + uint32_t contiguous; + const uint8_t *addr = supervisor_flash_xip_address(sector, &contiguous); + if (addr == NULL) { + mp_raise_OSError(MP_EOPNOTSUPP); // this build cannot map the drive here + } + FSIZE_t piece = (FSIZE_t)contiguous * FF_MIN_SS; + if (piece > span) { + piece = span; + } + // read-only: a stray write raises instead of silently hitting the flash window + mp_obj_list_append(views, mp_obj_new_memoryview('B', (size_t)piece, (void *)addr)); + span -= piece; + left -= piece; + sector += contiguous; + } + } + if (left != 0) { + mp_raise_OSError(MP_EIO); // chain shorter than the dir-entry size: corrupt + } + mapped_files_add(file->fp.obj.sclust); + mp_obj_list_t *list = MP_OBJ_TO_PTR(views); + return mp_obj_new_tuple(list->len, list->items); + #else + (void)file_in; + mp_raise_type(&mp_type_NotImplementedError); // this port does not map its drive + #endif +} diff --git a/supervisor/flash.h b/supervisor/flash.h index ca3e42735ff..fc37bf5f319 100644 --- a/supervisor/flash.h +++ b/supervisor/flash.h @@ -29,6 +29,12 @@ void supervisor_flash_init_vfs(struct _fs_user_mount_t *vfs); void supervisor_flash_flush(void); void supervisor_flash_release_cache(void); +#if CIRCUITPY_STORAGE_MAP_FILE +// storage.map_file: flash address of a CIRCUITPY FatFs sector, NULL if the port cannot map the +// drive there; *contiguous receives how many sectors from it are contiguous in the mapping. +const uint8_t *supervisor_flash_xip_address(uint32_t fatfs_sector, uint32_t *contiguous); +#endif + void supervisor_flash_set_extended(bool extended); bool supervisor_flash_get_extended(void); void supervisor_flash_update_extended(void); diff --git a/supervisor/shared/flash.c b/supervisor/shared/flash.c index ccb60efe321..f0d47a1cc37 100644 --- a/supervisor/shared/flash.c +++ b/supervisor/shared/flash.c @@ -149,6 +149,18 @@ static mp_uint_t flash_read_blocks(mp_obj_t self_in, uint8_t *dest, uint32_t blo return supervisor_flash_read_blocks(dest, block_num, num_blocks); } +#if CIRCUITPY_STORAGE_MAP_FILE +// The sector -> block translation of flash_read_blocks(); FatFs has already validated the +// cluster chain, so every sector it hands over lies inside the volume. +const uint8_t *supervisor_flash_xip_address(uint32_t fatfs_sector, uint32_t *contiguous) { + uint32_t block = fatfs_sector - PART1_START_BLOCK; + #if CIRCUITPY_SAVES_PARTITION_SIZE > 0 + block += CIRCUITPY_SAVES_PARTITION_SIZE / FILESYSTEM_BLOCK_SIZE; + #endif + return port_internal_flash_xip_address(block, contiguous); +} +#endif + static volatile bool filesystem_dirty = false; static mp_uint_t flash_write_blocks(mp_obj_t self_in, const uint8_t *src, uint32_t block_num, uint32_t num_blocks) { diff --git a/supervisor/shared/internal_flash.h b/supervisor/shared/internal_flash.h index 139e86cf49b..df3d2e9f6dd 100644 --- a/supervisor/shared/internal_flash.h +++ b/supervisor/shared/internal_flash.h @@ -8,3 +8,10 @@ #include "supervisor/internal_flash.h" // This is per-port. void port_internal_flash_flush(void); + +#if CIRCUITPY_STORAGE_MAP_FILE +// The memory-mapped address of a drive block, NULL if the port cannot map it; *contiguous +// receives how many blocks from it are contiguous in the mapping. A port that sets +// CIRCUITPY_STORAGE_MAP_FILE implements this. +const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous); +#endif