From 9fe7acae631780bab4d349bc9331bb36ccfd5ef4 Mon Sep 17 00:00:00 2001
From: Norbert Orzechowicz
Date: Sat, 11 Jul 2026 13:51:37 +0200
Subject: [PATCH 1/5] feat: Floe binary row serialization format
- new Flow\Floe binary format: writer, reader, merger, schema evolution,
extractor/loader + DSL
- new flow-php/flow-php-ext Rust extension for native Floe
encode/decode, with PHP fallback parity tests
- cache layer rewritten on Floe: CacheIndex, streaming FilesystemCache,
external sort buckets
- CI workflow, nix packaging and codecov flag for the extension
- documentation for Floe format and extension, upgrade notes
---
.codecov.yaml | 2 +
.github/workflows/job-flow-php-extension.yml | 192 +++
.github/workflows/monorepo-split.yml | 2 +
.github/workflows/test-suite.yml | 5 +
.nix/pkgs/flow-php/package.nix | 3 +
.nix/pkgs/php-flow-php-ext/package.nix | 66 +
Justfile | 4 +-
bin/docs.php | 1 +
composer.json | 4 +-
documentation/components/core/caching.md | 51 +-
documentation/components/core/core.md | 1 +
documentation/components/core/floe.md | 290 ++++
.../components/extensions/flow-php-ext.md | 77 +
.../installation/packages/flow-php-ext.md | 60 +
documentation/upgrading.md | 26 +
manifest.json | 8 +
phpunit.xml.dist | 3 +
shell.nix | 7 +-
src/core/etl/composer.json | 3 +-
.../etl/src/Flow/ETL/Attribute/Module.php | 1 +
src/core/etl/src/Flow/ETL/Cache.php | 13 +-
.../etl/src/Flow/ETL/Cache/CacheIndex.php | 40 +
.../Cache/Implementation/FilesystemCache.php | 85 +-
.../Cache/Implementation/InMemoryCache.php | 19 +-
.../Cache/Implementation/PSRSimpleCache.php | 31 +-
.../Cache/Implementation/TraceableCache.php | 38 +-
.../src/Flow/ETL/Config/Cache/CacheConfig.php | 2 +-
.../ETL/Config/Cache/CacheConfigBuilder.php | 32 +-
.../etl/src/Flow/ETL/Config/ConfigBuilder.php | 19 +-
src/core/etl/src/Flow/ETL/DSL/functions.php | 13 +-
.../src/Flow/ETL/Extractor/CacheExtractor.php | 14 +-
.../Flow/ETL/Processor/CachingProcessor.php | 2 +-
.../ETL/Processor/PartitioningProcessor.php | 2 +-
.../Flow/ETL/Processor/SortingProcessor.php | 4 +-
src/core/etl/src/Flow/ETL/Row/Entries.php | 16 +-
src/core/etl/src/Flow/ETL/Rows.php | 33 +-
.../BucketsCache/FilesystemBucketsCache.php | 57 +-
.../ETL/Transformer/SerializeTransformer.php | 7 +-
.../Transformer/UnserializeTransformer.php | 12 +-
src/core/etl/src/Flow/Floe/Codec.php | 18 +
.../etl/src/Flow/Floe/Codec/NoopCodec.php | 25 +
src/core/etl/src/Flow/Floe/DSL/functions.php | 58 +
.../src/Flow/Floe/Decoding/BooleanDecoder.php | 13 +
.../Flow/Floe/Decoding/DateTimeDecoder.php | 56 +
.../src/Flow/Floe/Decoding/DynamicDecoder.php | 81 +
.../src/Flow/Floe/Decoding/EnumDecoder.php | 39 +
.../src/Flow/Floe/Decoding/Float64Decoder.php | 18 +
.../Floe/Decoding/HtmlDocumentDecoder.php | 22 +
.../Flow/Floe/Decoding/HtmlElementDecoder.php | 22 +
.../src/Flow/Floe/Decoding/Int64Decoder.php | 18 +
.../Flow/Floe/Decoding/IntervalDecoder.php | 42 +
.../src/Flow/Floe/Decoding/JsonDecoder.php | 51 +
.../src/Flow/Floe/Decoding/ListDecoder.php | 30 +
.../etl/src/Flow/Floe/Decoding/MapDecoder.php | 33 +
.../src/Flow/Floe/Decoding/NullDecoder.php | 13 +
.../Flow/Floe/Decoding/OptionalDecoder.php | 25 +
.../Flow/Floe/Decoding/PackedListDecoder.php | 37 +
.../src/Flow/Floe/Decoding/StringDecoder.php | 20 +
.../Flow/Floe/Decoding/StructureDecoder.php | 59 +
.../Flow/Floe/Decoding/TimeZoneDecoder.php | 26 +
.../etl/src/Flow/Floe/Decoding/TimeZones.php | 20 +
.../src/Flow/Floe/Decoding/UuidDecoder.php | 47 +
.../src/Flow/Floe/Decoding/ValueDecoder.php | 13 +
.../Flow/Floe/Decoding/XmlDocumentDecoder.php | 23 +
.../Flow/Floe/Decoding/XmlElementDecoder.php | 23 +
src/core/etl/src/Flow/Floe/EncoderColumn.php | 17 +
src/core/etl/src/Flow/Floe/EncoderPlan.php | 17 +
.../src/Flow/Floe/Encoding/BooleanEncoder.php | 13 +
.../Flow/Floe/Encoding/DateTimeEncoder.php | 41 +
.../src/Flow/Floe/Encoding/DynamicEncoder.php | 90 +
.../src/Flow/Floe/Encoding/EnumEncoder.php | 19 +
.../src/Flow/Floe/Encoding/Float64Encoder.php | 15 +
.../Floe/Encoding/HtmlDocumentEncoder.php | 17 +
.../Flow/Floe/Encoding/HtmlElementEncoder.php | 16 +
.../src/Flow/Floe/Encoding/Int64Encoder.php | 15 +
.../Flow/Floe/Encoding/IntervalEncoder.php | 21 +
.../src/Flow/Floe/Encoding/JsonEncoder.php | 19 +
.../src/Flow/Floe/Encoding/ListEncoder.php | 28 +
.../etl/src/Flow/Floe/Encoding/MapEncoder.php | 29 +
.../src/Flow/Floe/Encoding/NullEncoder.php | 13 +
.../Flow/Floe/Encoding/OptionalEncoder.php | 19 +
.../Flow/Floe/Encoding/PackedListEncoder.php | 28 +
.../src/Flow/Floe/Encoding/StringEncoder.php | 17 +
.../Flow/Floe/Encoding/StringKeyEncoder.php | 21 +
.../Flow/Floe/Encoding/StructureEncoder.php | 53 +
.../Flow/Floe/Encoding/TimeZoneEncoder.php | 19 +
.../src/Flow/Floe/Encoding/UuidEncoder.php | 14 +
.../src/Flow/Floe/Encoding/ValueEncoder.php | 14 +
.../Flow/Floe/Encoding/XmlDocumentEncoder.php | 16 +
.../Flow/Floe/Encoding/XmlElementEncoder.php | 16 +
src/core/etl/src/Flow/Floe/EntryFactory.php | 63 +
.../etl/src/Flow/Floe/EntryInstantiator.php | 29 +
.../src/Flow/Floe/Exception/FloeException.php | 9 +
.../Exception/IncompatibleSchemaException.php | 7 +
.../etl/src/Flow/Floe/ExtRowFrameEncoder.php | 28 +
src/core/etl/src/Flow/Floe/FloeExtractor.php | 123 ++
src/core/etl/src/Flow/Floe/FloeFile.php | 1022 ++++++++++++
src/core/etl/src/Flow/Floe/FloeLoader.php | 76 +
src/core/etl/src/Flow/Floe/FloeMerger.php | 364 ++++
src/core/etl/src/Flow/Floe/FloeReader.php | 39 +
src/core/etl/src/Flow/Floe/FloeSerializer.php | 62 +
.../etl/src/Flow/Floe/FloeValueSerializer.php | 114 ++
src/core/etl/src/Flow/Floe/FloeWriter.php | 474 ++++++
src/core/etl/src/Flow/Floe/Footer.php | 144 ++
src/core/etl/src/Flow/Floe/Format.php | 151 ++
src/core/etl/src/Flow/Floe/FrameReader.php | 103 ++
src/core/etl/src/Flow/Floe/FrameSegment.php | 14 +
src/core/etl/src/Flow/Floe/HydratorColumn.php | 24 +
.../etl/src/Flow/Floe/PhpRowFrameEncoder.php | 55 +
src/core/etl/src/Flow/Floe/RowEncoder.php | 79 +
.../etl/src/Flow/Floe/RowFrameEncoder.php | 18 +
src/core/etl/src/Flow/Floe/RowHydrator.php | 50 +
src/core/etl/src/Flow/Floe/RowPadding.php | 59 +
.../etl/src/Flow/Floe/RowsValueMapper.php | 105 ++
src/core/etl/src/Flow/Floe/SchemaDecoder.php | 141 ++
src/core/etl/src/Flow/Floe/SchemaEncoder.php | 57 +
.../etl/src/Flow/Floe/SchemaEvolution.php | 60 +
src/core/etl/src/Flow/Floe/SchemaTracker.php | 84 +
src/core/etl/src/Flow/Floe/Section.php | 53 +
src/core/etl/src/Flow/Floe/ValueDecoder.php | 247 +++
src/core/etl/src/Flow/Floe/ValueEncoder.php | 206 +++
.../Double/FakeRandomOrdersExtractor.php | 4 +-
.../Double/FakeStaticOrdersExtractor.php | 7 +-
.../ETL/Tests/Fixtures/CustomDateTime.php | 9 +
.../ETL/Tests/FlowIntegrationTestCase.php | 5 +-
.../Integration/Cache/CacheBaseTestSuite.php | 130 --
.../Tests/Integration/Cache/CacheTestCase.php | 196 +++
.../Cache/FilesystemCacheStreamingTest.php | 120 ++
.../Integration/Cache/FilesystemCacheTest.php | 39 +
.../Cache/FilesystemCacheTestSuite.php | 18 -
...cheTestSuite.php => InMemoryCacheTest.php} | 2 +-
.../Cache/PSRSimpleFilesystemCacheTest.php | 38 +
.../PSRSimpleFilesystemCacheTestSuite.php | 20 -
.../Cache/PSRSimpleRedisCacheTest.php | 60 +
.../Cache/PSRSimpleRedisCacheTestSuite.php | 29 -
...heTestSuite.php => TraceableCacheTest.php} | 2 +-
.../Tests/Integration/DataFrame/CacheTest.php | 38 +-
.../DataFrame/ConfigBuilderTest.php | 35 +-
.../Extractor/CacheExtractorTest.php | 84 +-
.../Sort/ExternalSort/ExternalSortTest.php | 2 +-
.../FilesystemBucketsCacheTest.php | 81 +
.../ETL/Tests/Unit/Cache/CacheIndexTest.php | 78 +
.../Implementation/TraceableCacheTest.php | 90 +-
.../Unit/Processor/CachingProcessorTest.php | 15 +-
.../Flow/ETL/Tests/Unit/Row/EntriesTest.php | 33 +
.../Transformer/SerializeTransformerTest.php | 12 +-
.../UnserializeTransformerTest.php | 14 +-
.../Floe/Tests/Context/FloeFileContext.php | 151 ++
.../Flow/Floe/Tests/Double/CodecStub.php | 29 +
.../Floe/Tests/Double/PrefixingCodecStub.php | 34 +
.../Floe/Tests/Double/UnsizedFilesystem.php | 67 +
.../Floe/Tests/Double/UnsizedSourceStream.php | 56 +
.../Tests/Integration/FloeDataFrameTest.php | 123 ++
.../Floe/Tests/Integration/FloeFileTest.php | 158 ++
.../Tests/Integration/FloeMergeDSLTest.php | 50 +
.../FloeReaderExtensionParityTest.php | 253 +++
.../FloeSerializerExtensionParityTest.php | 58 +
...FloeValueSerializerExtensionParityTest.php | 111 ++
.../FloeWriterExtensionParityTest.php | 130 ++
.../Tests/Mother/DateTimeDecoderMother.php | 16 +
.../Tests/Mother/DynamicDecoderMother.php | 19 +
.../Flow/Floe/Tests/Mother/FooterMother.php | 42 +
.../Floe/Tests/Mother/FrameReaderMother.php | 23 +
.../Flow/Floe/Tests/Mother/RowsMother.php | 111 ++
.../Floe/Tests/Unit/Codec/NoopCodecTest.php | 26 +
.../Unit/Decoding/BooleanDecoderTest.php | 21 +
.../Unit/Decoding/DateTimeDecoderTest.php | 68 +
.../Unit/Decoding/DynamicDecoderTest.php | 71 +
.../Tests/Unit/Decoding/EnumDecoderTest.php | 50 +
.../Unit/Decoding/Float64DecoderTest.php | 21 +
.../Unit/Decoding/HtmlDocumentDecoderTest.php | 32 +
.../Unit/Decoding/HtmlElementDecoderTest.php | 30 +
.../Tests/Unit/Decoding/Int64DecoderTest.php | 28 +
.../Unit/Decoding/IntervalDecoderTest.php | 34 +
.../Tests/Unit/Decoding/JsonDecoderTest.php | 26 +
.../Tests/Unit/Decoding/ListDecoderTest.php | 25 +
.../Tests/Unit/Decoding/MapDecoderTest.php | 42 +
.../Tests/Unit/Decoding/NullDecoderTest.php | 19 +
.../Unit/Decoding/OptionalDecoderTest.php | 26 +
.../Unit/Decoding/PackedListDecoderTest.php | 43 +
.../Tests/Unit/Decoding/StringDecoderTest.php | 26 +
.../Unit/Decoding/StructureDecoderTest.php | 64 +
.../Unit/Decoding/TimeZoneDecoderTest.php | 26 +
.../Tests/Unit/Decoding/TimeZonesTest.php | 19 +
.../Tests/Unit/Decoding/UuidDecoderTest.php | 24 +
.../Unit/Decoding/XmlDocumentDecoderTest.php | 37 +
.../Unit/Decoding/XmlElementDecoderTest.php | 28 +
.../Unit/Encoding/BooleanEncoderTest.php | 19 +
.../Unit/Encoding/DateTimeEncoderTest.php | 49 +
.../Unit/Encoding/DynamicEncoderTest.php | 69 +
.../Tests/Unit/Encoding/EnumEncoderTest.php | 25 +
.../Unit/Encoding/Float64EncoderTest.php | 21 +
.../Unit/Encoding/HtmlDocumentEncoderTest.php | 30 +
.../Unit/Encoding/HtmlElementEncoderTest.php | 29 +
.../Tests/Unit/Encoding/Int64EncoderTest.php | 26 +
.../Unit/Encoding/IntervalEncoderTest.php | 28 +
.../Tests/Unit/Encoding/JsonEncoderTest.php | 22 +
.../Tests/Unit/Encoding/ListEncoderTest.php | 22 +
.../Tests/Unit/Encoding/MapEncoderTest.php | 30 +
.../Tests/Unit/Encoding/NullEncoderTest.php | 16 +
.../Unit/Encoding/OptionalEncoderTest.php | 23 +
.../Unit/Encoding/PackedListEncoderTest.php | 28 +
.../Tests/Unit/Encoding/StringEncoderTest.php | 21 +
.../Unit/Encoding/StringKeyEncoderTest.php | 24 +
.../Unit/Encoding/StructureEncoderTest.php | 43 +
.../Unit/Encoding/TimeZoneEncoderTest.php | 22 +
.../Tests/Unit/Encoding/UuidEncoderTest.php | 20 +
.../Unit/Encoding/XmlDocumentEncoderTest.php | 26 +
.../Unit/Encoding/XmlElementEncoderTest.php | 27 +
.../Flow/Floe/Tests/Unit/EntryFactoryTest.php | 38 +
.../Floe/Tests/Unit/EntryInstantiatorTest.php | 22 +
.../Tests/Unit/ExtRowFrameEncoderTest.php | 66 +
.../Flow/Floe/Tests/Unit/FloeDSLTest.php | 48 +
.../Floe/Tests/Unit/FloeExtractorTest.php | 215 +++
.../Flow/Floe/Tests/Unit/FloeLoaderTest.php | 112 ++
.../Flow/Floe/Tests/Unit/FloeMergerTest.php | 315 ++++
.../Flow/Floe/Tests/Unit/FloeReaderTest.php | 1288 +++++++++++++++
.../Floe/Tests/Unit/FloeSerializerTest.php | 118 ++
.../Tests/Unit/FloeValueRoundTripTest.php | 288 ++++
.../Tests/Unit/FloeValueSerializerTest.php | 163 ++
.../Flow/Floe/Tests/Unit/FloeWriterTest.php | 512 ++++++
.../tests/Flow/Floe/Tests/Unit/FooterTest.php | 157 ++
.../tests/Flow/Floe/Tests/Unit/FormatTest.php | 101 ++
.../Flow/Floe/Tests/Unit/FrameReaderTest.php | 127 ++
.../Flow/Floe/Tests/Unit/FrameSegmentTest.php | 29 +
.../Tests/Unit/PhpRowFrameEncoderTest.php | 131 ++
.../Flow/Floe/Tests/Unit/RowHydratorTest.php | 76 +
.../Flow/Floe/Tests/Unit/RowPaddingTest.php | 79 +
.../Floe/Tests/Unit/RowsValueMapperTest.php | 195 +++
.../Floe/Tests/Unit/SchemaDecoderTest.php | 73 +
.../Floe/Tests/Unit/SchemaEncoderTest.php | 39 +
.../Floe/Tests/Unit/SchemaEvolutionTest.php | 88 +
.../Floe/Tests/Unit/SchemaTrackerTest.php | 129 ++
.../Flow/Floe/Tests/Unit/SectionTest.php | 41 +
.../Flow/Floe/Tests/Unit/ValueDecoderTest.php | 244 +++
.../Flow/Floe/Tests/Unit/ValueEncoderTest.php | 134 ++
src/extension/flow-php-ext/.cargo/config.toml | 11 +
src/extension/flow-php-ext/.gitattributes | 13 +
.../.github/workflows/readonly.yaml | 28 +
.../.github/workflows/release.yml | 250 +++
src/extension/flow-php-ext/.gitignore | 40 +
src/extension/flow-php-ext/Cargo.lock | 1465 +++++++++++++++++
src/extension/flow-php-ext/Cargo.toml | 13 +
src/extension/flow-php-ext/Makefile | 81 +
src/extension/flow-php-ext/README.md | 39 +
src/extension/flow-php-ext/build.rs | 18 +
src/extension/flow-php-ext/composer.json | 36 +
src/extension/flow-php-ext/ext/config.m4 | 43 +
.../Floe/Exception/ExtensionException.php | 15 +
.../php/Flow/Floe/RowsDecoder.php | 54 +
.../php/Flow/Floe/RowsEncoder.php | 57 +
src/extension/flow-php-ext/src/ctx.rs | 809 +++++++++
src/extension/flow-php-ext/src/encode.rs | 768 +++++++++
src/extension/flow-php-ext/src/exception.rs | 27 +
src/extension/flow-php-ext/src/format.rs | 103 ++
src/extension/flow-php-ext/src/hydrate.rs | 109 ++
src/extension/flow-php-ext/src/lib.rs | 276 ++++
src/extension/flow-php-ext/src/plan.rs | 446 +++++
src/extension/flow-php-ext/src/section.rs | 284 ++++
src/extension/flow-php-ext/src/values.rs | 428 +++++
.../tests/phpt/001_extension_loaded.phpt | 16 +
.../tests/phpt/002_scalar_round_trip.phpt | 59 +
.../tests/phpt/003_null_flags.phpt | 46 +
.../flow-php-ext/tests/phpt/004_datetime.phpt | 46 +
.../tests/phpt/005_time_uuid.phpt | 46 +
.../phpt/006_schema_change_mid_stream.phpt | 33 +
.../tests/phpt/007_empty_rows.phpt | 23 +
.../flow-php-ext/tests/phpt/010_no_leaks.phpt | 81 +
.../tests/phpt/011_containers.phpt | 106 ++
.../tests/phpt/012_enum_json_date.phpt | 53 +
.../tests/phpt/013_xml_fallback.phpt | 43 +
.../tests/phpt/015_dynamic_values.phpt | 65 +
.../tests/phpt/016_numeric_string_keys.phpt | 70 +
.../tests/phpt/017_rows_decoder.phpt | 46 +
.../tests/phpt/019_html_fallback.phpt | 37 +
.../tests/phpt/020_row_encoder_parity.phpt | 105 ++
.../tests/phpt/021_batched_decode.phpt | 92 ++
.../phpt/022_row_encoder_round_trip.phpt | 64 +
.../phpt/023_encoder_datetime_subclass.phpt | 27 +
.../tests/phpt/027_rows_encoder_segments.phpt | 84 +
.../flow-php-ext/tests/phpt/bootstrap.php | 132 ++
.../Filesystem/Local/Memory/MemoryStream.php | 2 +
.../Local/MemoryFilesystemTest.php | 17 +
.../src/Flow/Types/Type/Native/EnumType.php | 11 +-
.../Tests/Unit/Type/Native/EnumTypeTest.php | 14 +
.../Tests/Integration/ManifestTest.php | 2 +-
.../assets/codemirror/completions/dsl.js | 76 +-
web/landing/resources/dsl.json | 2 +-
.../Website/Model/Documentation/Module.php | 2 +
.../documentation/navigation_left.html.twig | 1 +
.../documentation/navigation_right.html.twig | 1 +
291 files changed, 21937 insertions(+), 428 deletions(-)
create mode 100644 .github/workflows/job-flow-php-extension.yml
create mode 100644 .nix/pkgs/php-flow-php-ext/package.nix
create mode 100644 documentation/components/core/floe.md
create mode 100644 documentation/components/extensions/flow-php-ext.md
create mode 100644 documentation/installation/packages/flow-php-ext.md
create mode 100644 src/core/etl/src/Flow/Floe/Codec.php
create mode 100644 src/core/etl/src/Flow/Floe/Codec/NoopCodec.php
create mode 100644 src/core/etl/src/Flow/Floe/DSL/functions.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/BooleanDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/DateTimeDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/DynamicDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/EnumDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/Float64Decoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/HtmlDocumentDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/HtmlElementDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/Int64Decoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/IntervalDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/JsonDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/ListDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/MapDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/NullDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/OptionalDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/PackedListDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/StringDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/StructureDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/TimeZoneDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/TimeZones.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/UuidDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/ValueDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/XmlDocumentDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Decoding/XmlElementDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/EncoderColumn.php
create mode 100644 src/core/etl/src/Flow/Floe/EncoderPlan.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/BooleanEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/DateTimeEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/DynamicEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/EnumEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/Float64Encoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/HtmlDocumentEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/HtmlElementEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/Int64Encoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/IntervalEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/JsonEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/ListEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/MapEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/NullEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/OptionalEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/PackedListEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/StringEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/StringKeyEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/StructureEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/TimeZoneEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/UuidEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/ValueEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/XmlDocumentEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/Encoding/XmlElementEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/EntryFactory.php
create mode 100644 src/core/etl/src/Flow/Floe/EntryInstantiator.php
create mode 100644 src/core/etl/src/Flow/Floe/Exception/FloeException.php
create mode 100644 src/core/etl/src/Flow/Floe/Exception/IncompatibleSchemaException.php
create mode 100644 src/core/etl/src/Flow/Floe/ExtRowFrameEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/FloeExtractor.php
create mode 100644 src/core/etl/src/Flow/Floe/FloeFile.php
create mode 100644 src/core/etl/src/Flow/Floe/FloeLoader.php
create mode 100644 src/core/etl/src/Flow/Floe/FloeMerger.php
create mode 100644 src/core/etl/src/Flow/Floe/FloeReader.php
create mode 100644 src/core/etl/src/Flow/Floe/FloeSerializer.php
create mode 100644 src/core/etl/src/Flow/Floe/FloeValueSerializer.php
create mode 100644 src/core/etl/src/Flow/Floe/FloeWriter.php
create mode 100644 src/core/etl/src/Flow/Floe/Footer.php
create mode 100644 src/core/etl/src/Flow/Floe/Format.php
create mode 100644 src/core/etl/src/Flow/Floe/FrameReader.php
create mode 100644 src/core/etl/src/Flow/Floe/FrameSegment.php
create mode 100644 src/core/etl/src/Flow/Floe/HydratorColumn.php
create mode 100644 src/core/etl/src/Flow/Floe/PhpRowFrameEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/RowEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/RowFrameEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/RowHydrator.php
create mode 100644 src/core/etl/src/Flow/Floe/RowPadding.php
create mode 100644 src/core/etl/src/Flow/Floe/RowsValueMapper.php
create mode 100644 src/core/etl/src/Flow/Floe/SchemaDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/SchemaEncoder.php
create mode 100644 src/core/etl/src/Flow/Floe/SchemaEvolution.php
create mode 100644 src/core/etl/src/Flow/Floe/SchemaTracker.php
create mode 100644 src/core/etl/src/Flow/Floe/Section.php
create mode 100644 src/core/etl/src/Flow/Floe/ValueDecoder.php
create mode 100644 src/core/etl/src/Flow/Floe/ValueEncoder.php
create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Fixtures/CustomDateTime.php
delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/CacheBaseTestSuite.php
create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/CacheTestCase.php
create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheStreamingTest.php
create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheTest.php
delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheTestSuite.php
rename src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/{InMemoryCacheTestSuite.php => InMemoryCacheTest.php} (79%)
create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleFilesystemCacheTest.php
delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleFilesystemCacheTestSuite.php
create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleRedisCacheTest.php
delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleRedisCacheTestSuite.php
rename src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/{TraceableCacheTestSuite.php => TraceableCacheTest.php} (96%)
create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/FilesystemBucketsCacheTest.php
create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/CacheIndexTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Context/FloeFileContext.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Double/CodecStub.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Double/PrefixingCodecStub.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Double/UnsizedFilesystem.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Double/UnsizedSourceStream.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Integration/FloeDataFrameTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Integration/FloeFileTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Integration/FloeMergeDSLTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Integration/FloeReaderExtensionParityTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Integration/FloeSerializerExtensionParityTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Integration/FloeValueSerializerExtensionParityTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Integration/FloeWriterExtensionParityTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Mother/DateTimeDecoderMother.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Mother/DynamicDecoderMother.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Mother/FooterMother.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Mother/FrameReaderMother.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Mother/RowsMother.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Codec/NoopCodecTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/BooleanDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/DateTimeDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/DynamicDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/EnumDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/Float64DecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/HtmlDocumentDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/HtmlElementDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/Int64DecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/IntervalDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/JsonDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/ListDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/MapDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/NullDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/OptionalDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/PackedListDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/StringDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/StructureDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/TimeZoneDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/TimeZonesTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/UuidDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/XmlDocumentDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/XmlElementDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/BooleanEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/DateTimeEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/DynamicEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/EnumEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/Float64EncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/HtmlDocumentEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/HtmlElementEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/Int64EncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/IntervalEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/JsonEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/ListEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/MapEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/NullEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/OptionalEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/PackedListEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StringEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StringKeyEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StructureEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/TimeZoneEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/UuidEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/XmlDocumentEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/XmlElementEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/EntryFactoryTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/EntryInstantiatorTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/ExtRowFrameEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeDSLTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeLoaderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeMergerTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeReaderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeSerializerTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeValueRoundTripTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeValueSerializerTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeWriterTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FooterTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FormatTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FrameReaderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FrameSegmentTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/PhpRowFrameEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/RowHydratorTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/RowPaddingTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/RowsValueMapperTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaEncoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaEvolutionTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaTrackerTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/SectionTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/ValueDecoderTest.php
create mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/ValueEncoderTest.php
create mode 100644 src/extension/flow-php-ext/.cargo/config.toml
create mode 100644 src/extension/flow-php-ext/.gitattributes
create mode 100644 src/extension/flow-php-ext/.github/workflows/readonly.yaml
create mode 100644 src/extension/flow-php-ext/.github/workflows/release.yml
create mode 100644 src/extension/flow-php-ext/.gitignore
create mode 100644 src/extension/flow-php-ext/Cargo.lock
create mode 100644 src/extension/flow-php-ext/Cargo.toml
create mode 100644 src/extension/flow-php-ext/Makefile
create mode 100644 src/extension/flow-php-ext/README.md
create mode 100644 src/extension/flow-php-ext/build.rs
create mode 100644 src/extension/flow-php-ext/composer.json
create mode 100644 src/extension/flow-php-ext/ext/config.m4
create mode 100644 src/extension/flow-php-ext/php/Flow/Floe/Exception/ExtensionException.php
create mode 100644 src/extension/flow-php-ext/php/Flow/Floe/RowsDecoder.php
create mode 100644 src/extension/flow-php-ext/php/Flow/Floe/RowsEncoder.php
create mode 100644 src/extension/flow-php-ext/src/ctx.rs
create mode 100644 src/extension/flow-php-ext/src/encode.rs
create mode 100644 src/extension/flow-php-ext/src/exception.rs
create mode 100644 src/extension/flow-php-ext/src/format.rs
create mode 100644 src/extension/flow-php-ext/src/hydrate.rs
create mode 100644 src/extension/flow-php-ext/src/lib.rs
create mode 100644 src/extension/flow-php-ext/src/plan.rs
create mode 100644 src/extension/flow-php-ext/src/section.rs
create mode 100644 src/extension/flow-php-ext/src/values.rs
create mode 100644 src/extension/flow-php-ext/tests/phpt/001_extension_loaded.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/002_scalar_round_trip.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/003_null_flags.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/004_datetime.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/005_time_uuid.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/006_schema_change_mid_stream.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/007_empty_rows.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/010_no_leaks.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/011_containers.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/012_enum_json_date.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/013_xml_fallback.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/015_dynamic_values.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/016_numeric_string_keys.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/017_rows_decoder.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/019_html_fallback.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/020_row_encoder_parity.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/021_batched_decode.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/022_row_encoder_round_trip.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/023_encoder_datetime_subclass.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/027_rows_encoder_segments.phpt
create mode 100644 src/extension/flow-php-ext/tests/phpt/bootstrap.php
diff --git a/.codecov.yaml b/.codecov.yaml
index 41af3be00e..072e2a3d22 100644
--- a/.codecov.yaml
+++ b/.codecov.yaml
@@ -16,6 +16,8 @@ flag_management:
carryforward: true
- name: arrow-extension
carryforward: true
+ - name: flow-php-extension
+ carryforward: true
- name: telemetry-tests
carryforward: true
diff --git a/.github/workflows/job-flow-php-extension.yml b/.github/workflows/job-flow-php-extension.yml
new file mode 100644
index 0000000000..263baec2d8
--- /dev/null
+++ b/.github/workflows/job-flow-php-extension.yml
@@ -0,0 +1,192 @@
+name: Flow PHP Extension
+
+on:
+ workflow_call:
+ secrets:
+ CODECOV_TOKEN:
+ required: false
+
+jobs:
+ build:
+ name: Build and Test Extension
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: ['ubuntu-latest', 'macos-latest']
+ php: ['8.3', '8.4', '8.5']
+
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ persist-credentials: false
+
+ - name: Setup PHP Environment
+ uses: ./.github/actions/setup-php-env
+ with:
+ php-version: ${{ matrix.php }}
+ dependencies: locked
+ extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib, zstd'
+ tools: 'composer:v2, php-config'
+ coverage: 'pcov'
+ install-dependencies: 'false'
+
+ - name: Install Rust toolchain
+ uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+ with:
+ toolchain: stable
+
+ - name: Cache cargo registry and build
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ src/extension/flow-php-ext/target
+ key: ${{ runner.os }}-php${{ matrix.php }}-cargo-${{ hashFiles('src/extension/flow-php-ext/Cargo.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-php${{ matrix.php }}-cargo-
+
+ - name: Install build dependencies (Ubuntu)
+ if: runner.os == 'Linux'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y build-essential clang libclang-dev
+
+ - name: Install build dependencies (macOS)
+ if: runner.os == 'macOS'
+ run: |
+ brew install llvm
+ echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> "$GITHUB_ENV"
+ echo "CLANG_PATH=$(brew --prefix llvm)/bin/clang" >> "$GITHUB_ENV"
+
+ - name: Set LIBCLANG_PATH (Ubuntu)
+ if: runner.os == 'Linux'
+ run: |
+ LIBCLANG_SO=$(find /usr/lib \
+ \( -name 'libclang.so' -o -name 'libclang.so.*' -o -name 'libclang-*.so' -o -name 'libclang-*.so.*' \) \
+ -not -path '*/clang/*/lib/*' \
+ 2>/dev/null | head -1)
+ if [ -z "$LIBCLANG_SO" ]; then
+ echo "Could not locate libclang.so under /usr/lib" >&2
+ exit 1
+ fi
+ echo "LIBCLANG_PATH=$(dirname "$LIBCLANG_SO")" >> "$GITHUB_ENV"
+ echo "Detected libclang at: $LIBCLANG_SO"
+
+ - name: Build extension
+ working-directory: src/extension/flow-php-ext
+ run: make build
+
+ - name: Run PHPT tests
+ working-directory: src/extension/flow-php-ext
+ run: make test
+
+ - name: Install extension and verify
+ working-directory: src/extension/flow-php-ext
+ run: |
+ EXT_DIR=$(php -r 'echo ini_get("extension_dir");')
+ sudo cp ext/modules/flow_php.so "$EXT_DIR/"
+ echo "extension=flow_php.so" | sudo tee -a "$(php -r 'echo php_ini_loaded_file();')"
+ php -m | grep flow_php
+
+ - name: Install Composer Dependencies
+ uses: ./.github/actions/composer-install
+ with:
+ php-version: ${{ matrix.php }}
+ dependencies: locked
+
+ - name: Run ETL Unit Tests with Flow PHP Extension
+ run: just test --testsuite=etl-unit ${{ (matrix.php == '8.3' && matrix.os == 'ubuntu-latest') && '--coverage-clover=./var/phpunit/coverage/clover/coverage.xml' || '' }}
+
+ - name: Run ETL Integration Tests with Flow PHP Extension
+ run: just test --testsuite=etl-integration
+
+ - name: Upload to Codecov
+ if: ${{ !cancelled() && matrix.php == '8.3' && matrix.os == 'ubuntu-latest' }}
+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
+ directory: ./var/phpunit/coverage/clover
+ flags: flow-php-extension
+
+ pie-install-test:
+ name: Test PIE Installation
+ runs-on: ${{ matrix.os }}
+ env:
+ RUSTUP_TOOLCHAIN: stable
+ strategy:
+ fail-fast: false
+ matrix:
+ os: ['ubuntu-latest', 'macos-latest']
+
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ persist-credentials: false
+
+ - name: Setup PHP Environment
+ uses: ./.github/actions/setup-php-env
+ with:
+ php-version: '8.3'
+ dependencies: locked
+ extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib'
+ tools: 'composer:v2, phpize, php-config'
+
+ - name: Install Rust toolchain
+ uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+ with:
+ toolchain: stable
+
+ - name: Install build dependencies (Ubuntu)
+ if: runner.os == 'Linux'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y build-essential autoconf automake libtool clang libclang-dev
+
+ - name: Install build dependencies (macOS)
+ if: runner.os == 'macOS'
+ run: |
+ brew install autoconf automake libtool llvm
+ echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> "$GITHUB_ENV"
+ # See the build job: keep clang and libclang from the same llvm so
+ # bindgen does not mix brew's libclang with Xcode's clang headers.
+ echo "CLANG_PATH=$(brew --prefix llvm)/bin/clang" >> "$GITHUB_ENV"
+
+ - name: Set LIBCLANG_PATH (Ubuntu)
+ if: runner.os == 'Linux'
+ run: |
+ # Find the canonical libclang.so shared library, excluding the
+ # Clang runtime support libraries under llvm-*/lib/clang/*/lib/.
+ LIBCLANG_SO=$(find /usr/lib \
+ \( -name 'libclang.so' -o -name 'libclang.so.*' -o -name 'libclang-*.so' -o -name 'libclang-*.so.*' \) \
+ -not -path '*/clang/*/lib/*' \
+ 2>/dev/null | head -1)
+ if [ -z "$LIBCLANG_SO" ]; then
+ echo "Could not locate libclang.so under /usr/lib" >&2
+ exit 1
+ fi
+ echo "LIBCLANG_PATH=$(dirname "$LIBCLANG_SO")" >> "$GITHUB_ENV"
+ echo "Detected libclang at: $LIBCLANG_SO"
+
+ - name: Install PIE
+ run: |
+ sudo curl -L -o /usr/local/bin/pie https://github.com/php/pie/releases/latest/download/pie.phar
+ sudo chmod +x /usr/local/bin/pie
+ pie --version
+
+ - name: Prepare local package for PIE installation
+ working-directory: src/extension/flow-php-ext
+ run: |
+ jq '. + {"version": "0.0.9999"}' composer.json > composer.tmp.json
+ mv composer.tmp.json composer.json
+
+ - name: Install extension via PIE (from local)
+ run: |
+ sudo pie repository:remove packagist.org
+ sudo pie repository:add path ${{ github.workspace }}/src/extension/flow-php-ext
+ sudo env "PATH=$PATH" "LIBCLANG_PATH=$LIBCLANG_PATH" ${CLANG_PATH:+"CLANG_PATH=$CLANG_PATH"} "RUSTUP_TOOLCHAIN=$RUSTUP_TOOLCHAIN" "CARGO_HOME=$CARGO_HOME" "RUSTUP_HOME=$RUSTUP_HOME" pie install flow-php/flow-php-ext:0.0.9999@dev
+
+ - name: Verify extension is loaded
+ run: |
+ php -m | grep flow_php
diff --git a/.github/workflows/monorepo-split.yml b/.github/workflows/monorepo-split.yml
index 4de53cb4e3..dec932787d 100644
--- a/.github/workflows/monorepo-split.yml
+++ b/.github/workflows/monorepo-split.yml
@@ -128,6 +128,8 @@ jobs:
split_repository: 'pg-query-ext'
- local_path: 'src/extension/arrow-ext'
split_repository: 'arrow-ext'
+ - local_path: 'src/extension/flow-php-ext'
+ split_repository: 'flow-php-ext'
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
index acba5dfdcc..4770ddb2a1 100644
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -60,5 +60,10 @@ jobs:
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+ flow-php-extension:
+ uses: ./.github/workflows/job-flow-php-extension.yml
+ secrets:
+ CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
mutation-tests:
uses: ./.github/workflows/job-mutation-tests.yml
diff --git a/.nix/pkgs/flow-php/package.nix b/.nix/pkgs/flow-php/package.nix
index 1d6bbd7406..6d0af49a4d 100644
--- a/.nix/pkgs/flow-php/package.nix
+++ b/.nix/pkgs/flow-php/package.nix
@@ -7,11 +7,13 @@
php-zstd,
php-pg-query-ext,
php-arrow-ext,
+ php-flow-php-ext,
with-pcov ? true,
with-xdebug ? false,
with-blackfire ? false,
with-pg-query-ext ? false,
with-arrow-ext ? false,
+ with-flow-php-ext ? false,
with-grpc ? false,
with-protobuf ? true
}:
@@ -38,6 +40,7 @@ let
++ (if with-blackfire then [blackfire] else [])
++ (if with-pg-query-ext then [(php-pg-query-ext.override { inherit php; })] else [])
++ (if with-arrow-ext then [(php-arrow-ext.override { inherit php; })] else [])
+ ++ (if with-flow-php-ext then [(php-flow-php-ext.override { inherit php; })] else [])
++ (if with-grpc then [grpc] else [])
++ (if with-protobuf then [
(protobuf.overrideAttrs (old: {
diff --git a/.nix/pkgs/php-flow-php-ext/package.nix b/.nix/pkgs/php-flow-php-ext/package.nix
new file mode 100644
index 0000000000..50e21c59d3
--- /dev/null
+++ b/.nix/pkgs/php-flow-php-ext/package.nix
@@ -0,0 +1,66 @@
+{
+ php,
+ lib,
+ stdenv,
+ rustPlatform,
+ clang,
+ llvmPackages,
+ flow-php-ext-version ? "dev",
+}:
+
+let
+ extSrc = builtins.path {
+ path = ../../../src/extension/flow-php-ext;
+ name = "flow-php-ext-src";
+ filter = path: type:
+ let baseName = baseNameOf path;
+ in !(
+ baseName == "target" ||
+ baseName == "vendor" ||
+ baseName == "ext" ||
+ baseName == ".gitignore" ||
+ baseName == ".gitattributes" ||
+ baseName == "composer.json" ||
+ baseName == "composer.lock"
+ );
+ };
+ pkg = rustPlatform.buildRustPackage {
+ pname = "php-flow-php-ext";
+ version = flow-php-ext-version;
+
+ src = extSrc;
+
+ cargoLock.lockFile = ../../../src/extension/flow-php-ext/Cargo.lock;
+
+ nativeBuildInputs = [
+ clang
+ llvmPackages.libclang
+ php.unwrapped
+ php.unwrapped.dev
+ ];
+
+ env = {
+ LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
+ PHP_CONFIG = "${php.unwrapped.dev}/bin/php-config";
+ PHP = "${php.unwrapped}/bin/php";
+ FLOW_PHP_EXT_VERSION = flow-php-ext-version;
+ };
+
+ installPhase = let
+ targetDir = "target/${stdenv.hostPlatform.rust.rustcTargetSpec}/release";
+ in ''
+ runHook preInstall
+ mkdir -p $out/lib/php/extensions
+ cp ${targetDir}/libflow_php${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/php/extensions/flow_php.so
+ runHook postInstall
+ '';
+
+ doCheck = false;
+
+ meta = with lib; {
+ description = "Flow PHP native extension (Rust) - Floe frame-body encoder/decoder for DataFrame Rows";
+ license = licenses.mit;
+ };
+ };
+in
+pkg // { extensionName = "flow_php"; }
diff --git a/Justfile b/Justfile
index a7b62a811c..206b055744 100644
--- a/Justfile
+++ b/Justfile
@@ -48,7 +48,7 @@ lint-links:
# Audit GitHub Actions workflows (actionlint static checks + zizmor security audit).
# Workflows that live under src/ are split out to their own repos, so they are not
# auto-discovered with the root .github/workflows and must be listed explicitly:
-# - the arrow-ext release workflow (custom, full audit);
+# - the extension release workflows (custom, full audit);
# - every per-package subtree-split readonly.yaml (added by hand per package, so all
# are audited to catch drift; dangerous-triggers is exempted for them in
# .github/zizmor.yml — they only run `gh pr close`, never check out PR code).
@@ -56,7 +56,7 @@ lint-actions:
#!/usr/bin/env bash
set -uo pipefail
rc=0
- extra_workflows=(src/extension/arrow-ext/.github/workflows/release.yml)
+ extra_workflows=(src/extension/arrow-ext/.github/workflows/release.yml src/extension/flow-php-ext/.github/workflows/release.yml)
while IFS= read -r workflow; do
extra_workflows+=("$workflow")
done < <(find src -path '*/.github/workflows/readonly.yaml' | sort)
diff --git a/bin/docs.php b/bin/docs.php
index cc1b791c11..790549d99b 100755
--- a/bin/docs.php
+++ b/bin/docs.php
@@ -56,6 +56,7 @@ public function execute(InputInterface $input, OutputInterface $output): int
{
$paths = [
__DIR__ . '/../src/core/etl/src/Flow/ETL/DSL/functions.php',
+ __DIR__ . '/../src/core/etl/src/Flow/Floe/DSL/functions.php',
__DIR__ . '/../src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/functions.php',
__DIR__ . '/../src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/functions.php',
__DIR__ . '/../src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/functions.php',
diff --git a/composer.json b/composer.json
index 1c11a652ad..be7b19fa08 100644
--- a/composer.json
+++ b/composer.json
@@ -200,7 +200,8 @@
"src/lib/telemetry/src/Flow",
"src/lib/types/src/Flow",
"src/tools/documentation/src/Flow",
- "src/extension/arrow-ext/php/Flow"
+ "src/extension/arrow-ext/php/Flow",
+ "src/extension/flow-php-ext/php/Flow"
],
"Flow\\Doctrine\\Bulk\\": [
"src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk"
@@ -245,6 +246,7 @@
"src/bridge/telemetry/otlp/src/Flow/Bridge/Telemetry/OTLP/DSL/functions.php",
"src/cli/src/Flow/CLI/DSL/functions.php",
"src/core/etl/src/Flow/ETL/DSL/functions.php",
+ "src/core/etl/src/Flow/Floe/DSL/functions.php",
"src/functions.php",
"src/lib/array-dot/src/Flow/ArrayDot/array_dot.php",
"src/lib/azure-sdk/src/Flow/Azure/SDK/DSL/functions.php",
diff --git a/documentation/components/core/caching.md b/documentation/components/core/caching.md
index 3f61c73f7d..37aad8fdec 100644
--- a/documentation/components/core/caching.md
+++ b/documentation/components/core/caching.md
@@ -10,6 +10,9 @@ already transformed dataset.
Cache will run a pipeline, catching each Rows and saving them into cache
from where those rows can be later extracted.
+Internally, each cached batch of Rows is stored under its own cache key, and the pipeline's cache id
+points at an index - a single-column (`key`) Rows value listing those chunk keys in insertion order.
+
This is useful for operations that require full transformation of dataset before
moving forward, like, for example, sorting.
@@ -29,19 +32,21 @@ data_frame
```
By default, Flow is using Filesystem Cache, location of the cache storage can be adjusted through
-environment variable `CACHE_DIR_ENV`.
+the `FLOW_LOCAL_FILESYSTEM_CACHE_DIR` environment variable.
To use different cache implementation please use `ConfigBuilder`
```php
+cache(
new PSRSimpleCache(
new Psr16Cache(
new ArrayAdapter()
- ),
- new NativePHPSerializer()
+ )
)
);
```
@@ -52,5 +57,39 @@ The following implementations are available out of the box:
* [LocalFilesystem](/src/core/etl/src/Flow/ETL/Cache/Implementation/FilesystemCache.php)
* [PSRSimpleCache](/src/core/etl/src/Flow/ETL/Cache/Implementation/PSRSimpleCache.php)
-PSRSimpleCache makes possible to use any of the [psr/simple-cache-implementation](https://packagist.org/providers/psr/simple-cache-implementation)
-but it does not come with any out of the box.
\ No newline at end of file
+PSRSimpleCache makes possible to use any of
+the [psr/simple-cache-implementation](https://packagist.org/providers/psr/simple-cache-implementation)
+but it does not come with any out of the box.
+
+## Serialization
+
+Cache entries are stored as [Floe](/documentation/components/core/floe.md) files, streamed in both
+directions: writes go straight to the cache file batch by batch (the payload string is never
+materialized), reads decode one batch at a time. The batch size (default 1000 rows) bounds how many
+rows cross the engine at once:
+
+```php
+cacheSerializerBatchSize(1000);
+
+// or constructing the cache directly
+filesystem_cache(serializer_batch_size: 1000);
+```
+
+`Cache::get()` always returns a fully materialized `Rows` — the raw payload bytes are never held
+next to the decoded rows, but the decoded rows themselves are unbounded. To keep the decoded side
+bounded too, use `Cache::read()`:
+
+```php
+read('my-dataset') as $rows) {
+ // FilesystemCache: one batch of Rows at a time
+ // InMemory/PSRSimpleCache: the whole value, yielded once
+}
+```
\ No newline at end of file
diff --git a/documentation/components/core/core.md b/documentation/components/core/core.md
index 5e58907288..04975625c5 100644
--- a/documentation/components/core/core.md
+++ b/documentation/components/core/core.md
@@ -109,6 +109,7 @@ For detailed information about specific DataFrame operations, see the following
- **[Batch Processing](/documentation/components/core/batch-processing.md)** - Controlling batch sizes and memory collection
- **[Partitioning](/documentation/components/core/partitioning.md)** - Data partitioning for efficient processing
- **[Caching](/documentation/components/core/caching.md)** - Performance optimization through caching
+- **[Floe File Format](/documentation/components/core/floe.md)** - Flow's native self-describing binary row format
- **[Data Retrieval](/documentation/components/core/data-retrieval.md)** - Methods for getting processed data
### Data Quality & Validation
diff --git a/documentation/components/core/floe.md b/documentation/components/core/floe.md
new file mode 100644
index 0000000000..72ee2cbc8a
--- /dev/null
+++ b/documentation/components/core/floe.md
@@ -0,0 +1,290 @@
+# Floe File Format
+
+[DOC_LINK:/documentation/components/core/core.md]
+
+[TOC]
+
+Floe is Flow's native, self-describing binary file format for `Rows`. It stores the schema inside
+the file, evolves seamlessly across appended sections, and reads back through the DataFrame API with
+`from_floe()` / `to_floe()`. Files use the `.floe` extension.
+
+## Writing
+
+```php
+read(from_array([
+ ['id' => 1, 'name' => 'John'],
+ ['id' => 2, 'name' => 'Jane'],
+ ]))
+ ->write(to_floe(__DIR__ . '/output.floe'))
+ ->run();
+```
+
+## Reading
+
+```php
+read(from_floe(__DIR__ . '/output.floe'))
+ ->run();
+```
+
+## Save Modes
+
+`to_floe()` honors every [Save Mode](/documentation/components/core/save-mode.md) through the same
+machinery as other file loaders:
+
+- **ExceptionIfExists** (default) - throws if the destination exists.
+- **Overwrite** - replaces the destination.
+- **Ignore** - skips writing when the destination exists.
+- **Append** - writes a **new sibling `.floe` file** per run (like every other file format).
+ Reading a directory of `.floe` files returns their union.
+
+```php
+read(from_array([['id' => 3]]))
+ ->mode(SaveMode::Append)
+ ->write(to_floe(__DIR__ . '/data/dataset.floe'))
+ ->run();
+
+// reads dataset.floe plus every appended sibling
+data_frame()
+ ->read(from_floe(__DIR__ . '/data/*.floe'))
+ ->run();
+```
+
+> Floe additionally supports appending seamlessly into a **single** file with schema evolution
+> through the low-level `Flow\Floe\FloeWriter::append()` API; the DataFrame `Append` save mode uses
+> the sibling-file behavior for consistency with the rest of Flow.
+
+## Partitioning
+
+Partitioned datasets write one `.floe` file per partition directory. Reading prunes by path like
+other file-based sources:
+
+```php
+read(from_array([
+ ['id' => 1, 'country' => 'PL'],
+ ['id' => 2, 'country' => 'US'],
+ ]))
+ ->partitionBy('country')
+ ->write(to_floe(__DIR__ . '/data/dataset.floe'))
+ ->run();
+
+// prune to a single partition
+data_frame()
+ ->read(from_floe(__DIR__ . '/data/country=PL/dataset.floe'))
+ ->run();
+```
+
+## Limit & Offset Pushdown
+
+`limit()` stops reading early, and `withOffset()` skips whole file sections using the footer before
+reading, so neither scans the full file:
+
+```php
+read(from_floe(__DIR__ . '/output.floe')->withOffset(1_000))
+ ->limit(100)
+ ->run();
+```
+
+## Source Schema Without a Scan
+
+`FloeExtractor::schema()` reads only the file footers (two ranged reads per file), so the source
+schema is available without scanning any rows:
+
+```php
+schema(flow_context(config()));
+```
+
+## Reading the First or Last Rows
+
+The low-level reader pulls just the head or tail of a single `.floe` file without a DataFrame. `tail()`
+reads the row count from the footer and seeks past the leading sections, so it decodes only from the
+boundary section onward — the leading rows are never read:
+
+```php
+read(path(__DIR__ . '/output.floe'));
+
+foreach ($file->head(300) as $rows) {
+ // first 300 rows; stops reading once 300 are yielded
+}
+
+foreach ($file->tail(300) as $rows) {
+ // last 300 rows; decoded from the boundary section onward
+}
+```
+
+Both yield `Rows` in batches (default 1000, override with the second argument) and return every row when
+the file holds fewer than the requested count. For the first N through the DataFrame API use `->limit(N)`
+(pushed into the extractor); `tail()` is a reader-level convenience because it needs the file's total
+from the footer.
+
+## Merging Files
+
+`merge_floe()` combines several `.floe` files (same or append-compatible evolving schema) into one.
+The default byte-splices frame regions without re-encoding a single row — O(bytes); `compact: true`
+re-encodes every row, coalescing same-schema runs into fewer sections:
+
+```php
+ bytes │
+ └────────┴───────────────┴───────────────────────┘
+ 0x01 SCHEMA 0x02 ROW 0x03 PARTITIONS 0x06 FOOTER
+```
+
+- **SCHEMA (`0x01`)** — body is the JSON schema for the section that follows. A section's schema is
+ **grow-only**: it starts as the first row's schema and a new `SCHEMA` frame is written only when a row
+ introduces a new column or an incompatible type. Rows narrower than the section ride it (see the ROW
+ absent flag), so a heterogeneous stream needs far fewer sections than one frame per distinct shape.
+- **ROW (`0x02`)** — one row encoded **by column** against the current section schema: for each section
+ column, in order, a one-byte presence flag — `0x01` present (followed by the value encoded per the
+ column's schema type, no per-value tag), `0x00` null, `0x02` null-from-null, or `0x03` **absent** (the
+ row has no such column). Only dynamically typed values (mixed/union columns, dynamic map keys) carry a
+ one-byte type tag (`NULL`, `INTEGER`, `FLOAT`, `BOOLEAN`, `STRING`, `ARRAY`, `DATETIME`, `UUID`,
+ `JSON`). A row whose columns exactly match the section produces the same bytes as a plain positional
+ encode. One frame per row.
+- **PARTITIONS (`0x03`)** — for a partitioned file, the partition key/value pairs, written once before
+ the first section: a 4-byte count followed by repeated `[nameLen(4), name, valueLen(4), value]`.
+- **FOOTER (`0x06`)** — the footer JSON followed by the trailer (below).
+
+### Footer
+
+The `FOOTER` frame body is a JSON object carrying everything needed to read the file **without
+scanning rows**:
+
+```json
+{
+ "version": 1,
+ "writer": "1.x-dev",
+ "schemas": [ /* schema body per schemaId */ ],
+ "fileSchema": { /* merged schema of every section */ },
+ "sections": [ { "offset": 6, "schemaId": 0, "rowCount": 2 } ],
+ "partitions": { "country": "PL" },
+ "totalRows": 2,
+ "metadata": { /* typed key/value, Schema\Metadata */ }
+}
+```
+
+- **`sections`** map a byte `offset` → `schemaId` + `rowCount`, so a reader can skip whole sections
+ (offset/limit pushdown) and know each section's schema up front.
+- **`schemas`** is the deduplicated list of every schema written; **`fileSchema`** is their merge —
+ the source schema, available from the footer alone. On a seamless read, a row narrower than
+ `fileSchema` (an absent column, or a column added by a later section) is padded with null entries so
+ every yielded row conforms; `recover()` reads rows exactly as written, absent columns omitted.
+- A new column or incompatible type grows the section (new `SCHEMA` frame); this is also how a single
+ file evolves its schema across appended sections.
+
+### Trailer (last 8 bytes)
+
+```
+ ┌───────────────┬───────────────┐
+ │ footer length │ magic "FLOE" │
+ │ 4 bytes (LE) │ 4 bytes │
+ └───────────────┴───────────────┘
+```
+
+A reader seeks to `EOF − 8`, reads the trailer, verifies the trailing `FLOE` magic, then seeks back
+`footer length` bytes to parse the footer JSON — two ranged reads, no body scan. This is what powers
+`FloeExtractor::schema()` and offset/limit pushdown.
diff --git a/documentation/components/extensions/flow-php-ext.md b/documentation/components/extensions/flow-php-ext.md
new file mode 100644
index 0000000000..7b02e0f9d0
--- /dev/null
+++ b/documentation/components/extensions/flow-php-ext.md
@@ -0,0 +1,77 @@
+---
+package: flow-php/flow-php-ext
+---
+
+# Flow PHP Extension
+
+[PACKAGE_NAV]
+
+[TOC]
+
+Flow stores durable datasets (`to_floe()`) and caches intermediate ones (`DataFrame::cache()`,
+external-sort spill buckets) using the native **Floe** binary format (`.floe`) — the schema is written
+once per section and rows carry raw values only, which makes both the payload and the hydration
+dramatically cheaper than native PHP `serialize()`/`unserialize()`.
+
+This extension encodes and decodes Floe **frames** natively in Rust via
+[ext-php-rs](https://github.com/extphprs/ext-php-rs). The pure-PHP implementation in `Flow\Floe`
+(flow-php/etl) is the canonical behavior reference and works without the extension — loading it is
+purely an optimization. File header and footer assembly always stay in PHP.
+
+> You never need to call this extension directly. `Flow\Floe\FloeReader`/`FloeWriter` — used by
+> `from_floe()`/`to_floe()`, the cache and the external-sort buckets cache — route ROW/SCHEMA frame
+> bodies to it automatically when `extension_loaded('flow_php')` is true.
+
+## Loading the Extension
+
+### In php.ini
+
+```ini
+extension = flow_php
+```
+
+### During Development
+
+```bash
+php -d extension=./ext/modules/flow_php.so your_script.php
+```
+
+## Usage
+
+The extension is used implicitly through the ETL cache:
+
+```php
+read(from_array($bigDataset))
+ ->cache('my-dataset') // serialized with the extension when loaded
+ ->write(to_stream(__DIR__ . '/output.csv'))
+ ->run();
+```
+
+The low-level API works on bare frame bodies (`FloeReader`/`FloeWriter` add the framing):
+
+```php
+schema($schemaFrameBody);
+$row = $decoder->row($rowFrameBody); // Flow\ETL\Row
+$rows = $decoder->rows([$rowFrameBody, ...]); // batched: Flow\ETL\Rows
+
+$encoder = new RowsEncoder();
+$encoder->schema($schemaFrameBody);
+$rowFrameBody = $encoder->row($row); // byte-identical to Flow\Floe\RowEncoder
+
+$segments = (new RowsEncoder())->rows($rows); // batched: framed ROW frames per section segment,
+ // list of Flow\Floe\FrameSegment
+```
+
+All extension failures throw `Flow\Floe\Exception\ExtensionException`; `FloeReader`/`FloeWriter` wrap
+it as `Flow\Floe\Exception\FloeException`. There is no silent fallback to the PHP engine.
diff --git a/documentation/installation/packages/flow-php-ext.md b/documentation/installation/packages/flow-php-ext.md
new file mode 100644
index 0000000000..c5905dcbac
--- /dev/null
+++ b/documentation/installation/packages/flow-php-ext.md
@@ -0,0 +1,60 @@
+---
+package: flow-php/flow-php-ext
+seo_title: "Installing Flow PHP Extension"
+seo_description: >
+ How to install flow-php/flow-php-ext PHP extension using precompiled binaries or PIE.
+---
+
+# Flow PHP Extension
+
+[PACKAGE_NAV:install]
+
+[TOC]
+
+## Precompiled Binaries
+
+Precompiled binaries are available for download from the [GitHub Releases](https://github.com/flow-php/flow-php-ext/releases/latest) page.
+
+Binary names follow the [PIE naming convention](https://github.com/php/pie/blob/1.4.x/docs/extension-maintainers.md):
+
+`php_flow_php-{Version}_php{PhpVersion}-{Arch}-{OS}-{Libc}[-{TSMode}].zip`
+
+Available platforms:
+- macOS ARM64 (Apple Silicon)
+- Linux ARM64
+- Linux x86_64
+
+PHP versions: 8.3, 8.4, 8.5 (each with NTS and ZTS variants).
+
+### Installation
+
+Download the matching zip for your platform, then:
+
+```bash
+# Unzip the archive
+unzip php_flow_php-*.zip
+
+# Copy to your PHP extensions directory
+cp flow_php.so $(php -r "echo ini_get('extension_dir');")
+
+# Enable it
+echo "extension=flow_php" > $(php -r "echo PHP_CONFIG_FILE_SCAN_DIR;")/flow_php.ini
+
+# Verify
+php -m | grep flow_php
+```
+
+## PIE
+
+[PIE](https://github.com/php/pie) is the modern PHP extension installer.
+
+```bash
+pie install flow-php/flow-php-ext
+```
+
+## Build Prerequisites
+
+The Flow PHP extension is written in Rust and requires the following tools to compile:
+
+- **Rust toolchain** (rustc, cargo) - install via [rustup](https://rustup.rs/)
+- **clang** - required by the Rust build process
diff --git a/documentation/upgrading.md b/documentation/upgrading.md
index 25d0db31f2..d53f999b3e 100644
--- a/documentation/upgrading.md
+++ b/documentation/upgrading.md
@@ -268,6 +268,32 @@ built from the missing, mismatched (`MismatchedDefinition`), and unexpected defi
`{name}` in a path is now a partition placeholder resolved from `partitionBy()` columns; strip braces from partition
values before partitioning.
+### 19) `flow-php/etl` - `Cache` stores only `Rows` and gained `read()`; cache indexes are stored as `Rows`
+
+| Before | After |
+|---------------------------------------------------|---------------------------------------------------------------|
+| `Cache::get(string): Row\|Rows\|CacheIndex` | `Cache::get(string): Rows` |
+| `Cache::set(string, Row\|Rows\|CacheIndex): void` | `Cache::set(string, Rows): void` |
+| — | `Cache::read(string $key): Generator` (yields `Rows` batches) |
+| `$cache->set($id, $row)` | `$cache->set($id, rows($row))` |
+| `$cache->set($id, $cacheIndex)` | `$cache->set($id, $cacheIndex->toRows())` |
+| `$cache->get($id)` returning `CacheIndex` | `CacheIndex::fromRows($id, $cache->get($id))` |
+
+Custom `Cache` implementations must adopt the `Rows`-only signatures and add `read()`; the minimal
+implementation is `yield $this->get($key);`.
+
+### 20) `flow-php/etl` - cache and serialization use the Floe (`.floe`) binary format; datetime subclasses rejected
+
+| Before | After |
+|----------------------------------------------------------------------------------------|-----------------------------------------------|
+| `config_builder()->serializer()` default `Base64Serializer(new NativePHPSerializer())` | `Flow\Floe\FloeSerializer` |
+| `filesystem_cache($cache_dir, $filesystem, $serializer)` | `filesystem_cache($cache_dir, $filesystem)` |
+| `new FilesystemCache($filesystem, $serializer, $cacheDir)` | `new FilesystemCache($filesystem, $cacheDir)` |
+| `CacheConfigBuilder::build($fstab, $serializer, …)` | `CacheConfigBuilder::build($fstab, …)` |
+| caching or serializing a `DateTime`/`DateTimeImmutable` subclass (e.g. Carbon) | throws `Flow\Floe\Exception\FloeException` |
+
+Convert `DateTime`/`DateTimeImmutable` subclasses to `DateTime`/`DateTimeImmutable` before caching or serializing.
+
---
## Upgrading from 0.40.x to 0.41.x
diff --git a/manifest.json b/manifest.json
index f07dc0d8c4..8f83a5d59b 100644
--- a/manifest.json
+++ b/manifest.json
@@ -469,6 +469,14 @@
"documentation": "/documentation/components/extensions/pg-query-ext"
}
},
+ {
+ "name": "flow-php/flow-php-ext",
+ "path": "src/extension/flow-php-ext",
+ "type": "extension",
+ "links": {
+ "documentation": "/documentation/components/extensions/flow-php-ext"
+ }
+ },
{
"name": "flow-php/postgresql-valinor-bridge",
"path": "src/bridge/postgresql/valinor",
diff --git a/phpunit.xml.dist b/phpunit.xml.dist
index 81525a46ca..e647e40c4c 100644
--- a/phpunit.xml.dist
+++ b/phpunit.xml.dist
@@ -37,10 +37,13 @@
src/core/etl/tests/Flow/ETL/Tests/Unit
src/core/etl/tests/Flow/ArrayComparison/Tests/Unit
src/core/etl/tests/Flow/Serializer/Tests/Unit
+ src/core/etl/tests/Flow/Floe/Tests/Unit
src/core/etl/tests/Flow/Calculator/Tests/Unit
src/core/etl/tests/Flow/ETL/Tests/Integration
+ src/core/etl/tests/Flow/Serializer/Tests/Integration
+ src/core/etl/tests/Flow/Floe/Tests/Integration
src/cli/tests/Flow/CLI/Tests/Unit
diff --git a/shell.nix b/shell.nix
index 5d865d4f3b..13a773a29e 100644
--- a/shell.nix
+++ b/shell.nix
@@ -5,6 +5,7 @@
with-pcov ? !with-blackfire,
with-pg-query-ext ? !with-c,
with-arrow-ext ? !with-rust,
+ with-flow-php-ext ? !with-rust,
with-c ? false,
with-rust ? false,
with-terraform ? false,
@@ -15,6 +16,7 @@
}:
assert (!(with-rust && with-arrow-ext)) || builtins.throw "Cannot use --arg with-rust true and --arg with-arrow-ext true together. Use: --arg with-arrow-ext false --arg with-rust true";
+assert (!(with-rust && with-flow-php-ext)) || builtins.throw "Cannot use --arg with-rust true and --arg with-flow-php-ext true together. Use: --arg with-flow-php-ext false --arg with-rust true";
assert (!(with-c && with-pg-query-ext)) || builtins.throw "Cannot use --arg with-c true and --arg with-pg-query-ext true together. Use: --arg with-pg-query-ext false --arg with-c true";
let
@@ -62,11 +64,12 @@ let
php-zstd = pkgs.callPackage ./.nix/pkgs/php-zstd/package.nix { php = base-php; };
php-pg-query-ext = pkgs.callPackage ./.nix/pkgs/php-pg-query-ext/package.nix { php = base-php; };
php-arrow-ext = pkgs.callPackage ./.nix/pkgs/php-arrow-ext/package.nix { php = base-php; };
+ php-flow-php-ext = pkgs.callPackage ./.nix/pkgs/php-flow-php-ext/package.nix { php = base-php; };
php = pkgs.callPackage ./.nix/pkgs/flow-php/package.nix {
php = base-php;
- inherit php-snappy php-lz4 php-brotli php-zstd php-pg-query-ext php-arrow-ext
- with-pcov with-xdebug with-blackfire with-pg-query-ext with-arrow-ext with-grpc with-protobuf;
+ inherit php-snappy php-lz4 php-brotli php-zstd php-pg-query-ext php-arrow-ext php-flow-php-ext
+ with-pcov with-xdebug with-blackfire with-pg-query-ext with-arrow-ext with-flow-php-ext with-grpc with-protobuf;
};
in
pkgs.mkShell {
diff --git a/src/core/etl/composer.json b/src/core/etl/composer.json
index 8a4bd5bf6f..55f4df2e71 100644
--- a/src/core/etl/composer.json
+++ b/src/core/etl/composer.json
@@ -37,7 +37,8 @@
"license": "MIT",
"autoload": {
"files": [
- "src/Flow/ETL/DSL/functions.php"
+ "src/Flow/ETL/DSL/functions.php",
+ "src/Flow/Floe/DSL/functions.php"
],
"psr-4": {
"Flow\\": [
diff --git a/src/core/etl/src/Flow/ETL/Attribute/Module.php b/src/core/etl/src/Flow/ETL/Attribute/Module.php
index 4e50e6fd53..81a7ceb0bf 100644
--- a/src/core/etl/src/Flow/ETL/Attribute/Module.php
+++ b/src/core/etl/src/Flow/ETL/Attribute/Module.php
@@ -17,6 +17,7 @@ enum Module: string
case ELASTIC_SEARCH = 'ELASTIC_SEARCH';
case EXCEL = 'EXCEL';
case FILESYSTEM = 'FILESYSTEM';
+ case FLOE = 'FLOE';
case GOOGLE_SHEET = 'GOOGLE_SHEET';
case HTTP = 'HTTP';
case JSON = 'JSON';
diff --git a/src/core/etl/src/Flow/ETL/Cache.php b/src/core/etl/src/Flow/ETL/Cache.php
index fb29b3b965..acdc8c8ee1 100644
--- a/src/core/etl/src/Flow/ETL/Cache.php
+++ b/src/core/etl/src/Flow/ETL/Cache.php
@@ -4,8 +4,8 @@
namespace Flow\ETL;
-use Flow\ETL\Cache\CacheIndex;
use Flow\ETL\Exception\KeyNotInCacheException;
+use Generator;
interface Cache
{
@@ -16,9 +16,16 @@ public function delete(string $key): void;
/**
* @throws KeyNotInCacheException
*/
- public function get(string $key): Row|Rows|CacheIndex;
+ public function get(string $key): Rows;
+
+ /**
+ * @throws KeyNotInCacheException during iteration when the key is absent
+ *
+ * @return Generator
+ */
+ public function read(string $key): Generator;
public function has(string $key): bool;
- public function set(string $key, Row|Rows|CacheIndex $value): void;
+ public function set(string $key, Rows $value): void;
}
diff --git a/src/core/etl/src/Flow/ETL/Cache/CacheIndex.php b/src/core/etl/src/Flow/ETL/Cache/CacheIndex.php
index 64091a9c6b..d58d5ec385 100644
--- a/src/core/etl/src/Flow/ETL/Cache/CacheIndex.php
+++ b/src/core/etl/src/Flow/ETL/Cache/CacheIndex.php
@@ -4,6 +4,18 @@
namespace Flow\ETL\Cache;
+use Flow\ETL\Exception\InvalidArgumentException;
+use Flow\ETL\Row;
+use Flow\ETL\Rows;
+
+use function array_map;
+use function Flow\ETL\DSL\row;
+use function Flow\ETL\DSL\rows;
+use function Flow\ETL\DSL\str_entry;
+use function get_debug_type;
+use function is_string;
+use function sprintf;
+
final class CacheIndex
{
/**
@@ -15,11 +27,39 @@ public function __construct(
public readonly string $key,
) {}
+ /**
+ * @throws InvalidArgumentException
+ */
+ public static function fromRows(string $key, Rows $rows): self
+ {
+ $index = new self($key);
+
+ foreach ($rows->all() as $row) {
+ $value = $row->valueOf('key');
+
+ if (!is_string($value)) {
+ throw new InvalidArgumentException(sprintf(
+ 'CacheIndex expects rows with a string "key" entry, got: %s',
+ get_debug_type($value),
+ ));
+ }
+
+ $index->add($value);
+ }
+
+ return $index;
+ }
+
public function add(string $value): void
{
$this->index[] = $value;
}
+ public function toRows(): Rows
+ {
+ return rows(...array_map(static fn(string $value): Row => row(str_entry('key', $value)), $this->index));
+ }
+
/**
* @return array
*/
diff --git a/src/core/etl/src/Flow/ETL/Cache/Implementation/FilesystemCache.php b/src/core/etl/src/Flow/ETL/Cache/Implementation/FilesystemCache.php
index 69edf24fb7..947c48658e 100644
--- a/src/core/etl/src/Flow/ETL/Cache/Implementation/FilesystemCache.php
+++ b/src/core/etl/src/Flow/ETL/Cache/Implementation/FilesystemCache.php
@@ -5,15 +5,19 @@
namespace Flow\ETL\Cache\Implementation;
use Flow\ETL\Cache;
-use Flow\ETL\Cache\CacheIndex;
+use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Exception\KeyNotInCacheException;
use Flow\ETL\Hash\NativePHPHash;
-use Flow\ETL\Row;
use Flow\ETL\Rows;
use Flow\Filesystem\Filesystem;
use Flow\Filesystem\Path;
-use Flow\Serializer\Serializer;
+use Flow\Floe\Exception\FloeException;
+use Flow\Floe\FloeReader;
+use Flow\Floe\FloeWriter;
+use Flow\Floe\RowsValueMapper;
+use Generator;
+use function count;
use function str_split;
use function substr;
@@ -21,11 +25,19 @@
{
private Path $cacheDir;
+ /**
+ * @param int<1, max> $serializerBatchSize
+ */
public function __construct(
private Filesystem $filesystem,
- private Serializer $serializer,
- ?Path $cacheDir,
+ ?Path $cacheDir = null,
+ private int $serializerBatchSize = 1000,
) {
+ // @mago-ignore analysis:impossible-condition,redundant-comparison
+ if ($this->serializerBatchSize < 1) {
+ throw new InvalidArgumentException('Serializer batch size must be at least 1');
+ }
+
$this->cacheDir = $cacheDir ?? $this->filesystem->getSystemTmpDir();
}
@@ -39,7 +51,7 @@ public function delete(string $key): void
$this->filesystem->rm($this->cachePath($key));
}
- public function get(string $key): Row|Rows|CacheIndex
+ public function get(string $key): Rows
{
$path = $this->cachePath($key);
@@ -47,12 +59,26 @@ public function get(string $key): Row|Rows|CacheIndex
throw new KeyNotInCacheException($key);
}
- $stream = $this->filesystem->readFrom($path);
+ $file = (new FloeReader($this->filesystem))->read($path);
+
+ try {
+ $footer = $file->footer();
+ $rows = [];
- $serializedValue = $stream->content();
- $stream->close();
+ foreach ($file->recover($this->serializerBatchSize) as $batch) {
+ foreach ($batch->all() as $row) {
+ $rows[] = $row;
+ }
+ }
- return $this->serializer->unserialize($serializedValue, [Row::class, Rows::class, CacheIndex::class]);
+ if (count($rows) !== $footer->totalRows) {
+ throw new KeyNotInCacheException($key);
+ }
+
+ return RowsValueMapper::wrap(RowsValueMapper::reconstructFrom($rows, $footer));
+ } catch (FloeException) {
+ throw new KeyNotInCacheException($key);
+ }
}
public function has(string $key): bool
@@ -60,17 +86,46 @@ public function has(string $key): bool
return $this->filesystem->status($this->cachePath($key)) !== null;
}
- public function set(string $key, CacheIndex|Rows|Row $value): void
+ public function read(string $key): Generator
{
- $cacheStream = $this->filesystem->writeTo($this->cachePath($key));
- $cacheStream->append($this->serializer->serialize($value));
- $cacheStream->close();
+ $path = $this->cachePath($key);
+
+ if (!$this->filesystem->status($path)) {
+ throw new KeyNotInCacheException($key);
+ }
+
+ $file = (new FloeReader($this->filesystem))->read($path);
+
+ try {
+ $partitions = RowsValueMapper::partitionsFrom($file->footer());
+ } catch (FloeException) {
+ throw new KeyNotInCacheException($key);
+ }
+
+ foreach ($file->recover($this->serializerBatchSize) as $batch) {
+ // recover() attaches partitions in on-wire order; the caller-visible order is footer metadata
+ yield $partitions === [] ? $batch : Rows::partitioned($batch->all(), $partitions);
+ }
+ }
+
+ public function set(string $key, Rows $value): void
+ {
+ $writer = new FloeWriter($this->filesystem);
+ $writer->create($this->cachePath($key), RowsValueMapper::metadataFor($value));
+
+ // an empty Rows still crosses once - chunks() would yield nothing and an
+ // empty-but-partitioned value would lose its PARTITIONS frame (byte parity)
+ foreach ($value->count() === 0 ? [$value] : $value->chunks($this->serializerBatchSize) as $chunk) {
+ $writer->write($chunk);
+ }
+
+ $writer->close();
}
private function cachePath(string $key): Path
{
return $this->cacheDir->suffix(
- implode('/', str_split(substr(NativePHPHash::xxh128($key), 0, 8), 2)) . '/' . $key . '.php.cache',
+ implode('/', str_split(substr(NativePHPHash::xxh128($key), 0, 8), 2)) . '/' . $key . '.floe',
);
}
}
diff --git a/src/core/etl/src/Flow/ETL/Cache/Implementation/InMemoryCache.php b/src/core/etl/src/Flow/ETL/Cache/Implementation/InMemoryCache.php
index d0f992ae83..4f7749948e 100644
--- a/src/core/etl/src/Flow/ETL/Cache/Implementation/InMemoryCache.php
+++ b/src/core/etl/src/Flow/ETL/Cache/Implementation/InMemoryCache.php
@@ -5,17 +5,16 @@
namespace Flow\ETL\Cache\Implementation;
use Flow\ETL\Cache;
-use Flow\ETL\Cache\CacheIndex;
use Flow\ETL\Exception\KeyNotInCacheException;
-use Flow\ETL\Row;
use Flow\ETL\Rows;
+use Generator;
use function array_key_exists;
final class InMemoryCache implements Cache
{
/**
- * @var array
+ * @var array
*/
private array $cache = [];
@@ -38,7 +37,7 @@ public function delete(string $key): void
/**
* @throws KeyNotInCacheException
*/
- public function get(string $key): Row|Rows|CacheIndex
+ public function get(string $key): Rows
{
if (!array_key_exists($key, $this->cache)) {
throw new KeyNotInCacheException($key);
@@ -52,7 +51,17 @@ public function has(string $key): bool
return array_key_exists($key, $this->cache);
}
- public function set(string $key, CacheIndex|Rows|Row $value): void
+ /**
+ * @throws KeyNotInCacheException
+ *
+ * @return Generator
+ */
+ public function read(string $key): Generator
+ {
+ yield $this->get($key);
+ }
+
+ public function set(string $key, Rows $value): void
{
$this->cache[$key] = $value;
}
diff --git a/src/core/etl/src/Flow/ETL/Cache/Implementation/PSRSimpleCache.php b/src/core/etl/src/Flow/ETL/Cache/Implementation/PSRSimpleCache.php
index aa50eedb60..2244dc7a29 100644
--- a/src/core/etl/src/Flow/ETL/Cache/Implementation/PSRSimpleCache.php
+++ b/src/core/etl/src/Flow/ETL/Cache/Implementation/PSRSimpleCache.php
@@ -6,12 +6,12 @@
use DateInterval;
use Flow\ETL\Cache;
-use Flow\ETL\Cache\CacheIndex;
use Flow\ETL\Exception\KeyNotInCacheException;
-use Flow\ETL\Row;
use Flow\ETL\Rows;
-use Flow\Serializer\NativePHPSerializer;
+use Flow\Floe\FloeSerializer;
+use Flow\Serializer\Exception\SerializationException;
use Flow\Serializer\Serializer;
+use Generator;
use Psr\SimpleCache\CacheInterface;
use Psr\SimpleCache\InvalidArgumentException;
@@ -22,7 +22,7 @@
public function __construct(
private CacheInterface $cache,
private int|DateInterval|null $ttl = null,
- private Serializer $serializer = new NativePHPSerializer(),
+ private Serializer $serializer = new FloeSerializer(),
) {}
public function clear(): void
@@ -35,7 +35,7 @@ public function delete(string $key): void
$this->cache->delete($key);
}
- public function get(string $key): Row|Rows|CacheIndex
+ public function get(string $key): Rows
{
// @mago-ignore analysis:mixed-assignment
$serializedValue = $this->cache->get($key);
@@ -44,10 +44,11 @@ public function get(string $key): Row|Rows|CacheIndex
throw new KeyNotInCacheException($key);
}
- return $this->serializer->unserialize(
- is_string($serializedValue) ? $serializedValue : '',
- [Row::class, Rows::class, CacheIndex::class],
- );
+ try {
+ return $this->serializer->unserialize(is_string($serializedValue) ? $serializedValue : '', [Rows::class]);
+ } catch (SerializationException) {
+ throw new KeyNotInCacheException($key);
+ }
}
public function has(string $key): bool
@@ -59,7 +60,17 @@ public function has(string $key): bool
}
}
- public function set(string $key, CacheIndex|Rows|Row $value): void
+ /**
+ * @throws KeyNotInCacheException
+ *
+ * @return Generator
+ */
+ public function read(string $key): Generator
+ {
+ yield $this->get($key);
+ }
+
+ public function set(string $key, Rows $value): void
{
$this->cache->set($key, $this->serializer->serialize($value), $this->ttl);
}
diff --git a/src/core/etl/src/Flow/ETL/Cache/Implementation/TraceableCache.php b/src/core/etl/src/Flow/ETL/Cache/Implementation/TraceableCache.php
index 4e8e82741a..40df264236 100644
--- a/src/core/etl/src/Flow/ETL/Cache/Implementation/TraceableCache.php
+++ b/src/core/etl/src/Flow/ETL/Cache/Implementation/TraceableCache.php
@@ -6,10 +6,8 @@
use DateTimeImmutable;
use Flow\ETL\Cache;
-use Flow\ETL\Cache\CacheIndex;
use Flow\ETL\Config\Telemetry\TelemetryAttributes;
use Flow\ETL\Exception\KeyNotInCacheException;
-use Flow\ETL\Row;
use Flow\ETL\Rows;
use Flow\Telemetry\CacheAttributes;
use Flow\Telemetry\Meter\Instrument\Counter;
@@ -19,6 +17,7 @@
use Flow\Telemetry\Tracer\SpanKind;
use Flow\Telemetry\Tracer\SpanStatus;
use Flow\Telemetry\Tracer\Tracer;
+use Generator;
use Throwable;
final readonly class TraceableCache implements Cache
@@ -79,7 +78,7 @@ public function delete(string $key): void
}
}
- public function get(string $key): Row|Rows|CacheIndex
+ public function get(string $key): Rows
{
$attributes = [TelemetryAttributes::ATTR_DATAFRAME_NAME => $this->dataframeName];
@@ -95,6 +94,29 @@ public function get(string $key): Row|Rows|CacheIndex
}
}
+ /**
+ * @throws KeyNotInCacheException
+ *
+ * @return Generator
+ */
+ public function read(string $key): Generator
+ {
+ $attributes = [TelemetryAttributes::ATTR_DATAFRAME_NAME => $this->dataframeName];
+ $batches = $this->cache->read($key);
+
+ try {
+ $batches->rewind();
+ } catch (KeyNotInCacheException $exception) {
+ $this->missCounter->add(1, $attributes);
+
+ throw $exception;
+ }
+
+ $this->hitCounter->add(1, $attributes);
+
+ yield from $batches;
+ }
+
public function has(string $key): bool
{
$attributes = [TelemetryAttributes::ATTR_DATAFRAME_NAME => $this->dataframeName];
@@ -109,18 +131,12 @@ public function has(string $key): bool
return $exists;
}
- public function set(string $key, Row|Rows|CacheIndex $value): void
+ public function set(string $key, Rows $value): void
{
- $valueType = match (true) {
- $value instanceof Row => 'Row',
- $value instanceof Rows => 'Rows',
- $value instanceof CacheIndex => 'CacheIndex',
- };
-
$span = $this->tracer->span(CacheAttributes::SPAN_SET, SpanKind::CLIENT, [
CacheAttributes::CACHE_OPERATION => 'set',
CacheAttributes::CACHE_KEY => $key,
- CacheAttributes::CACHE_VALUE_TYPE => $valueType,
+ CacheAttributes::CACHE_VALUE_TYPE => 'Rows',
]);
try {
diff --git a/src/core/etl/src/Flow/ETL/Config/Cache/CacheConfig.php b/src/core/etl/src/Flow/ETL/Config/Cache/CacheConfig.php
index bcaa29d102..72626f6ac8 100644
--- a/src/core/etl/src/Flow/ETL/Config/Cache/CacheConfig.php
+++ b/src/core/etl/src/Flow/ETL/Config/Cache/CacheConfig.php
@@ -18,6 +18,6 @@ public function __construct(
public Cache $cache,
public Path $localFilesystemCacheDir,
public int $externalSortBucketsCount,
- public string $filesystemProtocol = 'file',
+ public string $filesystemMount = 'file',
) {}
}
diff --git a/src/core/etl/src/Flow/ETL/Config/Cache/CacheConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/Cache/CacheConfigBuilder.php
index e8913cda44..5d3f4c66d8 100644
--- a/src/core/etl/src/Flow/ETL/Config/Cache/CacheConfigBuilder.php
+++ b/src/core/etl/src/Flow/ETL/Config/Cache/CacheConfigBuilder.php
@@ -10,7 +10,6 @@
use Flow\ETL\Config\Telemetry\TelemetryConfig;
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\Filesystem\FilesystemTable;
-use Flow\Serializer\Serializer;
use function Flow\Filesystem\DSL\path_real;
use function getenv;
@@ -25,11 +24,15 @@ final class CacheConfigBuilder
*/
private int $externalSortBucketsCount = 100;
- private string $filesystemProtocol = 'file';
+ private string $filesystemMount = 'file';
+
+ /**
+ * @var int<1, max>
+ */
+ private int $serializerBatchSize = 1000;
public function build(
FilesystemTable $fstab,
- Serializer $serializer,
?TelemetryConfig $telemetryConfig = null,
string $dataframeName = 'flow_dataframe',
): CacheConfig {
@@ -37,9 +40,9 @@ public function build(
$cachePath = path_real($cachePath !== '' ? $cachePath : sys_get_temp_dir() . '/flow_php/cache');
$cache = $this->cache ?? new FilesystemCache(
- $fstab->for($this->filesystemProtocol),
- $serializer,
+ $fstab->for($this->filesystemMount),
cacheDir: $cachePath,
+ serializerBatchSize: $this->serializerBatchSize,
);
if ($telemetryConfig !== null && $telemetryConfig->options->traceCache) {
@@ -50,7 +53,7 @@ public function build(
cache: $cache,
localFilesystemCacheDir: $cachePath,
externalSortBucketsCount: $this->externalSortBucketsCount,
- filesystemProtocol: $this->filesystemProtocol,
+ filesystemMount: $this->filesystemMount,
);
}
@@ -76,9 +79,22 @@ public function externalSortBucketsCount(int $externalSortBucketsCount): self
return $this;
}
- public function filesystemProtocol(string $protocol): self
+ public function filesystemMount(string $mount): self
+ {
+ $this->filesystemMount = $mount;
+
+ return $this;
+ }
+
+ /**
+ * Serializer batch size for the default FilesystemCache; ignored when a custom Cache was
+ * injected via cache().
+ *
+ * @param int<1, max> $serializerBatchSize
+ */
+ public function serializerBatchSize(int $serializerBatchSize): self
{
- $this->filesystemProtocol = $protocol;
+ $this->serializerBatchSize = $serializerBatchSize;
return $this;
}
diff --git a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php
index 02606bbd83..a51f741215 100644
--- a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php
+++ b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php
@@ -22,8 +22,7 @@
use Flow\ETL\Row\EntryFactory;
use Flow\Filesystem\Filesystem;
use Flow\Filesystem\FilesystemTable;
-use Flow\Serializer\Base64Serializer;
-use Flow\Serializer\NativePHPSerializer;
+use Flow\Floe\FloeSerializer;
use Flow\Serializer\Serializer;
use Flow\Telemetry\PackageVersion;
use Flow\Telemetry\Telemetry;
@@ -89,7 +88,7 @@ public function analyze(Analyze $analyze): self
public function build(EntryFactory $entryFactory = new EntryFactory()): Config
{
$this->id ??= 'flow-php-' . $this->randomValueGenerator->string(32);
- $this->serializer ??= new Base64Serializer(new NativePHPSerializer());
+ $this->serializer ??= new FloeSerializer();
$this->optimizer ??= new Optimizer(new LimitOptimization(), new BatchSizeOptimization(batchSize: 1000));
$serializer = $this->serializer;
@@ -107,7 +106,7 @@ public function build(EntryFactory $entryFactory = new EntryFactory()): Config
$optimizer,
$this->putInputIntoRows,
$entryFactory,
- $this->cache->build($this->fstab(), $serializer, $this->telemetryConfig, $dataframeName),
+ $this->cache->build($this->fstab(), $this->telemetryConfig, $dataframeName),
$this->sort->build(),
$this->analyze,
$this->telemetryConfig ?? TelemetryConfig::default($this->getClock()),
@@ -123,7 +122,17 @@ public function cache(Cache $cache): self
public function cacheFilesystem(string $protocol): self
{
- $this->cache->filesystemProtocol($protocol);
+ $this->cache->filesystemMount($protocol);
+
+ return $this;
+ }
+
+ /**
+ * @param int<1, max> $serializerBatchSize
+ */
+ public function cacheSerializerBatchSize(int $serializerBatchSize): self
+ {
+ $this->cache->serializerBatchSize($serializerBatchSize);
return $this;
}
diff --git a/src/core/etl/src/Flow/ETL/DSL/functions.php b/src/core/etl/src/Flow/ETL/DSL/functions.php
index b8919c2243..224597c04b 100644
--- a/src/core/etl/src/Flow/ETL/DSL/functions.php
+++ b/src/core/etl/src/Flow/ETL/DSL/functions.php
@@ -253,8 +253,6 @@
use Flow\Filesystem\Path;
use Flow\Filesystem\Stream\Mode;
use Flow\Filesystem\Telemetry\FilesystemTelemetryOptions;
-use Flow\Serializer\NativePHPSerializer;
-use Flow\Serializer\Serializer;
use Flow\Types\Type;
use Flow\Types\Type\Logical\DateTimeType;
use Flow\Types\Type\Logical\DateType;
@@ -411,13 +409,20 @@ function files(string|Path $directory): FilesExtractor
return new FilesExtractor(is_string($directory) ? path($directory) : $directory);
}
+/**
+ * @param int<1, max> $serializer_batch_size
+ */
#[DocumentationDSL(module: Module::CORE, type: DSLType::DATA_FRAME)]
function filesystem_cache(
Path|string|null $cache_dir = null,
Filesystem $filesystem = new NativeLocalFilesystem(),
- Serializer $serializer = new NativePHPSerializer(),
+ int $serializer_batch_size = 1000,
): FilesystemCache {
- return new FilesystemCache($filesystem, $serializer, is_string($cache_dir) ? path_real($cache_dir) : $cache_dir);
+ return new FilesystemCache(
+ $filesystem,
+ is_string($cache_dir) ? path_real($cache_dir) : $cache_dir,
+ $serializer_batch_size,
+ );
}
/**
diff --git a/src/core/etl/src/Flow/ETL/Extractor/CacheExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/CacheExtractor.php
index e70142e145..b67cc9bf66 100644
--- a/src/core/etl/src/Flow/ETL/Extractor/CacheExtractor.php
+++ b/src/core/etl/src/Flow/ETL/Extractor/CacheExtractor.php
@@ -36,17 +36,15 @@ public function extract(FlowContext $context): Generator
}
}
} else {
- /** @var CacheIndex $index */
- $index = $context->cache()->get($this->id);
+ $index = CacheIndex::fromRows($this->id, $context->cache()->get($this->id));
foreach ($index->values() as $cacheKey) {
- /** @var Rows $rows */
- $rows = $context->cache()->get($cacheKey);
-
- $signal = yield $rows;
+ foreach ($context->cache()->read($cacheKey) as $rows) {
+ $signal = yield $rows;
- if ($signal === Signal::STOP) {
- return;
+ if ($signal === Signal::STOP) {
+ return;
+ }
}
if ($this->clear) {
diff --git a/src/core/etl/src/Flow/ETL/Processor/CachingProcessor.php b/src/core/etl/src/Flow/ETL/Processor/CachingProcessor.php
index 2e820de278..eda366cf60 100644
--- a/src/core/etl/src/Flow/ETL/Processor/CachingProcessor.php
+++ b/src/core/etl/src/Flow/ETL/Processor/CachingProcessor.php
@@ -47,6 +47,6 @@ public function process(Generator $rows, FlowContext $context): Generator
yield $batch;
}
- $context->cache()->set($id, $index);
+ $context->cache()->set($id, $index->toRows());
}
}
diff --git a/src/core/etl/src/Flow/ETL/Processor/PartitioningProcessor.php b/src/core/etl/src/Flow/ETL/Processor/PartitioningProcessor.php
index 6c1a40a017..f7e408828e 100644
--- a/src/core/etl/src/Flow/ETL/Processor/PartitioningProcessor.php
+++ b/src/core/etl/src/Flow/ETL/Processor/PartitioningProcessor.php
@@ -83,7 +83,7 @@ public function process(Generator $rows, FlowContext $context): Generator
}
foreach ($partitionIndexes as $partitionIndex) {
- $context->cache()->set($partitionIndex->key, $partitionIndex);
+ $context->cache()->set($partitionIndex->key, $partitionIndex->toRows());
}
yield from from_all(...array_map(
diff --git a/src/core/etl/src/Flow/ETL/Processor/SortingProcessor.php b/src/core/etl/src/Flow/ETL/Processor/SortingProcessor.php
index d8d34a0877..701f904e90 100644
--- a/src/core/etl/src/Flow/ETL/Processor/SortingProcessor.php
+++ b/src/core/etl/src/Flow/ETL/Processor/SortingProcessor.php
@@ -55,9 +55,7 @@ private function externalSort(Generator $rows, FlowContext $context): Generator
return (new ExternalSort(
new FilesystemBucketsCache(
$context->filesystem($context->config->sort->filesystemProtocol),
- $context->config->serializer(),
- 100,
- $context->config->cache->localFilesystemCacheDir->suffix('/flow-php-external-sort/'),
+ cacheDir: $context->config->cache->localFilesystemCacheDir->suffix('/flow-php-external-sort/'),
),
$context->config->cache->externalSortBucketsCount,
))->sortGenerator($rows, $context, $this->refs);
diff --git a/src/core/etl/src/Flow/ETL/Row/Entries.php b/src/core/etl/src/Flow/ETL/Row/Entries.php
index 39e51d8109..9caa46914d 100644
--- a/src/core/etl/src/Flow/ETL/Row/Entries.php
+++ b/src/core/etl/src/Flow/ETL/Row/Entries.php
@@ -16,6 +16,7 @@
use IteratorAggregate;
use function array_key_exists;
+use function array_keys;
use function array_map;
use function array_merge;
use function array_values;
@@ -209,6 +210,14 @@ public function merge(self $entries): self
return self::recreate($newEntries);
}
+ /**
+ * @return array
+ */
+ public function names(): array
+ {
+ return array_keys($this->entries);
+ }
+
/**
* @param array-key $offset
*
@@ -430,11 +439,14 @@ private function find(string|Reference $reference): ?Entry
/**
* Internal function used to create entries that are already indexed and validated against duplicates.
- * It comes with a significant performance boost, only to be used inside of this collection.
+ * It comes with a significant performance boost, only to be used on trusted hot paths where the
+ * caller guarantees keys equal entry names and are unique (this collection, Floe row hydration).
*
* @param array> $entries
+ *
+ * @internal
*/
- private static function recreate(array $entries): self
+ public static function recreate(array $entries): self
{
$instance = new self();
$instance->entries = $entries;
diff --git a/src/core/etl/src/Flow/ETL/Rows.php b/src/core/etl/src/Flow/ETL/Rows.php
index 5ced9f6dc7..3ac377eec2 100644
--- a/src/core/etl/src/Flow/ETL/Rows.php
+++ b/src/core/etl/src/Flow/ETL/Rows.php
@@ -30,7 +30,6 @@
use Iterator;
use IteratorAggregate;
-use function array_chunk;
use function array_filter;
use function array_map;
use function array_merge;
@@ -60,7 +59,7 @@ final class Rows implements ArrayAccess, Countable, IteratorAggregate
/**
* @var array
*/
- private readonly array $rows;
+ private array $rows;
private ?Schema $schema = null;
@@ -129,9 +128,35 @@ public function all(): array
*/
public function chunks(int $size): Generator
{
- foreach (array_chunk($this->rows, $size) as $chunk) {
- yield self::partitioned($chunk, $this->partitions);
+ $chunk = [];
+
+ foreach ($this->rows as $row) {
+ $chunk[] = $row;
+
+ if (count($chunk) === $size) {
+ yield self::fromList($chunk, $this->partitions);
+ $chunk = [];
+ }
}
+
+ if (count($chunk)) {
+ yield self::fromList($chunk, $this->partitions);
+ }
+ }
+
+ /**
+ * Adopts an already-built list of rows without the variadic re-copy - chunking
+ * a large Rows must not copy every row pointer three times.
+ *
+ * @param array $rows
+ */
+ private static function fromList(array $rows, Partitions $partitions): self
+ {
+ $instance = new self();
+ $instance->rows = $rows;
+ $instance->partitions = $partitions;
+
+ return $instance;
}
public function count(): int
diff --git a/src/core/etl/src/Flow/ETL/Sort/ExternalSort/BucketsCache/FilesystemBucketsCache.php b/src/core/etl/src/Flow/ETL/Sort/ExternalSort/BucketsCache/FilesystemBucketsCache.php
index adb29c3fdc..37fa47910c 100644
--- a/src/core/etl/src/Flow/ETL/Sort/ExternalSort/BucketsCache/FilesystemBucketsCache.php
+++ b/src/core/etl/src/Flow/ETL/Sort/ExternalSort/BucketsCache/FilesystemBucketsCache.php
@@ -4,37 +4,28 @@
namespace Flow\ETL\Sort\ExternalSort\BucketsCache;
-use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Hash\NativePHPHash;
use Flow\ETL\Row;
use Flow\ETL\Rows;
use Flow\ETL\Sort\ExternalSort\BucketsCache;
use Flow\Filesystem\Filesystem;
use Flow\Filesystem\Path;
-use Flow\Serializer\NativePHPSerializer;
-use Flow\Serializer\Serializer;
+use Flow\Floe\FloeReader;
+use Flow\Floe\FloeWriter;
use Generator;
+use function count;
+
final readonly class FilesystemBucketsCache implements BucketsCache
{
+ private const int WRITE_BATCH_SIZE = 1000;
+
private Path $cacheDir;
- /**
- * @param Filesystem $filesystem
- * @param Serializer $serializer
- * @param int<1, max> $chunkSize - number of rows to be written into cache in one go, higher number can reduce IO but increase memory consumption
- */
public function __construct(
private Filesystem $filesystem,
- private Serializer $serializer = new NativePHPSerializer(),
- private int $chunkSize = 100,
?Path $cacheDir = null,
) {
- // @mago-ignore analysis:impossible-condition,redundant-comparison
- if ($this->chunkSize < 1) {
- throw new InvalidArgumentException('Chunk size must be greater than 0');
- }
-
$this->cacheDir = ($cacheDir ?? $this->filesystem->getSystemTmpDir())->suffix('/flow-php-external-sort/');
}
@@ -49,13 +40,11 @@ public function get(string $bucketId): Generator
return;
}
- $stream = $this->filesystem->readFrom($path);
-
- foreach ($stream->readLines() as $serializedRow) {
- yield $this->serializer->unserialize($serializedRow, [Row::class]);
+ foreach ((new FloeReader($this->filesystem))
+ ->read($path)
+ ->recover() as $batch) {
+ yield from $batch->all();
}
-
- $stream->close();
}
public function remove(string $bucketId): void
@@ -70,33 +59,29 @@ public function remove(string $bucketId): void
*/
public function set(string $bucketId, iterable $rows): void
{
- $path = $this->keyPath($bucketId);
-
- $stream = $this->filesystem->writeTo($path);
+ $writer = new FloeWriter($this->filesystem);
+ $writer->create($this->keyPath($bucketId));
- $serializedRows = '';
- $counter = 0;
+ $batch = [];
foreach ($rows as $row) {
- $serializedRows .= $this->serializer->serialize($row) . "\n";
- $counter++;
+ $batch[] = $row;
- if ($counter >= $this->chunkSize) {
- $stream->append($serializedRows);
- $serializedRows = '';
- $counter = 0;
+ if (count($batch) >= self::WRITE_BATCH_SIZE) {
+ $writer->write(new Rows(...$batch));
+ $batch = [];
}
}
- if ($counter > 0) {
- $stream->append($serializedRows);
+ if ($batch !== []) {
+ $writer->write(new Rows(...$batch));
}
- $stream->close();
+ $writer->close();
}
private function keyPath(string $key): Path
{
- return $this->cacheDir->suffix(NativePHPHash::xxh128($key) . '/' . $key . '.php.cache');
+ return $this->cacheDir->suffix(NativePHPHash::xxh128($key) . '/' . $key . '.floe');
}
}
diff --git a/src/core/etl/src/Flow/ETL/Transformer/SerializeTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/SerializeTransformer.php
index 3f970eec65..1b7bc8a8ea 100644
--- a/src/core/etl/src/Flow/ETL/Transformer/SerializeTransformer.php
+++ b/src/core/etl/src/Flow/ETL/Transformer/SerializeTransformer.php
@@ -10,6 +10,7 @@
use Flow\ETL\Row\Reference;
use Flow\ETL\Rows;
use Flow\ETL\Transformer;
+use Flow\Serializer\Base64Serializer;
use Throwable;
use function Flow\ETL\DSL\ref;
@@ -29,10 +30,12 @@ public function transform(Rows $rows, FlowContext $context): Rows
try {
$target = $this->target instanceof Reference ? $this->target : ref($this->target);
+ // base64 keeps serialized rows text-safe inside string entries, no matter which serializer is configured
+ $serializer = new Base64Serializer($context->config->serializer());
$result = $rows->map(fn(Row $row) => $this->standalone
- ? row(str_entry($target->name(), $context->config->serializer()->serialize($row)))
- : $row->add(str_entry($target->name(), $context->config->serializer()->serialize($row))));
+ ? row(str_entry($target->name(), $serializer->serialize($row)))
+ : $row->add(str_entry($target->name(), $serializer->serialize($row))));
$context->telemetry()->transformationCompleted($this, [
TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(),
diff --git a/src/core/etl/src/Flow/ETL/Transformer/UnserializeTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/UnserializeTransformer.php
index 2f89a43668..485a1a4b27 100644
--- a/src/core/etl/src/Flow/ETL/Transformer/UnserializeTransformer.php
+++ b/src/core/etl/src/Flow/ETL/Transformer/UnserializeTransformer.php
@@ -10,6 +10,7 @@
use Flow\ETL\Row\Reference;
use Flow\ETL\Rows;
use Flow\ETL\Transformer;
+use Flow\Serializer\Base64Serializer;
use Flow\Serializer\Exception\SerializationException;
use Throwable;
@@ -35,8 +36,10 @@ public function transform(Rows $rows, FlowContext $context): Rows
try {
$source = $this->source instanceof Reference ? $this->source : ref($this->source);
+ // base64 keeps serialized rows text-safe inside string entries, no matter which serializer is configured
+ $serializer = new Base64Serializer($context->config->serializer());
- $result = $rows->map(function (Row $row) use ($source, $context): Row {
+ $result = $rows->map(function (Row $row) use ($source, $serializer): Row {
if (!$row->has($source->name())) {
return $row;
}
@@ -50,11 +53,8 @@ public function transform(Rows $rows, FlowContext $context): Rows
try {
return (
$this->merge
- ? $row->merge(
- $context->config->serializer()->unserialize($serialized, [Row::class]),
- $this->mergePrefix,
- )
- : $context->config->serializer()->unserialize($serialized, [Row::class])
+ ? $row->merge($serializer->unserialize($serialized, [Row::class]), $this->mergePrefix)
+ : $serializer->unserialize($serialized, [Row::class])
);
} catch (SerializationException) {
return $row;
diff --git a/src/core/etl/src/Flow/Floe/Codec.php b/src/core/etl/src/Flow/Floe/Codec.php
new file mode 100644
index 0000000000..15ceb0ee63
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Codec.php
@@ -0,0 +1,18 @@
+ $sources
+ */
+#[DocumentationDSL(module: Module::FLOE, type: DSLType::HELPER)]
+function merge_floe(array $sources, string|Path $dest, bool $compact = false, ?Metadata $metadata = null): void
+{
+ (new FloeMerger(native_local_filesystem()))->merge(
+ array_map(static fn(string|Path $source): Path => is_string($source) ? path_real($source) : $source, $sources),
+ is_string($dest) ? path($dest) : $dest,
+ $compact,
+ $metadata,
+ );
+}
diff --git a/src/core/etl/src/Flow/Floe/Decoding/BooleanDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/BooleanDecoder.php
new file mode 100644
index 0000000000..885ff07310
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Decoding/BooleanDecoder.php
@@ -0,0 +1,13 @@
+setTimezone($this->timeZones->get($timezoneName));
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Decoding/DynamicDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/DynamicDecoder.php
new file mode 100644
index 0000000000..71a4fd01dc
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Decoding/DynamicDecoder.php
@@ -0,0 +1,81 @@
+decode($data, $position);
+ }
+
+ return $value;
+ case Format::TAG_DATETIME:
+ return $this->dateTime->decode($data, $position);
+ case Format::TAG_UUID:
+ return $this->uuid->decode($data, $position);
+ case Format::TAG_JSON:
+ return $this->json->decode($data, $position);
+ default:
+ throw new FloeException(sprintf('Floe found unknown dynamic value tag 0x%02X', $tag));
+ }
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Decoding/EnumDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/EnumDecoder.php
new file mode 100644
index 0000000000..3763a976b8
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Decoding/EnumDecoder.php
@@ -0,0 +1,39 @@
+y = $fields['y'];
+ // @mago-ignore analysis:invalid-property-write
+ $value->m = $fields['m'];
+ // @mago-ignore analysis:invalid-property-write
+ $value->d = $fields['d'];
+ // @mago-ignore analysis:invalid-property-write
+ $value->h = $fields['h'];
+ // @mago-ignore analysis:invalid-property-write
+ $value->i = $fields['i'];
+ // @mago-ignore analysis:invalid-property-write
+ $value->s = $fields['s'];
+ // @mago-ignore analysis:invalid-property-write
+ $value->f = $fraction;
+ // @mago-ignore analysis:invalid-property-write
+ $value->invert = $invert;
+
+ return $value;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Decoding/JsonDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/JsonDecoder.php
new file mode 100644
index 0000000000..695d23faac
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Decoding/JsonDecoder.php
@@ -0,0 +1,51 @@
+newInstanceWithoutConstructor();
+ // @mago-ignore analysis:invalid-property-write
+ $json->value = $value;
+ // @mago-ignore analysis:invalid-property-write
+ $json->isObject = $isObject;
+
+ return $json;
+ },
+ null,
+ Json::class,
+ );
+
+ $this->create = $create;
+ }
+
+ public function decode(string $data, int &$position): Json
+ {
+ $length = unpack('V', $data, $position)[1];
+ $json = substr($data, $position + 4, $length);
+ $position += 4 + $length;
+
+ return ($this->create)($json, $data[$position++] === "\x01");
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Decoding/ListDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/ListDecoder.php
new file mode 100644
index 0000000000..7bb9f1d9ec
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Decoding/ListDecoder.php
@@ -0,0 +1,30 @@
+
+ */
+ public function decode(string $data, int &$position): array
+ {
+ $count = unpack('V', $data, $position)[1];
+ $position += 4;
+ $values = [];
+
+ for ($i = 0; $i < $count; $i++) {
+ $values[] = $this->element->decode($data, $position);
+ }
+
+ return $values;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Decoding/MapDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/MapDecoder.php
new file mode 100644
index 0000000000..4213c1852e
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Decoding/MapDecoder.php
@@ -0,0 +1,33 @@
+
+ */
+ public function decode(string $data, int &$position): array
+ {
+ $count = unpack('V', $data, $position)[1];
+ $position += 4;
+ $values = [];
+
+ for ($i = 0; $i < $count; $i++) {
+ /** @var array-key $key */
+ $key = $this->key->decode($data, $position);
+ $values[$key] = $this->value->decode($data, $position);
+ }
+
+ return $values;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Decoding/NullDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/NullDecoder.php
new file mode 100644
index 0000000000..99ffeef51a
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Decoding/NullDecoder.php
@@ -0,0 +1,13 @@
+base->decode($data, $position);
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Decoding/PackedListDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/PackedListDecoder.php
new file mode 100644
index 0000000000..5596dc5f05
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Decoding/PackedListDecoder.php
@@ -0,0 +1,37 @@
+
+ */
+ public function decode(string $data, int &$position): array
+ {
+ $count = unpack('V', $data, $position)[1];
+ $position += 4;
+
+ if ($count === 0) {
+ return [];
+ }
+
+ $values = array_values(unpack($this->format . '*', substr($data, $position, $count * 8)));
+ $position += $count * 8;
+
+ return $values;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Decoding/StringDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/StringDecoder.php
new file mode 100644
index 0000000000..b9bba92ae4
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Decoding/StringDecoder.php
@@ -0,0 +1,20 @@
+ $elements structure element name => decoder
+ */
+ public function __construct(
+ private readonly array $elements,
+ private readonly bool $allowsExtra,
+ private readonly DynamicDecoder $dynamic,
+ ) {}
+
+ /**
+ * @return array
+ */
+ public function decode(string $data, int &$position): array
+ {
+ $structure = [];
+
+ foreach ($this->elements as $name => $element) {
+ $flag = ord($data[$position++]);
+
+ if ($flag === Format::VALUE_PRESENT) {
+ $structure[$name] = $element->decode($data, $position);
+ } elseif ($flag === Format::VALUE_NULL) {
+ $structure[$name] = null;
+ } elseif ($flag !== Format::VALUE_ABSENT) {
+ throw new FloeException(sprintf('Floe found unknown structure element flag 0x%02X', $flag));
+ }
+ }
+
+ if ($this->allowsExtra) {
+ $count = unpack('V', $data, $position)[1];
+ $position += 4;
+
+ for ($i = 0; $i < $count; $i++) {
+ $length = unpack('V', $data, $position)[1];
+ $key = substr($data, $position + 4, $length);
+ $position += 4 + $length;
+ $structure[$key] = $this->dynamic->decode($data, $position);
+ }
+ }
+
+ return $structure;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Decoding/TimeZoneDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/TimeZoneDecoder.php
new file mode 100644
index 0000000000..3c6e1aa1a4
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Decoding/TimeZoneDecoder.php
@@ -0,0 +1,26 @@
+timeZones->get($name);
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Decoding/TimeZones.php b/src/core/etl/src/Flow/Floe/Decoding/TimeZones.php
new file mode 100644
index 0000000000..c247298c3c
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Decoding/TimeZones.php
@@ -0,0 +1,20 @@
+
+ */
+ private array $cache = [];
+
+ public function get(string $name): DateTimeZone
+ {
+ return $this->cache[$name] ??= new DateTimeZone($name);
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Decoding/UuidDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/UuidDecoder.php
new file mode 100644
index 0000000000..99a2478bb1
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Decoding/UuidDecoder.php
@@ -0,0 +1,47 @@
+newInstanceWithoutConstructor();
+ // @mago-ignore analysis:invalid-property-write
+ $uuid->value = $value;
+
+ return $uuid;
+ },
+ null,
+ Uuid::class,
+ );
+
+ $this->create = $create;
+ }
+
+ public function decode(string $data, int &$position): Uuid
+ {
+ $value = ($this->create)(substr($data, $position, 36));
+ $position += 36;
+
+ return $value;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Decoding/ValueDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/ValueDecoder.php
new file mode 100644
index 0000000000..33016d4dd8
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Decoding/ValueDecoder.php
@@ -0,0 +1,13 @@
+ $columns section columns keyed by name
+ * @param string $schemaBody SCHEMA frame body describing this plan's columns
+ */
+ public function __construct(
+ public array $columns,
+ public string $schemaBody,
+ ) {}
+}
diff --git a/src/core/etl/src/Flow/Floe/Encoding/BooleanEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/BooleanEncoder.php
new file mode 100644
index 0000000000..92ae2a07ec
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Encoding/BooleanEncoder.php
@@ -0,0 +1,13 @@
+ chr(Format::DATETIME_IMMUTABLE),
+ DateTime::class => chr(Format::DATETIME_MUTABLE),
+ default => throw new FloeException(sprintf(
+ 'Floe supports only DateTime and DateTimeImmutable, got %s - convert custom datetime instances before writing',
+ $value::class,
+ )),
+ };
+
+ $timezone = $value->getTimezone()->getName();
+
+ return (
+ $head
+ . pack('P', $value->getTimestamp())
+ . pack('V', (int) $value->format('u'))
+ . pack('V', strlen($timezone))
+ . $timezone
+ );
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Encoding/DynamicEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/DynamicEncoder.php
new file mode 100644
index 0000000000..aa97488842
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Encoding/DynamicEncoder.php
@@ -0,0 +1,90 @@
+ $item) {
+ $buffer .= is_int($key)
+ ? chr(Format::KEY_INTEGER) . pack('P', $key)
+ : chr(Format::KEY_STRING) . pack('V', strlen($key)) . $key;
+ $buffer .= $this->encode($item);
+ }
+
+ return $buffer;
+ }
+
+ if ($value instanceof DateTimeInterface) {
+ return chr(Format::TAG_DATETIME) . $this->dateTime->encode($value);
+ }
+
+ if ($value instanceof Uuid) {
+ return chr(Format::TAG_UUID) . $value->toString();
+ }
+
+ if ($value instanceof Json) {
+ $json = $value->toString();
+
+ return chr(Format::TAG_JSON) . pack('V', strlen($json)) . $json . ($value->isObject() ? "\x01" : "\x00");
+ }
+
+ throw new FloeException(sprintf(
+ 'Floe does not support values of type "%s" in mixed/union context',
+ get_debug_type($value),
+ ));
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Encoding/EnumEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/EnumEncoder.php
new file mode 100644
index 0000000000..7ba06b134c
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Encoding/EnumEncoder.php
@@ -0,0 +1,19 @@
+name)) . $value->name;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Encoding/Float64Encoder.php b/src/core/etl/src/Flow/Floe/Encoding/Float64Encoder.php
new file mode 100644
index 0000000000..e2e54a91d9
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Encoding/Float64Encoder.php
@@ -0,0 +1,15 @@
+saveHtml());
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Encoding/HtmlElementEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/HtmlElementEncoder.php
new file mode 100644
index 0000000000..e81acf0fff
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Encoding/HtmlElementEncoder.php
@@ -0,0 +1,16 @@
+invert)
+ . pack('VVVVVV', $value->y, $value->m, $value->d, $value->h, $value->i, $value->s)
+ . pack('e', $value->f)
+ );
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Encoding/JsonEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/JsonEncoder.php
new file mode 100644
index 0000000000..9c4d056bae
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Encoding/JsonEncoder.php
@@ -0,0 +1,19 @@
+toString();
+
+ return pack('V', strlen($json)) . $json . ($value->isObject() ? "\x01" : "\x00");
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Encoding/ListEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/ListEncoder.php
new file mode 100644
index 0000000000..85ca125a62
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Encoding/ListEncoder.php
@@ -0,0 +1,28 @@
+ $value */
+ $buffer = pack('V', count($value));
+
+ // @mago-ignore analysis:mixed-assignment
+ foreach ($value as $item) {
+ $buffer .= $this->element->encode($item);
+ }
+
+ return $buffer;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Encoding/MapEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/MapEncoder.php
new file mode 100644
index 0000000000..45c2b85c68
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Encoding/MapEncoder.php
@@ -0,0 +1,29 @@
+ $value */
+ $buffer = pack('V', count($value));
+
+ // @mago-ignore analysis:mixed-assignment
+ foreach ($value as $key => $item) {
+ $buffer .= $this->key->encode($key) . $this->value->encode($item);
+ }
+
+ return $buffer;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Encoding/NullEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/NullEncoder.php
new file mode 100644
index 0000000000..7741bce503
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Encoding/NullEncoder.php
@@ -0,0 +1,13 @@
+base->encode($value);
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Encoding/PackedListEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/PackedListEncoder.php
new file mode 100644
index 0000000000..060eaff59d
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Encoding/PackedListEncoder.php
@@ -0,0 +1,28 @@
+ $value */
+ return count($value) === 0 ? pack('V', 0) : pack('V', count($value)) . pack($this->format . '*', ...$value);
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Encoding/StringEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/StringEncoder.php
new file mode 100644
index 0000000000..bf541b7033
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Encoding/StringEncoder.php
@@ -0,0 +1,17 @@
+ $elements structure element name => encoder
+ */
+ public function __construct(
+ private readonly array $elements,
+ private readonly bool $allowsExtra,
+ private readonly DynamicEncoder $dynamic,
+ ) {}
+
+ public function encode(mixed $value): string
+ {
+ /** @var array $value */
+ $buffer = '';
+
+ foreach ($this->elements as $name => $element) {
+ if (!array_key_exists($name, $value)) {
+ $buffer .= Format::VALUE_ABSENT_BYTE;
+ } elseif ($value[$name] === null) {
+ $buffer .= Format::VALUE_NULL_BYTE;
+ } else {
+ $buffer .= Format::VALUE_PRESENT_BYTE . $element->encode($value[$name]);
+ }
+ }
+
+ if ($this->allowsExtra) {
+ $extra = array_diff_key($value, $this->elements);
+ $buffer .= pack('V', count($extra));
+
+ // @mago-ignore analysis:mixed-assignment
+ foreach ($extra as $key => $item) {
+ $buffer .= pack('V', strlen((string) $key)) . $key . $this->dynamic->encode($item);
+ }
+ }
+
+ return $buffer;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Encoding/TimeZoneEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/TimeZoneEncoder.php
new file mode 100644
index 0000000000..2ab17e8f80
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Encoding/TimeZoneEncoder.php
@@ -0,0 +1,19 @@
+getName();
+
+ return pack('V', strlen($name)) . $name;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Encoding/UuidEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/UuidEncoder.php
new file mode 100644
index 0000000000..c1af8da043
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Encoding/UuidEncoder.php
@@ -0,0 +1,14 @@
+toString();
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Encoding/ValueEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/ValueEncoder.php
new file mode 100644
index 0000000000..7e25c608b5
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Encoding/ValueEncoder.php
@@ -0,0 +1,14 @@
+): Entry
+ */
+ private readonly Closure $instantiate;
+
+ /**
+ * @param \Closure(string, mixed, Definition): Entry $instantiate
+ */
+ private function __construct(Closure $instantiate)
+ {
+ $this->instantiate = $instantiate;
+ }
+
+ /**
+ * @param class-string> $entryClass
+ */
+ public static function forEntryClass(string $entryClass): self
+ {
+ $reflection = new ReflectionClass($entryClass);
+
+ /** @var \Closure(string, mixed, Definition): Entry $instantiate */
+ $instantiate = Closure::bind(
+ static function (string $name, mixed $value, Definition $definition) use ($reflection): Entry {
+ $entry = $reflection->newInstanceWithoutConstructor();
+ // @mago-ignore analysis:non-existent-property
+ $entry->name = $name;
+ // @mago-ignore analysis:non-existent-property
+ $entry->value = $value;
+ // @mago-ignore analysis:non-existent-property
+ $entry->definition = $definition;
+
+ return $entry;
+ },
+ null,
+ $entryClass,
+ );
+
+ return new self($instantiate);
+ }
+
+ /**
+ * @param Definition $definition
+ *
+ * @return Entry
+ */
+ public function create(string $name, mixed $value, Definition $definition): Entry
+ {
+ return ($this->instantiate)($name, $value, $definition);
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/EntryInstantiator.php b/src/core/etl/src/Flow/Floe/EntryInstantiator.php
new file mode 100644
index 0000000000..5d826cfbae
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/EntryInstantiator.php
@@ -0,0 +1,29 @@
+
+ */
+ private array $factories = [];
+
+ /**
+ * @param class-string> $entryClass
+ */
+ public function factoryFor(string $entryClass): EntryFactory
+ {
+ if (array_key_exists($entryClass, $this->factories)) {
+ return $this->factories[$entryClass];
+ }
+
+ return $this->factories[$entryClass] = EntryFactory::forEntryClass($entryClass);
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Exception/FloeException.php b/src/core/etl/src/Flow/Floe/Exception/FloeException.php
new file mode 100644
index 0000000000..13f2f713d3
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Exception/FloeException.php
@@ -0,0 +1,9 @@
+encoder = new RowsEncoder();
+ }
+
+ public function encode(Rows $rows): array
+ {
+ try {
+ return $this->encoder->rows($rows);
+ } catch (ExtensionException $e) {
+ throw new FloeException($e->getMessage(), 0, $e);
+ }
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/FloeExtractor.php b/src/core/etl/src/Flow/Floe/FloeExtractor.php
new file mode 100644
index 0000000000..59a182f62c
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/FloeExtractor.php
@@ -0,0 +1,123 @@
+resetLimit();
+ }
+
+ /**
+ * @return \Generator
+ */
+ public function extract(FlowContext $context): Generator
+ {
+ $putInputIntoRows = $context->config->shouldPutInputIntoRows();
+ $fileOffset = $this->offset ?? 0;
+
+ foreach ($this->readers($context) as [$reader, $uri]) {
+ $fileRows = $reader->totalRows();
+
+ if ($fileOffset >= $fileRows) {
+ $fileOffset -= $fileRows;
+
+ continue;
+ }
+
+ $limit = $this->limit();
+ $remaining = $limit === null ? null : $limit - $this->yieldedRows;
+
+ foreach ($reader->rows(1000, $fileOffset, $remaining) as $rows) {
+ if ($putInputIntoRows) {
+ $rows = $rows->map(static fn(Row $row): Row => $row->add(str_entry('_input_file_uri', $uri)));
+ }
+
+ $signal = yield $rows;
+
+ foreach ($rows as $row) {
+ $this->incrementReturnedRows();
+ }
+
+ if ($signal === Signal::STOP || $this->reachedLimit()) {
+ return;
+ }
+ }
+
+ $fileOffset = 0;
+ }
+ }
+
+ /**
+ * Footer-only source schema (two ranged reads per file, no row scan).
+ */
+ public function schema(FlowContext $context): Schema
+ {
+ $schema = new Schema();
+
+ foreach ($this->readers($context) as [$reader]) {
+ $schema = $schema->merge($reader->schema());
+ }
+
+ return $schema;
+ }
+
+ public function source(): Path
+ {
+ return $this->path;
+ }
+
+ public function withOffset(int $offset): self
+ {
+ if ($offset < 0) {
+ throw new InvalidArgumentException('Offset must be greater or equal to 0');
+ }
+
+ $this->offset = $offset;
+
+ return $this;
+ }
+
+ /**
+ * @return \Generator
+ */
+ private function readers(FlowContext $context): Generator
+ {
+ foreach ($context->streams()->list($this->path, $this->filter()) as $listed) {
+ $filePath = $listed->path();
+ $listed->close();
+
+ yield [
+ (new FloeReader($context->filesystem($this->path), $this->codec, $this->chunkSize))->read($filePath),
+ $filePath->uri(),
+ ];
+ }
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/FloeFile.php b/src/core/etl/src/Flow/Floe/FloeFile.php
new file mode 100644
index 0000000000..52e3de094b
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/FloeFile.php
@@ -0,0 +1,1022 @@
+schemaDecoder = new SchemaDecoder(new ValueDecoder(), new EntryInstantiator());
+ $this->rowHydrator = new RowHydrator();
+ $this->useExtension = $useExtension ?? extension_loaded('flow_php');
+ }
+
+ /**
+ * @throws FloeException
+ */
+ public function footer(): Footer
+ {
+ if ($this->footer !== null) {
+ return $this->footer;
+ }
+
+ $source = ($this->openSource)();
+ $size = $source->size();
+
+ if ($size === null) {
+ throw new FloeException(sprintf(
+ 'Floe footer requires a sized stream, "%s" does not report its size',
+ $this->uri,
+ ));
+ }
+
+ if ($size < (Format::HEADER_LENGTH + Format::TRAILER_LENGTH)) {
+ throw new FloeException(sprintf(
+ 'Floe file "%s" is torn, too small to hold a header and a trailer',
+ $this->uri,
+ ));
+ }
+
+ $flags = Format::validateHeader($source->read(Format::HEADER_LENGTH, 0));
+
+ if ($flags !== $this->codec->id()) {
+ throw new FloeException(sprintf(
+ 'Floe file "%s" was written with codec 0x%02X, expected 0x%02X',
+ $this->uri,
+ $flags,
+ $this->codec->id(),
+ ));
+ }
+
+ $footerLength = Format::parseTrailer($source->read(Format::TRAILER_LENGTH, $size - Format::TRAILER_LENGTH));
+
+ if (($size - Format::TRAILER_LENGTH - $footerLength) < Format::HEADER_LENGTH) {
+ throw new FloeException(sprintf('Floe file "%s" is torn, footer does not fit inside the file', $this->uri));
+ }
+
+ /** @var int<1, max> $footerLength */
+ $footer = Footer::fromJson($source->read($footerLength, $size - Format::TRAILER_LENGTH - $footerLength));
+ $source->close();
+
+ return $this->footer = $footer;
+ }
+
+ /**
+ * @throws FloeException
+ */
+ public function metadata(): Metadata
+ {
+ return $this->footer()->metadata;
+ }
+
+ /**
+ * Salvages complete frames from the beginning of the file sequentially - no
+ * footer, no ranged reads. Rows are yielded as they were written (no
+ * padding); reading ends silently at the first truncated or invalid frame.
+ *
+ * @param int<1, max> $batchSize
+ *
+ * @throws FloeException
+ *
+ * @return \Generator
+ */
+ public function recover(int $batchSize = 1000): Generator
+ {
+ $frames = (new FrameReader(($this->openSource)(), $this->codec->id(), $this->chunkSize))->frames(lenient: true);
+
+ $extDecoder = $this->useExtension ? new RowsDecoder() : null;
+ $plan = null;
+ $partitions = [];
+ $batch = [];
+ $pending = [];
+
+ foreach ($frames as [$frameType, $frameBody]) {
+ if ($frameType === Format::FRAME_ROW) {
+ if ($extDecoder !== null) {
+ try {
+ $pending[] = $this->codec->decode($frameBody);
+ } catch (Throwable) {
+ break;
+ }
+
+ if (count($pending) === $batchSize) {
+ [$ready, $stop] = $this->flushRecoverPending(
+ $extDecoder,
+ $pending,
+ $batch,
+ $batchSize,
+ $partitions,
+ );
+ $pending = [];
+
+ foreach ($ready as $rows) {
+ yield $rows;
+ }
+
+ if ($stop) {
+ break;
+ }
+ }
+
+ continue;
+ }
+
+ try {
+ $row = $this->hydrate(null, $plan, $this->codec->decode($frameBody));
+ } catch (Throwable) {
+ break;
+ }
+
+ $batch[] = $row;
+
+ if (count($batch) === $batchSize) {
+ yield $this->batch($batch, $partitions);
+ $batch = [];
+ }
+ } elseif ($frameType === Format::FRAME_SCHEMA) {
+ if ($extDecoder !== null && $pending !== []) {
+ [$ready, $stop] = $this->flushRecoverPending(
+ $extDecoder,
+ $pending,
+ $batch,
+ $batchSize,
+ $partitions,
+ );
+ $pending = [];
+
+ foreach ($ready as $rows) {
+ yield $rows;
+ }
+
+ if ($stop) {
+ break;
+ }
+ }
+
+ try {
+ if ($extDecoder !== null) {
+ $extDecoder->schema($frameBody);
+ } else {
+ $plan = $this->schemaDecoder->decode($frameBody);
+ }
+ } catch (Throwable) {
+ break;
+ }
+ } elseif ($frameType === Format::FRAME_PARTITIONS) {
+ if ($extDecoder !== null && $pending !== []) {
+ [$ready, $stop] = $this->flushRecoverPending(
+ $extDecoder,
+ $pending,
+ $batch,
+ $batchSize,
+ $partitions,
+ );
+ $pending = [];
+
+ foreach ($ready as $rows) {
+ yield $rows;
+ }
+
+ if ($stop) {
+ break;
+ }
+ }
+
+ $partitions = self::decodePartitions($frameBody);
+ } elseif ($frameType !== Format::FRAME_FOOTER) {
+ break;
+ }
+ }
+
+ if ($extDecoder !== null && $pending !== []) {
+ [$ready] = $this->flushRecoverPending($extDecoder, $pending, $batch, $batchSize, $partitions);
+
+ foreach ($ready as $rows) {
+ yield $rows;
+ }
+ }
+
+ if ($batch !== []) {
+ yield $this->batch($batch, $partitions);
+ }
+ }
+
+ /**
+ * A failed batch is replayed row by row so the prefix before the first
+ * corrupt body still surfaces; the true stop flag ends the recover read.
+ *
+ * @param array $pending
+ * @param array $batch
+ * @param int<1, max> $batchSize
+ * @param array $partitions
+ *
+ * @return array{0: array, 1: bool} ready batches + stop flag
+ */
+ private function flushRecoverPending(
+ RowsDecoder $extDecoder,
+ array $pending,
+ array &$batch,
+ int $batchSize,
+ array $partitions,
+ ): array {
+ $stop = false;
+
+ try {
+ $rows = $extDecoder->rows($pending)->all();
+ } catch (Throwable) {
+ $rows = [];
+
+ foreach ($pending as $body) {
+ try {
+ $rows[] = $extDecoder->row($body);
+ } catch (Throwable) {
+ break;
+ }
+ }
+
+ $stop = true;
+ }
+
+ $ready = [];
+
+ foreach ($rows as $row) {
+ $batch[] = $row;
+
+ if (count($batch) === $batchSize) {
+ $ready[] = $this->batch($batch, $partitions);
+ $batch = [];
+ }
+ }
+
+ return [$ready, $stop];
+ }
+
+ /**
+ * Hot path: frames are walked in-buffer, unlike recover() which keeps the
+ * reusable FrameReader. offset skips whole leading sections using the footer,
+ * then the remaining rows inside the start section; limit stops the read
+ * after that many rows.
+ *
+ * @param int<1, max> $batchSize
+ *
+ * @throws FloeException
+ *
+ * @return \Generator every batch conforms to the merged file schema
+ */
+ public function rows(int $batchSize = 1000, int $offset = 0, ?int $limit = null): Generator
+ {
+ if ($offset < 0) {
+ throw new FloeException('Floe offset must be greater or equal to 0');
+ }
+
+ if ($limit !== null && $limit < 1) {
+ throw new FloeException('Floe limit must be greater than 0');
+ }
+
+ if ($offset > 0) {
+ yield from $this->rowsFromOffset($batchSize, $offset, $limit);
+
+ return;
+ }
+
+ $footer = $this->footer();
+ $fileSchema = $footer->fileSchema();
+
+ $partitions = [];
+
+ foreach ($footer->partitions as $name => $value) {
+ $partitions[] = new Partition($name, $value);
+ }
+
+ /** @var int<1, max> $chunkSize */
+ $chunkSize = $this->chunkSize;
+ $chunks = ($this->openSource)()->iterate($chunkSize);
+ $buffer = '';
+ $position = 0;
+
+ $fill = static function (int $bytes) use (&$buffer, &$position, $chunks): bool {
+ while ((strlen($buffer) - $position) < $bytes) {
+ if (!$chunks->valid()) {
+ return false;
+ }
+
+ $buffer .= $chunks->current();
+ $chunks->next();
+ }
+
+ return true;
+ };
+
+ if (!$fill(Format::HEADER_LENGTH)) {
+ throw new FloeException('Floe stream is truncated, header is incomplete');
+ }
+
+ $flags = Format::validateHeader(substr($buffer, 0, Format::HEADER_LENGTH));
+
+ if ($flags !== $this->codec->id()) {
+ throw new FloeException(sprintf(
+ 'Floe stream was written with codec 0x%02X, expected 0x%02X',
+ $flags,
+ $this->codec->id(),
+ ));
+ }
+
+ $position = Format::HEADER_LENGTH;
+ $extDecoder = $this->useExtension ? new RowsDecoder() : null;
+ $noopCodec = $this->codec instanceof NoopCodec;
+ $plan = null;
+ $conform = RowPadding::forFileSchema($fileSchema, $this->schemaDecoder);
+ $fileSchemaNames = self::schemaColumnNames($fileSchema);
+ $fileSchemaCount = count($fileSchemaNames);
+ $sectionConforms = false;
+ $batch = [];
+ $yielded = 0;
+ $skip = 0;
+ $pending = [];
+ $stop = false;
+ $flushThreshold = $limit !== null && $limit < $batchSize ? $limit : $batchSize;
+
+ try {
+ while (true) {
+ if (!$fill(Format::FRAME_HEADER_LENGTH)) {
+ if ((strlen($buffer) - $position) === 0) {
+ break;
+ }
+
+ throw new FloeException('Floe stream is truncated, frame header is incomplete');
+ }
+
+ $frameType = ord($buffer[$position]);
+ $frameLength = unpack('V', $buffer, $position + 1)[1];
+ $position += Format::FRAME_HEADER_LENGTH;
+
+ if (!$fill($frameLength)) {
+ throw new FloeException('Floe stream is truncated, frame body is incomplete');
+ }
+
+ $frameEnd = $position + $frameLength;
+
+ if ($frameType === Format::FRAME_ROW) {
+ if ($extDecoder !== null) {
+ $pending[] = $noopCodec
+ ? substr($buffer, $position, $frameLength)
+ : $this->codec->decode(substr($buffer, $position, $frameLength));
+ $position = $frameEnd;
+
+ if (count($pending) === $flushThreshold) {
+ foreach ($this->emitExtBatch(
+ $extDecoder,
+ $pending,
+ $conform,
+ $sectionConforms,
+ $fileSchemaCount,
+ $batch,
+ $batchSize,
+ $partitions,
+ $limit,
+ $yielded,
+ $stop,
+ $skip,
+ ) as $ready) {
+ yield $ready;
+ }
+ $pending = [];
+
+ if ($stop) {
+ return;
+ }
+ }
+ } else {
+ if ($plan === null) {
+ throw new FloeException('Floe found a row frame before any schema frame');
+ }
+
+ if ($noopCodec) {
+ $row = $this->rowHydrator->hydrate($plan, $buffer, $position);
+
+ if ($position !== $frameEnd) {
+ throw new FloeException('Floe row frame length does not match its content');
+ }
+ } else {
+ $rowBody = $this->codec->decode(substr($buffer, $position, $frameLength));
+ $bodyPosition = 0;
+ $row = $this->rowHydrator->hydrate($plan, $rowBody, $bodyPosition);
+
+ if ($bodyPosition !== strlen($rowBody)) {
+ throw new FloeException('Floe row frame length does not match its content');
+ }
+
+ $position = $frameEnd;
+ }
+
+ $batch[] = $sectionConforms && $row->entries()->count() === $fileSchemaCount
+ ? $row
+ : $conform->apply($row);
+
+ if ($limit !== null && ++$yielded >= $limit) {
+ yield $this->batch($batch, $partitions);
+
+ return;
+ }
+
+ if (count($batch) === $batchSize) {
+ yield $this->batch($batch, $partitions);
+ $batch = [];
+ }
+ }
+ } elseif ($frameType === Format::FRAME_SCHEMA) {
+ $frameBody = substr($buffer, $position, $frameLength);
+
+ if ($extDecoder !== null) {
+ if ($pending !== []) {
+ foreach ($this->emitExtBatch(
+ $extDecoder,
+ $pending,
+ $conform,
+ $sectionConforms,
+ $fileSchemaCount,
+ $batch,
+ $batchSize,
+ $partitions,
+ $limit,
+ $yielded,
+ $stop,
+ $skip,
+ ) as $ready) {
+ yield $ready;
+ }
+ $pending = [];
+
+ if ($stop) {
+ return;
+ }
+ }
+
+ $extDecoder->schema($frameBody);
+ } else {
+ $plan = $this->schemaDecoder->decode($frameBody);
+ }
+
+ $sectionConforms = self::sectionNames($frameBody) === $fileSchemaNames;
+ $position = $frameEnd;
+ } elseif ($frameType === Format::FRAME_PARTITIONS || $frameType === Format::FRAME_FOOTER) {
+ $position = $frameEnd;
+ } else {
+ throw new FloeException(sprintf('Floe found unknown frame type 0x%02X', $frameType));
+ }
+
+ if ($position >= FrameReader::COMPACT_THRESHOLD) {
+ $buffer = substr($buffer, $position);
+ $position = 0;
+ }
+ }
+
+ if ($extDecoder !== null && $pending !== []) {
+ foreach ($this->emitExtBatch(
+ $extDecoder,
+ $pending,
+ $conform,
+ $sectionConforms,
+ $fileSchemaCount,
+ $batch,
+ $batchSize,
+ $partitions,
+ $limit,
+ $yielded,
+ $stop,
+ $skip,
+ ) as $ready) {
+ yield $ready;
+ }
+
+ if ($stop) {
+ return;
+ }
+ }
+ } catch (SerializationException|ExtensionException $e) {
+ throw new FloeException($e->getMessage(), 0, $e);
+ }
+
+ if ($batch !== []) {
+ yield $this->batch($batch, $partitions);
+ }
+ }
+
+ /**
+ * The first $count rows, without scanning the rest of the file.
+ *
+ * @param int<1, max> $batchSize
+ *
+ * @throws FloeException
+ *
+ * @return \Generator
+ */
+ public function head(int $count, int $batchSize = 1000): Generator
+ {
+ if ($count < 1) {
+ throw new FloeException('Floe head count must be greater than 0');
+ }
+
+ return $this->rows($batchSize, 0, $count);
+ }
+
+ /**
+ * The last $count rows, decoding only from the boundary section onward
+ * (footer offsets, no scan). A file with fewer rows yields them all.
+ *
+ * @param int<1, max> $batchSize
+ *
+ * @throws FloeException
+ *
+ * @return \Generator
+ */
+ public function tail(int $count, int $batchSize = 1000): Generator
+ {
+ if ($count < 1) {
+ throw new FloeException('Floe tail count must be greater than 0');
+ }
+
+ return $this->rows($batchSize, max(0, $this->totalRows() - $count), null);
+ }
+
+ /**
+ * Applies the same per-row skip / padding / batch-yield / limit logic as the
+ * pure-PHP path; $batch, $yielded, $stop and $skip are updated by reference.
+ *
+ * @param array $pending
+ * @param array $batch
+ * @param array $partitions
+ * @param int<1, max> $batchSize
+ *
+ * @throws ExtensionException
+ *
+ * @return array completed batches ready to yield
+ */
+ private function emitExtBatch(
+ RowsDecoder $extDecoder,
+ array $pending,
+ RowPadding $conform,
+ bool $sectionConforms,
+ int $fileSchemaCount,
+ array &$batch,
+ int $batchSize,
+ array $partitions,
+ ?int $limit,
+ int &$yielded,
+ bool &$stop,
+ int &$skip,
+ ): array {
+ $ready = [];
+
+ foreach ($extDecoder->rows($pending)->all() as $row) {
+ if ($skip > 0) {
+ $skip--;
+
+ continue;
+ }
+
+ $batch[] = $sectionConforms && $row->entries()->count() === $fileSchemaCount ? $row : $conform->apply($row);
+
+ if ($limit !== null && ++$yielded >= $limit) {
+ $ready[] = $this->batch($batch, $partitions);
+ $batch = [];
+ $stop = true;
+
+ return $ready;
+ }
+
+ if (count($batch) === $batchSize) {
+ $ready[] = $this->batch($batch, $partitions);
+ $batch = [];
+ }
+ }
+
+ return $ready;
+ }
+
+ /**
+ * @throws FloeException
+ */
+ public function schema(): Schema
+ {
+ return $this->footer()->fileSchema();
+ }
+
+ /**
+ * @throws FloeException
+ */
+ public function totalRows(): int
+ {
+ return $this->footer()->totalRows;
+ }
+
+ /**
+ * @param array $rows
+ * @param array $partitions
+ */
+ private function batch(array $rows, array $partitions): Rows
+ {
+ return $partitions === [] ? new Rows(...$rows) : Rows::partitioned($rows, $partitions);
+ }
+
+ /**
+ * @return \Generator
+ */
+ private function chunksFrom(SourceStream $source, int $startByte, int $size): Generator
+ {
+ $chunkSize = $this->chunkSize;
+ $readAt = $startByte;
+
+ while ($readAt < $size) {
+ /** @var int<1, max> $length */
+ $length = $chunkSize < ($size - $readAt) ? $chunkSize : $size - $readAt;
+
+ yield $source->read($length, $readAt);
+
+ $readAt += $length;
+ }
+ }
+
+ /**
+ * Offset pushdown: seeks to the start section's byte offset, preloads its
+ * schema (its ROWs may have no inline SCHEMA frame when the schema was
+ * reused) and skips the remaining rows inside it.
+ *
+ * @param int<1, max> $batchSize
+ * @param int<1, max> $offset
+ * @param null|int<1, max> $limit
+ *
+ * @throws FloeException
+ *
+ * @return \Generator
+ */
+ private function rowsFromOffset(int $batchSize, int $offset, ?int $limit): Generator
+ {
+ $footer = $this->footer();
+ $fileSchema = $footer->fileSchema();
+
+ $partitions = [];
+
+ foreach ($footer->partitions as $name => $value) {
+ $partitions[] = new Partition($name, $value);
+ }
+
+ $cumulative = 0;
+ $startSection = null;
+
+ foreach ($footer->sections as $section) {
+ if (($cumulative + $section->rowCount) > $offset) {
+ $startSection = $section;
+
+ break;
+ }
+
+ $cumulative += $section->rowCount;
+ }
+
+ if ($startSection === null) {
+ return;
+ }
+
+ $skip = $offset - $cumulative;
+ $schemaBody = $footer->schemaBody($startSection->schemaId);
+
+ $source = ($this->openSource)();
+ $size = $source->size();
+
+ if ($size === null) {
+ throw new FloeException(sprintf(
+ 'Floe offset read requires a sized stream, "%s" does not report its size',
+ $this->uri,
+ ));
+ }
+
+ $chunks = $this->chunksFrom($source, $startSection->offset, $size);
+
+ $buffer = '';
+ $position = 0;
+
+ $fill = static function (int $bytes) use (&$buffer, &$position, $chunks): bool {
+ while ((strlen($buffer) - $position) < $bytes) {
+ if (!$chunks->valid()) {
+ return false;
+ }
+
+ $buffer .= $chunks->current();
+ $chunks->next();
+ }
+
+ return true;
+ };
+
+ $extDecoder = $this->useExtension ? new RowsDecoder() : null;
+ $noopCodec = $this->codec instanceof NoopCodec;
+
+ if ($extDecoder !== null) {
+ $extDecoder->schema($schemaBody);
+ $plan = null;
+ } else {
+ $plan = $this->schemaDecoder->decode($schemaBody);
+ }
+
+ $conform = RowPadding::forFileSchema($fileSchema, $this->schemaDecoder);
+ $fileSchemaNames = self::schemaColumnNames($fileSchema);
+ $fileSchemaCount = count($fileSchemaNames);
+ $sectionConforms = self::sectionNames($schemaBody) === $fileSchemaNames;
+ $batch = [];
+ $yielded = 0;
+ // skipped rows are still decoded - a corrupt skipped row must throw in strict mode
+ $pending = [];
+ $stop = false;
+ $flushThreshold = $limit !== null && $limit < $batchSize ? $limit : $batchSize;
+
+ try {
+ while (true) {
+ if (!$fill(Format::FRAME_HEADER_LENGTH)) {
+ if ((strlen($buffer) - $position) === 0) {
+ break;
+ }
+
+ throw new FloeException('Floe stream is truncated, frame header is incomplete');
+ }
+
+ $frameType = ord($buffer[$position]);
+ $frameLength = unpack('V', $buffer, $position + 1)[1];
+ $position += Format::FRAME_HEADER_LENGTH;
+
+ if (!$fill($frameLength)) {
+ throw new FloeException('Floe stream is truncated, frame body is incomplete');
+ }
+
+ $frameEnd = $position + $frameLength;
+
+ if ($frameType === Format::FRAME_ROW) {
+ if ($extDecoder !== null) {
+ $pending[] = $noopCodec
+ ? substr($buffer, $position, $frameLength)
+ : $this->codec->decode(substr($buffer, $position, $frameLength));
+ $position = $frameEnd;
+
+ if (count($pending) === $flushThreshold) {
+ foreach ($this->emitExtBatch(
+ $extDecoder,
+ $pending,
+ $conform,
+ $sectionConforms,
+ $fileSchemaCount,
+ $batch,
+ $batchSize,
+ $partitions,
+ $limit,
+ $yielded,
+ $stop,
+ $skip,
+ ) as $ready) {
+ yield $ready;
+ }
+ $pending = [];
+
+ if ($stop) {
+ return;
+ }
+ }
+ } elseif ($plan === null) {
+ throw new FloeException('Floe found a row frame before any schema frame');
+ } else {
+ if ($noopCodec) {
+ $row = $this->rowHydrator->hydrate($plan, $buffer, $position);
+
+ if ($position !== $frameEnd) {
+ throw new FloeException('Floe row frame length does not match its content');
+ }
+ } else {
+ $rowBody = $this->codec->decode(substr($buffer, $position, $frameLength));
+ $bodyPosition = 0;
+ $row = $this->rowHydrator->hydrate($plan, $rowBody, $bodyPosition);
+
+ if ($bodyPosition !== strlen($rowBody)) {
+ throw new FloeException('Floe row frame length does not match its content');
+ }
+
+ $position = $frameEnd;
+ }
+
+ if ($skip > 0) {
+ $skip--;
+ } else {
+ $batch[] = $sectionConforms && $row->entries()->count() === $fileSchemaCount
+ ? $row
+ : $conform->apply($row);
+
+ if ($limit !== null && ++$yielded >= $limit) {
+ yield $this->batch($batch, $partitions);
+
+ return;
+ }
+
+ if (count($batch) === $batchSize) {
+ yield $this->batch($batch, $partitions);
+ $batch = [];
+ }
+ }
+ }
+ } elseif ($frameType === Format::FRAME_SCHEMA) {
+ $frameBody = substr($buffer, $position, $frameLength);
+
+ if ($extDecoder !== null) {
+ if ($pending !== []) {
+ foreach ($this->emitExtBatch(
+ $extDecoder,
+ $pending,
+ $conform,
+ $sectionConforms,
+ $fileSchemaCount,
+ $batch,
+ $batchSize,
+ $partitions,
+ $limit,
+ $yielded,
+ $stop,
+ $skip,
+ ) as $ready) {
+ yield $ready;
+ }
+ $pending = [];
+
+ if ($stop) {
+ return;
+ }
+ }
+
+ $extDecoder->schema($frameBody);
+ } else {
+ $plan = $this->schemaDecoder->decode($frameBody);
+ }
+
+ $sectionConforms = self::sectionNames($frameBody) === $fileSchemaNames;
+ $position = $frameEnd;
+ } elseif ($frameType === Format::FRAME_PARTITIONS || $frameType === Format::FRAME_FOOTER) {
+ $position = $frameEnd;
+ } else {
+ throw new FloeException(sprintf('Floe found unknown frame type 0x%02X', $frameType));
+ }
+
+ if ($position >= FrameReader::COMPACT_THRESHOLD) {
+ $buffer = substr($buffer, $position);
+ $position = 0;
+ }
+ }
+
+ if ($extDecoder !== null && $pending !== []) {
+ foreach ($this->emitExtBatch(
+ $extDecoder,
+ $pending,
+ $conform,
+ $sectionConforms,
+ $fileSchemaCount,
+ $batch,
+ $batchSize,
+ $partitions,
+ $limit,
+ $yielded,
+ $stop,
+ $skip,
+ ) as $ready) {
+ yield $ready;
+ }
+
+ if ($stop) {
+ return;
+ }
+ }
+ } catch (SerializationException|ExtensionException $e) {
+ throw new FloeException($e->getMessage(), 0, $e);
+ }
+
+ if ($batch !== []) {
+ yield $this->batch($batch, $partitions);
+ }
+ }
+
+ /**
+ * @param null|array $plan
+ *
+ * @throws FloeException
+ */
+ private function hydrate(?RowsDecoder $extDecoder, ?array $plan, string $rowBody): Row
+ {
+ if ($extDecoder !== null) {
+ return $extDecoder->row($rowBody);
+ }
+
+ if ($plan === null) {
+ throw new FloeException('Floe found a row frame before any schema frame');
+ }
+
+ $position = 0;
+ $row = $this->rowHydrator->hydrate($plan, $rowBody, $position);
+
+ if ($position !== strlen($rowBody)) {
+ throw new FloeException('Floe row frame length does not match its content');
+ }
+
+ return $row;
+ }
+
+ /**
+ * @return array
+ */
+ private static function decodePartitions(string $body): array
+ {
+ $count = unpack('V', $body)[1];
+ $position = 4;
+ $partitions = [];
+
+ for ($i = 0; $i < $count; $i++) {
+ $nameLength = unpack('V', $body, $position)[1];
+ $name = substr($body, $position + 4, $nameLength);
+ $position += 4 + $nameLength;
+ $valueLength = unpack('V', $body, $position)[1];
+ $partitions[] = new Partition($name, substr($body, $position + 4, $valueLength));
+ $position += 4 + $valueLength;
+ }
+
+ return $partitions;
+ }
+
+ /**
+ * @return array ordered entry names of a SCHEMA frame body
+ */
+ private static function sectionNames(string $schemaBody): array
+ {
+ /** @var array $definitions */
+ $definitions = json_decode($schemaBody, true, 512, JSON_THROW_ON_ERROR);
+ $names = [];
+
+ foreach ($definitions as $definition) {
+ $names[] = $definition['ref'];
+ }
+
+ return $names;
+ }
+
+ /**
+ * @return array ordered column names of a Schema
+ */
+ private static function schemaColumnNames(Schema $schema): array
+ {
+ $names = [];
+
+ foreach (array_values($schema->definitions()) as $definition) {
+ $names[] = $definition->entry()->name();
+ }
+
+ return $names;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/FloeLoader.php b/src/core/etl/src/Flow/Floe/FloeLoader.php
new file mode 100644
index 0000000000..8f0430971b
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/FloeLoader.php
@@ -0,0 +1,76 @@
+
+ */
+ private array $writers = [];
+
+ public function __construct(
+ private readonly Path $path,
+ private readonly ?Metadata $metadata = null,
+ private readonly Codec $codec = new NoopCodec(),
+ ) {}
+
+ public function closure(FlowContext $context): void
+ {
+ foreach ($this->writers as $writer) {
+ $writer->close();
+ }
+
+ $context->streams()->closeStreams($this->path);
+ $this->writers = [];
+ }
+
+ public function destination(): Path
+ {
+ return $this->path;
+ }
+
+ public function load(Rows $rows, FlowContext $context): void
+ {
+ $context->telemetry()->loadingStarted($this, [
+ TelemetryAttributes::ATTR_LOADER_DESTINATION_URI => $this->path->uri(),
+ ]);
+
+ try {
+ $stream = $rows->partitions()->count()
+ ? $context->streams()->writeTo($this->path, $rows->partitions()->toArray())
+ : $context->streams()->writeTo($this->path);
+
+ $uri = $stream->path()->uri();
+
+ if (!array_key_exists($uri, $this->writers)) {
+ $writer = new FloeWriter($context->filesystem($this->path), $this->codec);
+ $writer->createOnStream($stream, $this->metadata);
+ $this->writers[$uri] = $writer;
+ }
+
+ $this->writers[$uri]->write($rows);
+
+ $context->telemetry()->loadingCompleted($this, [TelemetryAttributes::ATTR_LOADING_ROWS => $rows->count()]);
+ } catch (Throwable $e) {
+ $context->telemetry()->loadingFailed($this, $e);
+
+ throw $e;
+ }
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/FloeMerger.php b/src/core/etl/src/Flow/Floe/FloeMerger.php
new file mode 100644
index 0000000000..b5b100c9de
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/FloeMerger.php
@@ -0,0 +1,364 @@
+ $sources
+ * @param ?Metadata $metadata merged over the sources' footer metadata, new keys win
+ *
+ * @throws FloeException
+ * @throws IncompatibleSchemaException
+ */
+ public function merge(array $sources, Path $dest, bool $compact = false, ?Metadata $metadata = null): void
+ {
+ if ($sources === []) {
+ throw new FloeException('Floe merge requires at least one source file');
+ }
+
+ $sources = array_values($sources);
+ $reconciled = $this->reconcile($sources);
+
+ $metadata ??= Metadata::empty();
+
+ if ($compact) {
+ $this->mergeCompact($sources, $reconciled['layouts'], $dest, $metadata);
+
+ return;
+ }
+
+ $this->mergeSplice(
+ $sources,
+ $reconciled['layouts'],
+ $reconciled['merged'],
+ $reconciled['partitions'],
+ $dest,
+ $metadata,
+ );
+ }
+
+ /**
+ * Reads every source's footer/layout and validates the merge is possible:
+ * one shared partition combination and a schema that evolves cleanly.
+ *
+ * @param array $sources
+ *
+ * @throws FloeException
+ * @throws IncompatibleSchemaException
+ *
+ * @return array{layouts: array, merged: null|Schema, partitions: array}
+ */
+ private function reconcile(array $sources): array
+ {
+ $layouts = [];
+ $merged = null;
+ $partitions = null;
+
+ foreach ($sources as $index => $source) {
+ $layout = $this->readLayout($source);
+ $footer = $layout['footer'];
+
+ if ($index === 0) {
+ $partitions = $footer->partitions;
+ } elseif ($footer->partitions !== $partitions) {
+ throw new IncompatibleSchemaException(sprintf(
+ 'Floe merge requires all sources to share one partition combination, "%s" differs',
+ $source->uri(),
+ ));
+ }
+
+ $schema = $footer->fileSchema();
+
+ if ($merged !== null) {
+ $this->assertCompatible($merged, $schema, $source);
+ }
+
+ $merged = $merged === null ? $schema : $merged->merge($schema);
+ $layouts[$index] = $layout;
+ }
+
+ return ['layouts' => $layouts, 'merged' => $merged, 'partitions' => $partitions ?? []];
+ }
+
+ /**
+ * Columns shared with the running merged schema must keep a compatible type;
+ * columns present in only some sources are fine - Schema::merge auto-nullables
+ * them and the reader pads the sections that lack them.
+ *
+ * @throws IncompatibleSchemaException
+ */
+ private function assertCompatible(Schema $merged, Schema $incoming, Path $source): void
+ {
+ foreach ($incoming->definitions() as $definition) {
+ $existing = $merged->findDefinition($definition->entry()->name());
+
+ if ($existing !== null && !$existing->isCompatible($definition)) {
+ throw new IncompatibleSchemaException(sprintf(
+ 'Floe merge cannot reconcile column "%s" of "%s": %s is not compatible with %s',
+ $definition->entry()->name(),
+ $source->uri(),
+ $definition->type()->toString(),
+ $existing->type()->toString(),
+ ));
+ }
+ }
+ }
+
+ /**
+ * @param array $sources
+ * @param array $layouts
+ *
+ * @throws FloeException
+ * @throws IncompatibleSchemaException
+ */
+ private function mergeCompact(array $sources, array $layouts, Path $dest, Metadata $metadata): void
+ {
+ $mergedMetadata = Metadata::empty();
+
+ foreach ($layouts as $layout) {
+ $mergedMetadata = $mergedMetadata->merge($layout['footer']->metadata);
+ }
+
+ $writer = new FloeWriter($this->filesystem, useExtension: $this->useExtension);
+ $writer->create($dest, $mergedMetadata->merge($metadata));
+
+ foreach ($sources as $source) {
+ foreach ((new FloeReader($this->filesystem, useExtension: $this->useExtension))
+ ->read($source)
+ ->rows() as $batch) {
+ $writer->write($batch);
+ }
+ }
+
+ $writer->close();
+ }
+
+ /**
+ * @param array $sources
+ * @param array $layouts
+ * @param array $partitions
+ *
+ * @throws FloeException
+ */
+ private function mergeSplice(
+ array $sources,
+ array $layouts,
+ ?Schema $merged,
+ array $partitions,
+ Path $dest,
+ Metadata $metadata,
+ ): void {
+ /** @var array>> $schemas */
+ $schemas = [];
+ $sections = [];
+ $totalRows = 0;
+ $mergedMetadata = Metadata::empty();
+
+ $stream = $this->filesystem->writeTo($dest);
+ $stream->append(Format::header(0x00));
+ $destPosition = Format::HEADER_LENGTH;
+
+ foreach ($sources as $index => $source) {
+ $layout = $layouts[$index];
+ $footer = $layout['footer'];
+ $copyStart = $index === 0
+ ? Format::HEADER_LENGTH
+ : Format::HEADER_LENGTH + $layout['partitionsFrameLength'];
+ $regionLength = $layout['footerFrameStart'] - $copyStart;
+ $outputStart = $destPosition;
+
+ $schemaIdMap = $this->mergeSchemas($footer->schemas, $schemas);
+
+ if ($regionLength > 0) {
+ $sourceStream = $this->filesystem->readFrom($source);
+ $at = $copyStart;
+ $remaining = $regionLength;
+
+ while ($remaining > 0) {
+ /** @var int<1, max> $length */
+ $length = $remaining < self::COPY_CHUNK_SIZE ? $remaining : self::COPY_CHUNK_SIZE;
+ $stream->append($sourceStream->read($length, $at));
+ $at += $length;
+ $remaining -= $length;
+ $destPosition += $length;
+ }
+
+ $sourceStream->close();
+ }
+
+ foreach ($footer->sections as $section) {
+ $sections[] = new Section(
+ $section->offset - $copyStart + $outputStart,
+ $schemaIdMap[$section->schemaId],
+ $section->rowCount,
+ );
+ }
+
+ $totalRows += $footer->totalRows;
+ $mergedMetadata = $mergedMetadata->merge($footer->metadata);
+ }
+
+ /** @var array> $fileSchema */
+ $fileSchema = $merged?->normalize() ?? [];
+
+ $footerJson = (new Footer(
+ Format::VERSION,
+ FloeWriter::writerVersion(),
+ $schemas,
+ $fileSchema,
+ $sections,
+ $partitions,
+ $totalRows,
+ $mergedMetadata->merge($metadata),
+ ))->toJson();
+
+ $stream->append(Format::frame(Format::FRAME_FOOTER, $footerJson . Format::trailer(strlen($footerJson))));
+ $stream->close();
+ }
+
+ /**
+ * Adds a source's schemas to the combined table (dedup by equality) and
+ * returns its old->new schema id mapping.
+ *
+ * @param array>> $sourceSchemas
+ * @param array>> $schemas combined table, updated in place
+ *
+ * @return array
+ */
+ private function mergeSchemas(array $sourceSchemas, array &$schemas): array
+ {
+ $map = [];
+
+ foreach ($sourceSchemas as $oldId => $decoded) {
+ $newId = null;
+
+ foreach ($schemas as $id => $known) {
+ if ($known == $decoded) {
+ $newId = $id;
+
+ break;
+ }
+ }
+
+ if ($newId === null) {
+ $newId = count($schemas);
+ $schemas[] = $decoded;
+ }
+
+ $map[$oldId] = $newId;
+ }
+
+ return $map;
+ }
+
+ /**
+ * @throws FloeException
+ *
+ * @return array{footer: Footer, footerFrameStart: int, partitionsFrameLength: int}
+ */
+ private function readLayout(Path $source): array
+ {
+ if ($this->filesystem->status($source) === null) {
+ throw new FloeException(sprintf('Floe merge source "%s" does not exist', $source->uri()));
+ }
+
+ $stream = $this->filesystem->readFrom($source);
+ $size = $stream->size();
+
+ if ($size === null) {
+ $stream->close();
+
+ throw new FloeException(sprintf(
+ 'Floe merge requires sized sources, "%s" does not report its size',
+ $source->uri(),
+ ));
+ }
+
+ if ($size < (Format::HEADER_LENGTH + Format::TRAILER_LENGTH)) {
+ $stream->close();
+
+ throw new FloeException(sprintf(
+ 'Floe merge source "%s" is torn, too small to hold a header and a trailer',
+ $source->uri(),
+ ));
+ }
+
+ $flags = Format::validateHeader($stream->read(Format::HEADER_LENGTH, 0));
+
+ if ($flags !== 0x00) {
+ $stream->close();
+
+ throw new FloeException(sprintf(
+ 'Floe merge supports only the no-op codec, source "%s" uses codec 0x%02X',
+ $source->uri(),
+ $flags,
+ ));
+ }
+
+ $footerLength = Format::parseTrailer($stream->read(Format::TRAILER_LENGTH, $size - Format::TRAILER_LENGTH));
+ $footerFrameStart = $size - Format::TRAILER_LENGTH - $footerLength - Format::FRAME_HEADER_LENGTH;
+
+ if ($footerFrameStart < Format::HEADER_LENGTH) {
+ $stream->close();
+
+ throw new FloeException(sprintf(
+ 'Floe merge source "%s" is torn, footer does not fit inside the file',
+ $source->uri(),
+ ));
+ }
+
+ /** @var int<1, max> $footerLength */
+ $footer = Footer::fromJson($stream->read($footerLength, $size - Format::TRAILER_LENGTH - $footerLength));
+ $stream->close();
+
+ return [
+ 'footer' => $footer,
+ 'footerFrameStart' => $footerFrameStart,
+ 'partitionsFrameLength' => $this->partitionsFrameLength($footer->partitions),
+ ];
+ }
+
+ /**
+ * Byte length of the leading PARTITIONS frame (0 when not partitioned),
+ * computed from the footer so no frame walk is needed. Order-independent:
+ * only the summed name/value lengths matter.
+ *
+ * @param array $partitions
+ */
+ private function partitionsFrameLength(array $partitions): int
+ {
+ if ($partitions === []) {
+ return 0;
+ }
+
+ $bodyLength = 4;
+
+ foreach ($partitions as $name => $value) {
+ $bodyLength += 4 + strlen($name) + 4 + strlen($value);
+ }
+
+ return Format::FRAME_HEADER_LENGTH + $bodyLength;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/FloeReader.php b/src/core/etl/src/Flow/Floe/FloeReader.php
new file mode 100644
index 0000000000..7e3961b8d7
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/FloeReader.php
@@ -0,0 +1,39 @@
+codec->id());
+ }
+
+ public function read(Path $path): FloeFile
+ {
+ return new FloeFile(
+ fn(): SourceStream => $this->filesystem->readFrom($path),
+ $path->uri(),
+ $this->codec,
+ $this->chunkSize,
+ $this->useExtension,
+ );
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/FloeSerializer.php b/src/core/etl/src/Flow/Floe/FloeSerializer.php
new file mode 100644
index 0000000000..144f3ac09e
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/FloeSerializer.php
@@ -0,0 +1,62 @@
+ $batchSize
+ */
+ public function __construct(int $batchSize = 1000, ?bool $useExtension = null)
+ {
+ $this->serializer = new FloeValueSerializer($batchSize, $useExtension);
+ }
+
+ public function serialize(object $serializable): string
+ {
+ if (!$serializable instanceof Rows && !$serializable instanceof Row) {
+ throw new SerializationException(sprintf(
+ 'FloeSerializer supports only Rows and Row, got: %s',
+ get_debug_type($serializable),
+ ));
+ }
+
+ return $this->serializer->encode($serializable);
+ }
+
+ public function unserialize(string $serialized, array $classes): object
+ {
+ try {
+ $value = $this->serializer->decode($serialized);
+ } catch (FloeException $e) {
+ throw new SerializationException($e->getMessage(), 0, $e);
+ }
+
+ foreach ($classes as $class) {
+ if (is_a($value, $class)) {
+ return $value;
+ }
+ }
+
+ throw new SerializationException(sprintf(
+ 'FloeSerializer::unserialize must return instance of {%s}, got: %s',
+ implode(', ', $classes),
+ get_debug_type($value),
+ ));
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/FloeValueSerializer.php b/src/core/etl/src/Flow/Floe/FloeValueSerializer.php
new file mode 100644
index 0000000000..4553ca5378
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/FloeValueSerializer.php
@@ -0,0 +1,114 @@
+ $batchSize
+ */
+ public function __construct(
+ private readonly int $batchSize = 1000,
+ private readonly ?bool $useExtension = null,
+ ) {
+ // @mago-ignore analysis:impossible-condition,redundant-comparison
+ if ($this->batchSize < 1) {
+ throw new FloeException('Serializer batch size must be at least 1');
+ }
+ }
+
+ public function decode(string $bytes): Row|Rows
+ {
+ try {
+ $footer = $this->readFooter($bytes);
+
+ $filesystem = memory_filesystem();
+ $path = path('memory://floe-value.floe');
+ $filesystem->writeTo($path)->append($bytes)->close();
+
+ $rows = [];
+
+ foreach ((new FloeReader($filesystem, useExtension: $this->useExtension))
+ ->read($path)
+ ->recover($this->batchSize) as $batch) {
+ foreach ($batch->all() as $row) {
+ $rows[] = $row;
+ }
+ }
+
+ // recover() salvages a readable prefix; a whole-value decode must not
+ // silently return partial data
+ if (count($rows) !== $footer->totalRows) {
+ throw new FloeException(sprintf(
+ 'Floe payload is corrupted, recovered %d of %d rows',
+ count($rows),
+ $footer->totalRows,
+ ));
+ }
+
+ return RowsValueMapper::reconstructFrom($rows, $footer);
+ } catch (ExtensionException $e) {
+ throw new FloeException($e->getMessage(), 0, $e);
+ }
+ }
+
+ public function encode(Row|Rows $value): string
+ {
+ $rows = RowsValueMapper::wrap($value);
+
+ $filesystem = memory_filesystem();
+ $path = path('memory://floe-value.floe');
+
+ try {
+ $writer = new FloeWriter($filesystem, useExtension: $this->useExtension);
+ $writer->create($path, RowsValueMapper::metadataFor($value));
+
+ // an empty Rows still crosses once - chunks() would yield nothing and an
+ // empty-but-partitioned value would lose its PARTITIONS frame (byte parity)
+ foreach ($rows->count() === 0 ? [$rows] : $rows->chunks($this->batchSize) as $chunk) {
+ $writer->write($chunk);
+ }
+
+ $writer->close();
+ } catch (ExtensionException $e) {
+ throw new FloeException($e->getMessage(), 0, $e);
+ }
+
+ return $filesystem->readFrom($path)->content();
+ }
+
+ /**
+ * @throws FloeException
+ */
+ private function readFooter(string $bytes): Footer
+ {
+ $length = strlen($bytes);
+
+ if ($length < (Format::HEADER_LENGTH + Format::TRAILER_LENGTH)) {
+ throw new FloeException('Floe payload is torn, too small to hold a header and a trailer');
+ }
+
+ $footerLength = Format::parseTrailer(substr($bytes, $length - Format::TRAILER_LENGTH));
+ $footerStart = $length - Format::TRAILER_LENGTH - $footerLength;
+
+ if ($footerStart < Format::HEADER_LENGTH) {
+ throw new FloeException('Floe payload is torn, footer does not fit inside the payload');
+ }
+
+ return Footer::fromJson(substr($bytes, $footerStart, $footerLength));
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/FloeWriter.php b/src/core/etl/src/Flow/Floe/FloeWriter.php
new file mode 100644
index 0000000000..c5990479f7
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/FloeWriter.php
@@ -0,0 +1,474 @@
+ null until the first write fixes the file's partitions
+ */
+ private ?array $partitions = null;
+
+ private ?Schema $appendBaseSchema = null;
+
+ private ?int $lastSectionSchemaId = null;
+
+ /**
+ * @var array>>
+ */
+ private array $schemas = [];
+
+ /**
+ * @var array
+ */
+ private array $sections = [];
+
+ private int $sectionOffset = 0;
+
+ private int $sectionRowCount = 0;
+
+ private ?int $sectionSchemaId = null;
+
+ private ?DestinationStream $stream = null;
+
+ private int $totalRows = 0;
+
+ /**
+ * @param null|bool $useExtension null auto-detects flow_php; even when true, a non-Noop codec gets the PHP engine
+ *
+ * @throws FloeException
+ */
+ public function __construct(
+ private readonly Filesystem $filesystem,
+ private readonly Codec $codec = new NoopCodec(),
+ private readonly ?bool $useExtension = null,
+ ) {
+ Format::validateCodecId($this->codec->id());
+ $this->encoder =
+ ($this->useExtension ?? extension_loaded('flow_php')) && $this->codec instanceof NoopCodec
+ ? new ExtRowFrameEncoder()
+ : new PhpRowFrameEncoder($this->codec);
+ }
+
+ /**
+ * Opens an append session on an existing file; a missing or empty file is
+ * created instead. Torn files (invalid trailer) throw.
+ *
+ * @param ?Metadata $metadata merged over the existing footer metadata, new keys win
+ *
+ * @throws FloeException
+ */
+ public function append(Path $path, ?Metadata $metadata = null): void
+ {
+ $this->guardNotOpen();
+
+ if ($this->filesystem->status($path) === null) {
+ $this->create($path, $metadata);
+
+ return;
+ }
+
+ $source = $this->filesystem->readFrom($path);
+ $size = $source->size();
+
+ if ($size === null) {
+ $source->close();
+
+ throw new FloeException(sprintf(
+ 'Floe append requires a sized stream, "%s" does not report its size',
+ $path->uri(),
+ ));
+ }
+
+ if ($size === 0) {
+ $source->close();
+ $this->create($path, $metadata);
+
+ return;
+ }
+
+ if ($size < (Format::HEADER_LENGTH + Format::TRAILER_LENGTH)) {
+ $source->close();
+
+ throw new FloeException(sprintf(
+ 'Floe file "%s" is torn, too small to hold a header and a trailer',
+ $path->uri(),
+ ));
+ }
+
+ $flags = Format::validateHeader($source->read(Format::HEADER_LENGTH, 0));
+
+ if ($flags !== $this->codec->id()) {
+ $source->close();
+
+ throw new FloeException(sprintf(
+ 'Floe file "%s" was written with codec 0x%02X, expected 0x%02X',
+ $path->uri(),
+ $flags,
+ $this->codec->id(),
+ ));
+ }
+
+ $footerLength = Format::parseTrailer($source->read(Format::TRAILER_LENGTH, $size - Format::TRAILER_LENGTH));
+
+ if (($size - Format::TRAILER_LENGTH - $footerLength) < Format::HEADER_LENGTH) {
+ $source->close();
+
+ throw new FloeException(sprintf(
+ 'Floe file "%s" is torn, footer does not fit inside the file',
+ $path->uri(),
+ ));
+ }
+
+ /** @var int<1, max> $footerLength */
+ $footer = Footer::fromJson($source->read($footerLength, $size - Format::TRAILER_LENGTH - $footerLength));
+ $source->close();
+
+ $lastSection = $footer->sections === [] ? null : $footer->sections[count($footer->sections) - 1];
+
+ $this->stream = $this->filesystem->appendTo($path);
+ $this->metadata = $footer->metadata->merge($metadata ?? Metadata::empty());
+ $this->schemas = $footer->schemas;
+ $this->sections = $footer->sections;
+ $this->appendBaseSchema = $footer->fileSchema();
+ $this->lastSectionSchemaId = $lastSection?->schemaId;
+ $this->length = $size;
+ $this->mergedSchema = $footer->fileSchema();
+ $this->partitions = $footer->partitions;
+ $this->totalRows = $footer->totalRows;
+ $this->open = true;
+ }
+
+ /**
+ * @throws FloeException
+ */
+ public function close(): void
+ {
+ $this->guardOpen();
+
+ $this->closeSection();
+
+ /** @var array> $fileSchema */
+ $fileSchema = $this->mergedSchema?->normalize() ?? [];
+
+ $footerJson = (new Footer(
+ Format::VERSION,
+ self::writerVersion(),
+ $this->schemas,
+ $fileSchema,
+ $this->sections,
+ $this->partitions ?? [],
+ $this->totalRows,
+ $this->metadata,
+ ))->toJson();
+
+ $this->buffer .= Format::frame(Format::FRAME_FOOTER, $footerJson . Format::trailer(strlen($footerJson)));
+
+ $this->flush();
+ $this->stream()->close();
+ $this->open = false;
+ }
+
+ /**
+ * Opens a create session, overwriting an existing file.
+ *
+ * @param ?Metadata $metadata stored in the footer
+ *
+ * @throws FloeException
+ */
+ public function create(Path $path, ?Metadata $metadata = null): void
+ {
+ $this->beginCreate($this->filesystem->writeTo($path), $metadata ?? Metadata::empty());
+ }
+
+ /**
+ * Opens a create session over an already-open destination stream (e.g. one
+ * provided by the ETL FilesystemStreams machinery). Produces byte-identical
+ * output to create(); the only difference is who owns the stream.
+ *
+ * @param ?Metadata $metadata stored in the footer
+ *
+ * @throws FloeException
+ */
+ public function createOnStream(DestinationStream $stream, ?Metadata $metadata = null): void
+ {
+ $this->beginCreate($stream, $metadata ?? Metadata::empty());
+ }
+
+ /**
+ * @throws FloeException
+ * @throws IncompatibleSchemaException
+ */
+ public function write(Rows $rows): void
+ {
+ $this->guardOpen();
+
+ $this->fixPartitions($rows);
+
+ foreach ($this->encoder->encode($rows) as $segment) {
+ if ($segment->schemaBody !== null) {
+ $this->startSectionFromBody($segment->schemaBody);
+ }
+
+ $this->buffer .= $segment->frames;
+ $this->length += strlen($segment->frames);
+ $this->sectionRowCount += $segment->rowCount;
+ $this->totalRows += $segment->rowCount;
+
+ if (strlen($this->buffer) >= self::IO_BUFFER_SIZE) {
+ $this->flush();
+ }
+ }
+ }
+
+ public static function writerVersion(): string
+ {
+ return InstalledVersions::isInstalled('flow-php/etl')
+ ? InstalledVersions::getPrettyVersion('flow-php/etl') ?? 'unknown'
+ : 'unknown';
+ }
+
+ public static function growSectionPlan(?string $currentSchemaBody, Row $row): EncoderPlan
+ {
+ $rowSchema = self::rowSchema($row);
+
+ if ($currentSchemaBody === null) {
+ $schema = $rowSchema;
+ } else {
+ /** @var array> $decoded */
+ $decoded = json_decode($currentSchemaBody, true, 512, JSON_THROW_ON_ERROR);
+ $schema = self::growUnion(Schema::fromArray($decoded), $rowSchema);
+ }
+
+ return (new SchemaEncoder(new ValueEncoder()))->encodeSchema($schema);
+ }
+
+ /**
+ * @throws FloeException
+ */
+ private function beginCreate(DestinationStream $stream, Metadata $metadata): void
+ {
+ $this->guardNotOpen();
+
+ $this->stream = $stream;
+ $this->metadata = $metadata;
+ $this->length = Format::HEADER_LENGTH;
+ $this->buffer = Format::header($this->codec->id());
+ $this->open = true;
+ }
+
+ private function closeSection(): void
+ {
+ if ($this->sectionSchemaId !== null) {
+ $this->sections[] = new Section($this->sectionOffset, $this->sectionSchemaId, $this->sectionRowCount);
+ $this->lastSectionSchemaId = $this->sectionSchemaId;
+ $this->sectionSchemaId = null;
+ $this->sectionRowCount = 0;
+ }
+ }
+
+ /**
+ * @throws FloeException
+ */
+ private function fixPartitions(Rows $rows): void
+ {
+ $incoming = [];
+
+ foreach ($rows->partitions() as $partition) {
+ $incoming[$partition->name] = $partition->value;
+ }
+
+ ksort($incoming);
+
+ if ($this->partitions === null) {
+ $this->partitions = $incoming;
+
+ if ($incoming !== []) {
+ $body = pack('V', count($incoming));
+
+ foreach ($rows->partitions() as $partition) {
+ $body .=
+ pack('V', strlen($partition->name))
+ . $partition->name
+ . pack('V', strlen($partition->value))
+ . $partition->value;
+ }
+
+ $this->buffer .= Format::frame(Format::FRAME_PARTITIONS, $body);
+ $this->length += Format::FRAME_HEADER_LENGTH + strlen($body);
+ }
+
+ return;
+ }
+
+ if ($incoming !== $this->partitions) {
+ throw new FloeException(sprintf(
+ 'Floe file holds one partition combination, got rows partitioned by "%s" into a file partitioned by "%s"',
+ json_encode($incoming, JSON_THROW_ON_ERROR),
+ json_encode($this->partitions, JSON_THROW_ON_ERROR),
+ ));
+ }
+ }
+
+ private function flush(): void
+ {
+ if ($this->buffer !== '') {
+ $this->stream()->append($this->buffer);
+ $this->buffer = '';
+ }
+ }
+
+ /**
+ * @throws FloeException
+ */
+ private function guardNotOpen(): void
+ {
+ if ($this->stream !== null) {
+ throw new FloeException('Floe writer session is already open');
+ }
+ }
+
+ /**
+ * @throws FloeException
+ */
+ private function guardOpen(): void
+ {
+ if (!$this->open) {
+ throw new FloeException('Floe writer session is not open');
+ }
+ }
+
+ /**
+ * Bookkeeping half of a section change - the engine already grew the union into $schemaBody.
+ *
+ * @throws FloeException
+ * @throws IncompatibleSchemaException
+ */
+ private function startSectionFromBody(string $schemaBody): void
+ {
+ /** @var array> $decoded */
+ $decoded = json_decode($schemaBody, true, 512, JSON_THROW_ON_ERROR);
+ $schema = Schema::fromArray($decoded);
+
+ if ($this->appendBaseSchema !== null) {
+ (new SchemaEvolution())->validate($this->appendBaseSchema, $schema);
+ }
+
+ $this->closeSection();
+
+ $schemaId = null;
+
+ foreach ($this->schemas as $id => $known) {
+ if ($known == $decoded) {
+ $schemaId = $id;
+
+ break;
+ }
+ }
+
+ if ($schemaId === null) {
+ $schemaId = count($this->schemas);
+ $this->schemas[] = $decoded;
+ }
+
+ $this->sectionOffset = $this->length;
+
+ if ($schemaId !== $this->lastSectionSchemaId) {
+ $this->buffer .= Format::frame(Format::FRAME_SCHEMA, $schemaBody);
+ $this->length += Format::FRAME_HEADER_LENGTH + strlen($schemaBody);
+ }
+
+ $this->mergedSchema = $this->mergedSchema === null ? $schema : $this->mergedSchema->merge($schema);
+ $this->sectionSchemaId = $schemaId;
+ $this->sectionRowCount = 0;
+ }
+
+ /**
+ * The row's schema built from its entry definitions, without touching the
+ * row's lazily-cached schema() (the writer must not mutate caller rows).
+ */
+ private static function rowSchema(Row $row): Schema
+ {
+ $definitions = [];
+
+ foreach ($row->entries()->all() as $entry) {
+ $definitions[] = $entry->definition();
+ }
+
+ return new Schema(...$definitions);
+ }
+
+ /**
+ * Unlike Schema::merge, never nullables a column absent from one side - in
+ * Floe absence is carried by the row's absent marker, not by a null.
+ */
+ private static function growUnion(Schema $current, Schema $incoming): Schema
+ {
+ $definitions = [];
+
+ foreach (array_values($current->definitions()) as $definition) {
+ $definitions[$definition->entry()->name()] = $definition;
+ }
+
+ foreach (array_values($incoming->definitions()) as $definition) {
+ $name = $definition->entry()->name();
+ $definitions[$name] = array_key_exists($name, $definitions)
+ ? $definitions[$name]->merge($definition)
+ : $definition;
+ }
+
+ return new Schema(...array_values($definitions));
+ }
+
+ /**
+ * @throws FloeException
+ */
+ private function stream(): DestinationStream
+ {
+ return $this->stream ?? throw new FloeException('Floe writer session is not open');
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Footer.php b/src/core/etl/src/Flow/Floe/Footer.php
new file mode 100644
index 0000000000..e0db4e2a3b
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Footer.php
@@ -0,0 +1,144 @@
+>> $schemas decoded SCHEMA frame bodies, index = schemaId
+ * @param array> $fileSchema normalized merged schema
+ * @param array $sections
+ * @param array $partitions
+ */
+ public function __construct(
+ public int $version,
+ public string $writer,
+ public array $schemas,
+ public array $fileSchema,
+ public array $sections,
+ public array $partitions,
+ public int $totalRows,
+ public Metadata $metadata,
+ ) {}
+
+ /**
+ * @throws FloeException
+ */
+ public static function fromJson(string $json): self
+ {
+ try {
+ // @mago-ignore analysis:mixed-assignment
+ $data = json_decode($json, true, 512, JSON_THROW_ON_ERROR);
+ } catch (JsonException $e) {
+ throw new FloeException('Floe failed to decode footer JSON: ' . $e->getMessage(), 0, $e);
+ }
+
+ try {
+ $data = type_structure([
+ 'version' => type_integer(),
+ 'writer' => type_string(),
+ 'schemas' => type_array(),
+ 'fileSchema' => type_array(),
+ 'sections' => type_list(type_array()),
+ 'partitions' => type_array(),
+ 'totalRows' => type_integer(),
+ 'metadata' => type_array(),
+ ])->assert($data);
+ } catch (InvalidTypeException $e) {
+ throw new FloeException('Floe footer is malformed: ' . $e->getMessage(), 0, $e);
+ }
+
+ $sections = [];
+
+ /** @var array> $sectionsData */
+ $sectionsData = $data['sections'];
+
+ foreach ($sectionsData as $section) {
+ $sections[] = Section::fromArray($section);
+ }
+
+ /** @var array>> $schemas */
+ $schemas = $data['schemas'];
+ /** @var array> $fileSchema */
+ $fileSchema = $data['fileSchema'];
+ /** @var array $partitions */
+ $partitions = $data['partitions'];
+
+ try {
+ /** @var array|bool|float|int|string> $rawMetadata */
+ $rawMetadata = $data['metadata'];
+ $metadata = Metadata::fromArray($rawMetadata);
+ } catch (InvalidArgumentException $e) {
+ throw new FloeException('Floe footer metadata is malformed: ' . $e->getMessage(), 0, $e);
+ }
+
+ return new self(
+ $data['version'],
+ $data['writer'],
+ $schemas,
+ $fileSchema,
+ $sections,
+ $partitions,
+ $data['totalRows'],
+ $metadata,
+ );
+ }
+
+ public function fileSchema(): Schema
+ {
+ return Schema::fromArray($this->fileSchema);
+ }
+
+ /**
+ * @throws FloeException
+ */
+ public function schemaBody(int $schemaId): string
+ {
+ if (!array_key_exists($schemaId, $this->schemas)) {
+ throw new FloeException(sprintf('Floe footer does not hold schema with id %d', $schemaId));
+ }
+
+ return json_encode($this->schemas[$schemaId], JSON_THROW_ON_ERROR);
+ }
+
+ public function toJson(): string
+ {
+ $sections = [];
+
+ foreach ($this->sections as $section) {
+ $sections[] = $section->normalize();
+ }
+
+ return json_encode([
+ 'version' => $this->version,
+ 'writer' => $this->writer,
+ 'schemas' => $this->schemas,
+ 'fileSchema' => $this->fileSchema,
+ 'sections' => $sections,
+ 'partitions' => (object) $this->partitions,
+ 'totalRows' => $this->totalRows,
+ 'metadata' => (object) $this->metadata->normalize(),
+ ], JSON_THROW_ON_ERROR);
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Format.php b/src/core/etl/src/Flow/Floe/Format.php
new file mode 100644
index 0000000000..8d62f9fac0
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Format.php
@@ -0,0 +1,151 @@
+ frame type and frame body
+ */
+ public function frames(bool $lenient = false): Generator
+ {
+ /** @var int<1, max> $chunkSize */
+ $chunkSize = $this->chunkSize;
+ $chunks = $this->stream->iterate($chunkSize);
+ $buffer = '';
+ $position = 0;
+
+ $fill = static function (int $bytes) use (&$buffer, &$position, $chunks): bool {
+ while ((strlen($buffer) - $position) < $bytes) {
+ if (!$chunks->valid()) {
+ return false;
+ }
+
+ $buffer .= $chunks->current();
+ $chunks->next();
+ }
+
+ return true;
+ };
+
+ if (!$fill(Format::HEADER_LENGTH)) {
+ if (strlen($buffer) === 0 && $lenient) {
+ return;
+ }
+
+ throw new FloeException('Floe stream is truncated, header is incomplete');
+ }
+
+ $flags = Format::validateHeader(substr($buffer, 0, Format::HEADER_LENGTH));
+
+ if ($flags !== $this->expectedFlags) {
+ throw new FloeException(sprintf(
+ 'Floe stream was written with codec 0x%02X, expected 0x%02X',
+ $flags,
+ $this->expectedFlags,
+ ));
+ }
+
+ $position = Format::HEADER_LENGTH;
+
+ while (true) {
+ if (!$fill(Format::FRAME_HEADER_LENGTH)) {
+ if ((strlen($buffer) - $position) === 0 || $lenient) {
+ return;
+ }
+
+ throw new FloeException('Floe stream is truncated, frame header is incomplete');
+ }
+
+ $frameType = ord($buffer[$position]);
+ $frameLength = unpack('V', $buffer, $position + 1)[1];
+
+ if (!$fill(Format::FRAME_HEADER_LENGTH + $frameLength)) {
+ if ($lenient) {
+ return;
+ }
+
+ throw new FloeException('Floe stream is truncated, frame body is incomplete');
+ }
+
+ yield [$frameType, substr($buffer, $position + Format::FRAME_HEADER_LENGTH, $frameLength)];
+
+ $position += Format::FRAME_HEADER_LENGTH + $frameLength;
+
+ if ($position >= self::COMPACT_THRESHOLD) {
+ $buffer = substr($buffer, $position);
+ $position = 0;
+ }
+ }
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/FrameSegment.php b/src/core/etl/src/Flow/Floe/FrameSegment.php
new file mode 100644
index 0000000000..870e1cac0f
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/FrameSegment.php
@@ -0,0 +1,14 @@
+ $definition
+ * @param Definition $nullableDefinition
+ * @param Definition $fromNullDefinition
+ */
+ public function __construct(
+ public string $name,
+ public Definition $definition,
+ public Definition $nullableDefinition,
+ public Definition $fromNullDefinition,
+ public Decoding\ValueDecoder $decoder,
+ public EntryFactory $entryFactory,
+ ) {}
+}
diff --git a/src/core/etl/src/Flow/Floe/PhpRowFrameEncoder.php b/src/core/etl/src/Flow/Floe/PhpRowFrameEncoder.php
new file mode 100644
index 0000000000..8dc86d2416
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/PhpRowFrameEncoder.php
@@ -0,0 +1,55 @@
+all() as $row) {
+ if ($this->plan === null || !$this->schemaTracker->fits($this->plan, $row)) {
+ if ($rowCount > 0 || $schemaBody !== null) {
+ $segments[] = new FrameSegment($schemaBody, $frames, $rowCount);
+ }
+
+ $this->plan = FloeWriter::growSectionPlan($this->plan?->schemaBody, $row);
+ $schemaBody = $this->plan->schemaBody;
+ $frames = '';
+ $rowCount = 0;
+ }
+
+ $body = $this->codec->encode($this->rowEncoder->encode($this->plan, $row));
+
+ $frames .= chr(Format::FRAME_ROW) . pack('V', strlen($body)) . $body;
+ $rowCount++;
+ }
+
+ if ($rowCount > 0) {
+ $segments[] = new FrameSegment($schemaBody, $frames, $rowCount);
+ }
+
+ return $segments;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/RowEncoder.php b/src/core/etl/src/Flow/Floe/RowEncoder.php
new file mode 100644
index 0000000000..54a757a182
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/RowEncoder.php
@@ -0,0 +1,79 @@
+entries()->all();
+
+ // fast path: the row's columns align positionally with the plan (every row
+ // of a homogeneous batch), so no per-row name-keyed map is needed
+ if (count($entries) === count($plan->columns)) {
+ $body = '';
+ $index = 0;
+
+ foreach ($plan->columns as $column) {
+ $entry = $entries[$index++];
+
+ if ($entry->name() !== $column->name) {
+ return $this->encodeByName($plan, $entries);
+ }
+
+ $body .= $this->entryBytes($column, $entry);
+ }
+
+ return $body;
+ }
+
+ return $this->encodeByName($plan, $entries);
+ }
+
+ /**
+ * @param array> $entries
+ */
+ private function encodeByName(EncoderPlan $plan, array $entries): string
+ {
+ $keyed = [];
+
+ foreach ($entries as $entry) {
+ $keyed[$entry->name()] = $entry;
+ }
+
+ $body = '';
+
+ foreach ($plan->columns as $column) {
+ $entry = $keyed[$column->name] ?? null;
+
+ $body .= $entry === null ? Format::VALUE_ABSENT_BYTE : $this->entryBytes($column, $entry);
+ }
+
+ return $body;
+ }
+
+ /**
+ * @param Entry $entry
+ */
+ private function entryBytes(EncoderColumn $column, Entry $entry): string
+ {
+ // @mago-ignore analysis:mixed-assignment
+ $value = $entry->value();
+
+ if ($value === null) {
+ return $entry->definition()->metadata()->has(Metadata::FROM_NULL)
+ ? Format::VALUE_NULL_FROM_NULL_BYTE
+ : Format::VALUE_NULL_BYTE;
+ }
+
+ return Format::VALUE_PRESENT_BYTE . $column->encoder->encode($value);
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/RowFrameEncoder.php b/src/core/etl/src/Flow/Floe/RowFrameEncoder.php
new file mode 100644
index 0000000000..bd946185ac
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/RowFrameEncoder.php
@@ -0,0 +1,18 @@
+
+ */
+ public function encode(Rows $rows): array;
+}
diff --git a/src/core/etl/src/Flow/Floe/RowHydrator.php b/src/core/etl/src/Flow/Floe/RowHydrator.php
new file mode 100644
index 0000000000..d5b3a7795e
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/RowHydrator.php
@@ -0,0 +1,50 @@
+ $plan
+ */
+ public function hydrate(array $plan, string $data, int &$position): Row
+ {
+ $entries = [];
+
+ foreach ($plan as $column) {
+ $flag = ord($data[$position++]);
+
+ if ($flag === Format::VALUE_ABSENT) {
+ // The row had no such column; reconstruct it as written (omitted).
+ continue;
+ }
+
+ if ($flag === Format::VALUE_PRESENT) {
+ // @mago-ignore analysis:mixed-assignment
+ $value = $column->decoder->decode($data, $position);
+ $definition = clone $column->definition;
+ } elseif ($flag === Format::VALUE_NULL) {
+ $value = null;
+ $definition = clone $column->nullableDefinition;
+ } elseif ($flag === Format::VALUE_NULL_FROM_NULL) {
+ $value = null;
+ $definition = clone $column->fromNullDefinition;
+ } else {
+ throw new SerializationException(sprintf('Floe found unknown value flag 0x%02X', $flag));
+ }
+
+ $entries[$column->name] = $column->entryFactory->create($column->name, $value, $definition);
+ }
+
+ return new Row(Entries::recreate($entries));
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/RowPadding.php b/src/core/etl/src/Flow/Floe/RowPadding.php
new file mode 100644
index 0000000000..4483c93a74
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/RowPadding.php
@@ -0,0 +1,59 @@
+ $order file-schema column names, in order
+ * @param array> $nulls shared null entry per column, used for absent columns
+ */
+ private function __construct(
+ private readonly array $order,
+ private readonly array $nulls,
+ ) {}
+
+ public static function forFileSchema(Schema $fileSchema, SchemaDecoder $decoder): self
+ {
+ $order = [];
+ $nulls = [];
+
+ foreach (array_values($fileSchema->definitions()) as $definition) {
+ $name = $definition->entry()->name();
+ $order[] = $name;
+ $column = $decoder->decode(json_encode([$definition->normalize()], JSON_THROW_ON_ERROR))[0];
+ $nulls[$name] = $column->entryFactory->create($name, null, clone $column->fromNullDefinition);
+ }
+
+ return new self($order, $nulls);
+ }
+
+ public function apply(Row $row): Row
+ {
+ $byName = [];
+
+ foreach ($row->entries()->all() as $entry) {
+ $byName[$entry->name()] = $entry;
+ }
+
+ $entries = [];
+
+ foreach ($this->order as $name) {
+ $entries[$name] = $byName[$name] ?? $this->nulls[$name];
+ }
+
+ return new Row(Entries::recreate($entries));
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/RowsValueMapper.php b/src/core/etl/src/Flow/Floe/RowsValueMapper.php
new file mode 100644
index 0000000000..9898394c83
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/RowsValueMapper.php
@@ -0,0 +1,105 @@
+partitions()->count() > 0) {
+ $metadata = $metadata->add(self::PARTITION_ORDER_KEY, array_map(
+ static fn(Partition $p): string => $p->name,
+ $value->partitions()->toArray(),
+ ));
+ }
+
+ return $metadata;
+ }
+
+ /**
+ * The value's partitions in the caller's original order (footer metadata) when recorded,
+ * in footer order otherwise.
+ *
+ * @throws FloeException
+ *
+ * @return array
+ */
+ public static function partitionsFrom(Footer $footer): array
+ {
+ $partitions = [];
+
+ if ($footer->metadata->has(self::PARTITION_ORDER_KEY)) {
+ try {
+ /** @var list $order */
+ $order = $footer->metadata->getAs(self::PARTITION_ORDER_KEY, type_list(type_string()), []);
+ } catch (CastingException $e) {
+ throw new FloeException('Floe cache partition order metadata is malformed: ' . $e->getMessage(), 0, $e);
+ }
+
+ foreach ($order as $name) {
+ $partitions[] = new Partition($name, $footer->partitions[$name]);
+ }
+ } else {
+ foreach ($footer->partitions as $name => $value) {
+ $partitions[] = new Partition($name, $value);
+ }
+ }
+
+ return $partitions;
+ }
+
+ /**
+ * Rebuilds the original Row|Rows from already-decoded (un-partitioned) rows and the file footer:
+ * reattaches partitions in the caller's original order (footer metadata) and unwraps a tagged
+ * single Row.
+ *
+ * @param array $rows
+ *
+ * @throws FloeException
+ */
+ public static function reconstructFrom(array $rows, Footer $footer): Row|Rows
+ {
+ $partitions = self::partitionsFrom($footer);
+
+ $valueType = $footer->metadata->has(self::VALUE_TYPE_KEY)
+ ? $footer->metadata->get(self::VALUE_TYPE_KEY)
+ : self::VALUE_TYPE_ROWS;
+
+ if ($valueType === self::VALUE_TYPE_ROW) {
+ return $rows[0] ?? throw new FloeException('Floe value tagged as a single Row contained no rows');
+ }
+
+ return $partitions === [] ? new Rows(...$rows) : Rows::partitioned($rows, $partitions);
+ }
+
+ public static function wrap(Row|Rows $value): Rows
+ {
+ return $value instanceof Row ? new Rows($value) : $value;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/SchemaDecoder.php b/src/core/etl/src/Flow/Floe/SchemaDecoder.php
new file mode 100644
index 0000000000..beaa4be228
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/SchemaDecoder.php
@@ -0,0 +1,141 @@
+ BooleanEntry::class,
+ DateDefinition::class => DateEntry::class,
+ DateTimeDefinition::class => DateTimeEntry::class,
+ EnumDefinition::class => EnumEntry::class,
+ FloatDefinition::class => FloatEntry::class,
+ HTMLDefinition::class => HTMLEntry::class,
+ HTMLElementDefinition::class => HTMLElementEntry::class,
+ IntegerDefinition::class => IntegerEntry::class,
+ JsonDefinition::class => JsonEntry::class,
+ ListDefinition::class => ListEntry::class,
+ MapDefinition::class => MapEntry::class,
+ StringDefinition::class => StringEntry::class,
+ StructureDefinition::class => StructureEntry::class,
+ TimeDefinition::class => TimeEntry::class,
+ UuidDefinition::class => UuidEntry::class,
+ XMLDefinition::class => XMLEntry::class,
+ XMLElementDefinition::class => XMLElementEntry::class,
+ ];
+
+ public function __construct(
+ private readonly ValueDecoder $valueDecoder,
+ private readonly EntryInstantiator $entryInstantiator,
+ ) {}
+
+ /**
+ * @return array
+ */
+ public function decode(string $schemaJson): array
+ {
+ try {
+ // @mago-ignore analysis:mixed-assignment
+ $definitions = json_decode($schemaJson, true, 512, JSON_THROW_ON_ERROR);
+ } catch (JsonException $e) {
+ throw new FloeException('Floe failed to decode schema JSON: ' . $e->getMessage(), 0, $e);
+ }
+
+ try {
+ $definitions = type_list(type_array())->assert($definitions);
+ } catch (InvalidTypeException $e) {
+ throw new FloeException(
+ 'Floe expected schema JSON to be a list of definitions: ' . $e->getMessage(),
+ 0,
+ $e,
+ );
+ }
+
+ $plan = [];
+
+ /** @var array{ref: string, type: array, nullable?: bool, metadata?: array|bool|float|int|string>} $normalized */
+ foreach ($definitions as $normalized) {
+ $metadata = $normalized['metadata'] ?? [];
+ unset($metadata[Metadata::FROM_NULL]);
+
+ $definition = definition_from_array([
+ 'ref' => $normalized['ref'],
+ 'type' => $normalized['type'],
+ 'nullable' => false,
+ 'metadata' => $metadata,
+ ]);
+
+ $entryClass = self::ENTRY_CLASSES[$definition::class] ?? throw new FloeException(sprintf(
+ 'Floe cannot hydrate entries for definition "%s"',
+ $definition::class,
+ ));
+
+ $fromNullDefinition = $definition->makeNullable();
+ $fromNullDefinition->setMetadata(
+ $fromNullDefinition->metadata()->merge(Metadata::fromArray([Metadata::FROM_NULL => true])),
+ );
+
+ /** @var class-string> $entryClass */
+ $plan[] = new HydratorColumn(
+ $normalized['ref'],
+ $definition,
+ $definition->makeNullable(),
+ $fromNullDefinition,
+ $this->valueDecoder->decoderFor($definition->type()),
+ $this->entryInstantiator->factoryFor($entryClass),
+ );
+ }
+
+ return $plan;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/SchemaEncoder.php b/src/core/etl/src/Flow/Floe/SchemaEncoder.php
new file mode 100644
index 0000000000..20a890732d
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/SchemaEncoder.php
@@ -0,0 +1,57 @@
+definitions()) as $definition) {
+ $name = $definition->entry()->name();
+ $columns[$name] = $this->column($name, $definition->type());
+ $definitions[] = $definition->normalize();
+ }
+
+ return new EncoderPlan($columns, json_encode($definitions, JSON_THROW_ON_ERROR));
+ } catch (JsonException $e) {
+ throw new FloeException('Floe failed to encode schema as JSON: ' . $e->getMessage(), 0, $e);
+ }
+ }
+
+ /**
+ * @param Type $type
+ *
+ * @throws JsonException
+ */
+ private function column(string $name, Type $type): EncoderColumn
+ {
+ return new EncoderColumn(
+ $name,
+ json_encode($type->normalize(), JSON_THROW_ON_ERROR),
+ $this->valueEncoder->encoderFor($type),
+ );
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/SchemaEvolution.php b/src/core/etl/src/Flow/Floe/SchemaEvolution.php
new file mode 100644
index 0000000000..9dd1add295
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/SchemaEvolution.php
@@ -0,0 +1,60 @@
+definitions() as $incomingDefinition) {
+ $name = $incomingDefinition->entry()->name();
+ $existing = $fileSchema->findDefinition($name);
+
+ if ($existing === null) {
+ if (!$incomingDefinition->isNullable()) {
+ throw new IncompatibleSchemaException(sprintf(
+ 'Floe append adds new column "%s" which must be nullable - rows already in the file hold null for it',
+ $name,
+ ));
+ }
+
+ continue;
+ }
+
+ if (!$existing->isCompatible($incomingDefinition)) {
+ throw new IncompatibleSchemaException(sprintf(
+ 'Floe append changes column "%s" from %s to %s which is not compatible',
+ $name,
+ $existing->type()->toString(),
+ $incomingDefinition->type()->toString(),
+ ));
+ }
+ }
+
+ foreach ($fileSchema->definitions() as $existingDefinition) {
+ $name = $existingDefinition->entry()->name();
+
+ if ($incoming->findDefinition($name) === null && !$existingDefinition->isNullable()) {
+ throw new IncompatibleSchemaException(sprintf(
+ 'Floe append omits column "%s" which is not nullable in the file schema',
+ $name,
+ ));
+ }
+ }
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/SchemaTracker.php b/src/core/etl/src/Flow/Floe/SchemaTracker.php
new file mode 100644
index 0000000000..2ab6c05745
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/SchemaTracker.php
@@ -0,0 +1,84 @@
+ true,
+ PositiveIntegerType::class => true,
+ FloatType::class => true,
+ BooleanType::class => true,
+ StringType::class => true,
+ NonEmptyStringType::class => true,
+ NumericStringType::class => true,
+ ScalarType::class => true,
+ DateTimeType::class => true,
+ DateType::class => true,
+ TimeType::class => true,
+ HTMLType::class => true,
+ HTMLElementType::class => true,
+ TimeZoneType::class => true,
+ UuidType::class => true,
+ JsonType::class => true,
+ XMLType::class => true,
+ XMLElementType::class => true,
+ ];
+
+ /**
+ * @var array
+ */
+ private array $fingerprints = [];
+
+ public function fits(EncoderPlan $plan, Row $row): bool
+ {
+ foreach ($row->entries()->all() as $entry) {
+ $name = $entry->name();
+
+ if (!array_key_exists($name, $plan->columns)) {
+ return false;
+ }
+
+ $type = $entry->definition()->type();
+
+ $fingerprint = array_key_exists($type::class, self::CONSTANT_NORMALIZE_TYPES)
+ ? ($this->fingerprints[$type::class] ??= json_encode($type->normalize(), JSON_THROW_ON_ERROR))
+ : json_encode($type->normalize(), JSON_THROW_ON_ERROR);
+
+ if ($fingerprint !== $plan->columns[$name]->typeFingerprint) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/Section.php b/src/core/etl/src/Flow/Floe/Section.php
new file mode 100644
index 0000000000..88ef9e60df
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/Section.php
@@ -0,0 +1,53 @@
+ $data
+ *
+ * @throws FloeException
+ */
+ public static function fromArray(array $data): self
+ {
+ try {
+ $data = type_structure([
+ 'offset' => type_integer(),
+ 'schemaId' => type_integer(),
+ 'rowCount' => type_integer(),
+ ])->assert($data);
+ } catch (InvalidTypeException $e) {
+ throw new FloeException('Floe footer section is malformed: ' . $e->getMessage(), 0, $e);
+ }
+
+ return new self($data['offset'], $data['schemaId'], $data['rowCount']);
+ }
+
+ /**
+ * @return array{offset: int, schemaId: int, rowCount: int}
+ */
+ public function normalize(): array
+ {
+ return ['offset' => $this->offset, 'schemaId' => $this->schemaId, 'rowCount' => $this->rowCount];
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/ValueDecoder.php b/src/core/etl/src/Flow/Floe/ValueDecoder.php
new file mode 100644
index 0000000000..f00dd899aa
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/ValueDecoder.php
@@ -0,0 +1,247 @@
+timeZones = new TimeZones();
+ $this->dateTimeDecoder = new DateTimeDecoder($this->timeZones);
+ $this->uuidDecoder = new UuidDecoder();
+ $this->jsonDecoder = new JsonDecoder();
+ }
+
+ /**
+ * @param Type $type
+ *
+ * @throws FloeException
+ */
+ public function decoderFor(Type $type): Decoding\ValueDecoder
+ {
+ return match ($type::class) {
+ IntegerType::class, PositiveIntegerType::class => new Int64Decoder(),
+ FloatType::class => new Float64Decoder(),
+ BooleanType::class => new BooleanDecoder(),
+ StringType::class,
+ NonEmptyStringType::class,
+ NumericStringType::class,
+ ClassStringType::class,
+ => new StringDecoder(),
+ TimeZoneType::class => new TimeZoneDecoder($this->timeZones),
+ DateTimeType::class, DateType::class => $this->dateTimeDecoder,
+ TimeType::class => new IntervalDecoder(),
+ UuidType::class => $this->uuidDecoder,
+ JsonType::class => $this->jsonDecoder,
+ EnumType::class => new EnumDecoder(),
+ XMLType::class => new XmlDocumentDecoder(),
+ XMLElementType::class => new XmlElementDecoder(),
+ HTMLType::class => new HtmlDocumentDecoder(),
+ HTMLElementType::class => new HtmlElementDecoder(),
+ ListType::class => $this->listDecoder($type),
+ MapType::class => $this->mapDecoder($type),
+ StructureType::class => $this->structureDecoder($type),
+ OptionalType::class => $this->optionalDecoder($type),
+ NullType::class => new NullDecoder(),
+ MixedType::class,
+ UnionType::class,
+ ScalarType::class,
+ LiteralType::class,
+ ArrayType::class,
+ => $this->dynamicDecoder(),
+ default => throw new FloeException(sprintf('Floe does not support values of type "%s"', $type->toString())),
+ };
+ }
+
+ public static function htmlDocumentFromString(string $html): object
+ {
+ if (!class_exists('\Dom\HTMLDocument')) {
+ throw new FloeException('Floe cannot restore HTML values, \Dom\HTMLDocument requires PHP 8.4+');
+ }
+
+ // @mago-expect analysis:unavailable-method
+ return HTMLDocument::createFromString($html, LIBXML_NOERROR);
+ }
+
+ public static function htmlElementFromString(string $html): object
+ {
+ if (!class_exists('\Dom\HTMLDocument')) {
+ throw new FloeException('Floe cannot restore HTML values, \Dom\HTMLDocument requires PHP 8.4+');
+ }
+
+ // @mago-expect analysis:unavailable-method
+ $element = HTMLDocument::createFromString(
+ '' . $html . '',
+ LIBXML_NOERROR,
+ )->body?->firstElementChild;
+
+ if ($element === null) {
+ throw new FloeException(sprintf('Floe failed to restore HTMLElement from "%s"', $html));
+ }
+
+ return $element;
+ }
+
+ public static function xmlDocumentFromString(string $xml): DOMDocument
+ {
+ $document = new DOMDocument();
+
+ if (!@$document->loadXML($xml)) {
+ throw new FloeException(sprintf('Floe failed to restore DOMDocument from "%s"', $xml));
+ }
+
+ return $document;
+ }
+
+ public static function xmlElementFromString(string $xml): DOMElement
+ {
+ $element = self::xmlDocumentFromString($xml)->documentElement;
+
+ if ($element === null) {
+ throw new FloeException(sprintf('Floe failed to restore DOMElement from "%s"', $xml));
+ }
+
+ return $element;
+ }
+
+ private function dynamicDecoder(): DynamicDecoder
+ {
+ return new DynamicDecoder($this->dateTimeDecoder, $this->uuidDecoder, $this->jsonDecoder);
+ }
+
+ /**
+ * @param Type $type
+ */
+ private function listDecoder(Type $type): Decoding\ValueDecoder
+ {
+ /** @var ListType $type */
+ $element = $type->element();
+
+ if ($element instanceof IntegerType) {
+ return new PackedListDecoder('P');
+ }
+
+ if ($element instanceof FloatType) {
+ return new PackedListDecoder('e');
+ }
+
+ return new ListDecoder($this->decoderFor($element));
+ }
+
+ /**
+ * @param Type $type
+ */
+ private function mapDecoder(Type $type): Decoding\ValueDecoder
+ {
+ /** @var MapType $type */
+ $key = $type->key();
+
+ $keyDecoder = match (true) {
+ $key instanceof IntegerType => new Int64Decoder(),
+ $key instanceof StringType => new StringDecoder(),
+ default => $this->dynamicDecoder(),
+ };
+
+ return new MapDecoder($keyDecoder, $this->decoderFor($type->value()));
+ }
+
+ /**
+ * @param Type $type
+ */
+ private function optionalDecoder(Type $type): Decoding\ValueDecoder
+ {
+ /** @var OptionalType $type */
+ return new OptionalDecoder($this->decoderFor($type->base()));
+ }
+
+ /**
+ * @param Type $type
+ */
+ private function structureDecoder(Type $type): Decoding\ValueDecoder
+ {
+ /** @var StructureType $type */
+ $elements = [];
+
+ foreach ($type->elements() as $name => $elementType) {
+ $elements[$name] = $this->decoderFor($elementType);
+ }
+
+ foreach ($type->optionalElements() as $name => $elementType) {
+ $elements[$name] = $this->decoderFor($elementType);
+ }
+
+ return new StructureDecoder($elements, $type->allowsExtra(), $this->dynamicDecoder());
+ }
+}
diff --git a/src/core/etl/src/Flow/Floe/ValueEncoder.php b/src/core/etl/src/Flow/Floe/ValueEncoder.php
new file mode 100644
index 0000000000..a0a456d599
--- /dev/null
+++ b/src/core/etl/src/Flow/Floe/ValueEncoder.php
@@ -0,0 +1,206 @@
+ $type
+ *
+ * @throws FloeException
+ */
+ public function encoderFor(Type $type): Encoding\ValueEncoder
+ {
+ return match ($type::class) {
+ IntegerType::class, PositiveIntegerType::class => new Int64Encoder(),
+ FloatType::class => new Float64Encoder(),
+ BooleanType::class => new BooleanEncoder(),
+ StringType::class,
+ NonEmptyStringType::class,
+ NumericStringType::class,
+ ClassStringType::class,
+ => new StringEncoder(),
+ TimeZoneType::class => new TimeZoneEncoder(),
+ DateTimeType::class, DateType::class => new DateTimeEncoder(),
+ TimeType::class => new IntervalEncoder(),
+ UuidType::class => new UuidEncoder(),
+ JsonType::class => new JsonEncoder(),
+ EnumType::class => new EnumEncoder(),
+ XMLType::class => new XmlDocumentEncoder(),
+ XMLElementType::class => new XmlElementEncoder(),
+ HTMLType::class => new HtmlDocumentEncoder(),
+ HTMLElementType::class => new HtmlElementEncoder(),
+ ListType::class => $this->listEncoder($type),
+ MapType::class => $this->mapEncoder($type),
+ StructureType::class => $this->structureEncoder($type),
+ OptionalType::class => new OptionalEncoder($this->encoderFor($type->base())),
+ NullType::class => new NullEncoder(),
+ MixedType::class,
+ UnionType::class,
+ ScalarType::class,
+ LiteralType::class,
+ ArrayType::class,
+ => new DynamicEncoder(),
+ default => throw new FloeException(sprintf('Floe does not support values of type "%s"', $type->toString())),
+ };
+ }
+
+ public static function lengthPrefixed(string $value): string
+ {
+ return pack('V', strlen($value)) . $value;
+ }
+
+ public static function htmlElementToString(object $value): string
+ {
+ /** @var \Dom\HTMLElement $value */
+ // @mago-ignore analysis:non-existent-method
+ // @mago-ignore analysis:mixed-assignment
+ $html = $value->ownerDocument?->saveHtml($value);
+
+ if (!is_string($html)) {
+ throw new FloeException('Floe failed to convert HTMLElement to HTML string');
+ }
+
+ return $html;
+ }
+
+ public static function xmlDocumentToString(DOMDocument $value): string
+ {
+ $xml = $value->saveXML();
+
+ if ($xml === false) {
+ throw new FloeException('Floe failed to convert DOMDocument to XML string');
+ }
+
+ return $xml;
+ }
+
+ public static function xmlElementToString(DOMElement $value): string
+ {
+ $xml = $value->ownerDocument?->saveXML($value);
+
+ if ($xml === null || $xml === false) {
+ throw new FloeException('Floe failed to convert DOMElement to XML string');
+ }
+
+ return $xml;
+ }
+
+ /**
+ * @param Type $type
+ */
+ private function listEncoder(Type $type): Encoding\ValueEncoder
+ {
+ /** @var ListType $type */
+ $element = $type->element();
+
+ if ($element instanceof IntegerType) {
+ return new PackedListEncoder('P');
+ }
+
+ if ($element instanceof FloatType) {
+ return new PackedListEncoder('e');
+ }
+
+ return new ListEncoder($this->encoderFor($element));
+ }
+
+ /**
+ * @param Type $type
+ */
+ private function mapEncoder(Type $type): Encoding\ValueEncoder
+ {
+ /** @var MapType $type */
+ $key = $type->key();
+
+ $keyEncoder = match (true) {
+ $key instanceof IntegerType => new Int64Encoder(),
+ $key instanceof StringType => new StringKeyEncoder(),
+ default => new DynamicEncoder(),
+ };
+
+ return new MapEncoder($keyEncoder, $this->encoderFor($type->value()));
+ }
+
+ /**
+ * @param Type $type
+ */
+ private function structureEncoder(Type $type): Encoding\ValueEncoder
+ {
+ /** @var StructureType $type */
+ $elements = [];
+
+ foreach ($type->elements() as $name => $elementType) {
+ $elements[$name] = $this->encoderFor($elementType);
+ }
+
+ foreach ($type->optionalElements() as $name => $elementType) {
+ $elements[$name] = $this->encoderFor($elementType);
+ }
+
+ return new StructureEncoder($elements, $type->allowsExtra(), new DynamicEncoder());
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeRandomOrdersExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeRandomOrdersExtractor.php
index f109d123df..2a1cb78056 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeRandomOrdersExtractor.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeRandomOrdersExtractor.php
@@ -67,8 +67,10 @@ public static function schema(): Schema
public function extract(FlowContext $context): Generator
{
+ $schema = self::schema();
+
foreach ($this->rawData() as $row) {
- yield array_to_rows($row, $context->entryFactory(), schema: self::schema());
+ yield array_to_rows($row, $context->entryFactory(), schema: $schema);
}
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeStaticOrdersExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeStaticOrdersExtractor.php
index 75e7418942..e15bf25d16 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeStaticOrdersExtractor.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeStaticOrdersExtractor.php
@@ -66,8 +66,10 @@ public static function schema(): Schema
public function extract(FlowContext $context): Generator
{
+ $schema = self::schema();
+
foreach ($this->rawData() as $row) {
- yield array_to_rows($row, $context->entryFactory(), schema: self::schema());
+ yield array_to_rows($row, $context->entryFactory(), schema: $schema);
}
}
@@ -128,9 +130,10 @@ public function rawData(): Generator
public function toRows(EntryFactory $entryFactory = new EntryFactory()): Rows
{
$rows = rows();
+ $schema = self::schema();
foreach ($this->rawData() as $row) {
- $rows = $rows->merge(array_to_rows($row, entryFactory: $entryFactory, schema: self::schema()));
+ $rows = $rows->merge(array_to_rows($row, entryFactory: $entryFactory, schema: $schema));
}
return $rows;
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Fixtures/CustomDateTime.php b/src/core/etl/tests/Flow/ETL/Tests/Fixtures/CustomDateTime.php
new file mode 100644
index 0000000000..5d5a88a00c
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Fixtures/CustomDateTime.php
@@ -0,0 +1,9 @@
+fs = new NativeLocalFilesystem();
$this->fstab = new FilesystemTable($this->fs, new StdOutFilesystem());
- $this->serializer = new Base64Serializer(new NativePHPSerializer());
+ $this->serializer = new FloeSerializer();
$this->cleanupCacheDir($this->cacheDir);
mkdir($this->cacheDir->path(), recursive: true);
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/CacheBaseTestSuite.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/CacheBaseTestSuite.php
deleted file mode 100644
index 8e67089087..0000000000
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/CacheBaseTestSuite.php
+++ /dev/null
@@ -1,130 +0,0 @@
-cache()->clear();
- }
-
- #[Override]
- protected function tearDown(): void
- {
- parent::tearDown();
-
- $this->cache()->clear();
- }
-
- public function test_caching_index(): void
- {
- $cache = $this->cache();
-
- static::assertFalse($cache->has('index'));
-
- $cache->set('index', $index = new CacheIndex('index'));
-
- static::assertTrue($cache->has('index'));
-
- static::assertEquals($index, $cache->get('index'));
- }
-
- public function test_caching_row(): void
- {
- $cache = $this->cache();
-
- static::assertFalse($cache->has('row'));
-
- $cache->set('row', $row = row(str_entry('name', 'John')));
-
- static::assertTrue($cache->has('row'));
-
- static::assertEquals($row, $cache->get('row'));
- }
-
- public function test_caching_rows(): void
- {
- $cache = $this->cache();
-
- static::assertFalse($cache->has('rows'));
-
- $cache->set('rows', $rows = rows(row(str_entry('name', 'John')), row(str_entry('name', 'Jane'))));
-
- static::assertTrue($cache->has('rows'));
-
- static::assertEquals($rows, $cache->get('rows'));
- }
-
- public function test_checking_on_non_existing_cache_key(): void
- {
- $cache = $this->cache();
-
- static::assertFalse($cache->has('non-existing'));
- }
-
- public function test_clearing_cache(): void
- {
- $cache = $this->cache();
-
- $cache->set('index', new CacheIndex('index'));
- $cache->set('row', row(str_entry('name', 'John')));
- $cache->set('rows', rows(row(str_entry('name', 'John')), row(str_entry('name', 'Jane'))));
-
- $cache->clear();
-
- static::assertFalse($cache->has('index'));
- static::assertFalse($cache->has('row'));
- static::assertFalse($cache->has('rows'));
- }
-
- public function test_getting_non_existing_cache_key(): void
- {
- $cache = $this->cache();
-
- $this->expectException(KeyNotInCacheException::class);
-
- $cache->get('non-existing');
- }
-
- public function test_removing_from_cache(): void
- {
- $cache = $this->cache();
-
- $cache->set('index', new CacheIndex('index'));
- $cache->set('row', row(str_entry('name', 'John')));
- $cache->set('rows', rows(row(str_entry('name', 'John')), row(str_entry('name', 'Jane'))));
-
- $cache->delete('row');
-
- static::assertTrue($cache->has('index'));
- static::assertFalse($cache->has('row'));
- static::assertTrue($cache->has('rows'));
- }
-
- public function test_removing_non_existing_cache_key(): void
- {
- $cache = $this->cache();
-
- $cache->delete('non-existing');
-
- static::assertFalse($cache->has('non-existing'));
- }
-
- abstract protected function cache(): Cache;
-}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/CacheTestCase.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/CacheTestCase.php
new file mode 100644
index 0000000000..6e3671f037
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/CacheTestCase.php
@@ -0,0 +1,196 @@
+cache()->clear();
+ }
+
+ #[Override]
+ protected function tearDown(): void
+ {
+ parent::tearDown();
+
+ $this->cache()->clear();
+ }
+
+ public function test_caching_index(): void
+ {
+ $cache = $this->cache();
+
+ static::assertFalse($cache->has('index'));
+
+ $index = new CacheIndex('index');
+ $index->add('chunk-1');
+ $index->add('chunk-2');
+
+ $cache->set('index', $index->toRows());
+
+ static::assertTrue($cache->has('index'));
+
+ $indexRows = $cache->get('index');
+
+ static::assertInstanceOf(Rows::class, $indexRows);
+ static::assertEquals($index, CacheIndex::fromRows('index', $indexRows));
+ }
+
+ public function test_caching_rows(): void
+ {
+ $cache = $this->cache();
+
+ static::assertFalse($cache->has('rows'));
+
+ $cache->set('rows', $rows = rows(row(str_entry('name', 'John')), row(str_entry('name', 'Jane'))));
+
+ static::assertTrue($cache->has('rows'));
+
+ static::assertEquals($rows, $cache->get('rows'));
+ }
+
+ public function test_caching_partitioned_rows(): void
+ {
+ $cache = $this->cache();
+
+ $cache->set(
+ 'partitioned',
+ $rows = Rows::partitioned([row(int_entry('id', 1), str_entry('country', 'PL'))], [new Partition(
+ 'country',
+ 'PL',
+ )]),
+ );
+
+ static::assertTrue($cache->has('partitioned'));
+ static::assertEquals($rows, $cache->get('partitioned'));
+ }
+
+ public function test_checking_on_non_existing_cache_key(): void
+ {
+ $cache = $this->cache();
+
+ static::assertFalse($cache->has('non-existing'));
+ }
+
+ public function test_clearing_cache(): void
+ {
+ $cache = $this->cache();
+
+ $cache->set('index', (new CacheIndex('index'))->toRows());
+ $cache->set('row', rows(row(str_entry('name', 'John'))));
+ $cache->set('rows', rows(row(str_entry('name', 'John')), row(str_entry('name', 'Jane'))));
+
+ $cache->clear();
+
+ static::assertFalse($cache->has('index'));
+ static::assertFalse($cache->has('row'));
+ static::assertFalse($cache->has('rows'));
+ }
+
+ public function test_getting_non_existing_cache_key(): void
+ {
+ $cache = $this->cache();
+
+ $this->expectException(KeyNotInCacheException::class);
+
+ $cache->get('non-existing');
+ }
+
+ public function test_reading_cached_value_in_batches(): void
+ {
+ $cache = $this->cache();
+
+ $cache->set('rows', $rows = rows(row(str_entry('name', 'John')), row(str_entry('name', 'Jane'))));
+
+ $batches = iterator_to_array($cache->read('rows'), preserve_keys: false);
+
+ static::assertNotEmpty($batches);
+ static::assertEquals(
+ $rows,
+ rows(...array_merge(...array_map(static fn(Rows $batch): array => $batch->all(), $batches))),
+ );
+ }
+
+ public function test_reading_non_existing_cache_key(): void
+ {
+ $cache = $this->cache();
+
+ $this->expectException(KeyNotInCacheException::class);
+
+ iterator_to_array($cache->read('non-existing'));
+ }
+
+ public function test_reading_partitioned_rows_in_batches(): void
+ {
+ $cache = $this->cache();
+
+ $cache->set(
+ 'partitioned',
+ $rows = Rows::partitioned([row(int_entry('id', 1), str_entry('country', 'PL'))], [new Partition(
+ 'country',
+ 'PL',
+ )]),
+ );
+
+ $batches = iterator_to_array($cache->read('partitioned'), preserve_keys: false);
+
+ static::assertNotEmpty($batches);
+
+ foreach ($batches as $batch) {
+ static::assertEquals($rows->partitions()->toArray(), $batch->partitions()->toArray());
+ }
+
+ static::assertEquals(
+ $rows->all(),
+ array_merge(...array_map(static fn(Rows $batch): array => $batch->all(), $batches)),
+ );
+ }
+
+ public function test_removing_from_cache(): void
+ {
+ $cache = $this->cache();
+
+ $cache->set('index', (new CacheIndex('index'))->toRows());
+ $cache->set('row', rows(row(str_entry('name', 'John'))));
+ $cache->set('rows', rows(row(str_entry('name', 'John')), row(str_entry('name', 'Jane'))));
+
+ $cache->delete('row');
+
+ static::assertTrue($cache->has('index'));
+ static::assertFalse($cache->has('row'));
+ static::assertTrue($cache->has('rows'));
+ }
+
+ public function test_removing_non_existing_cache_key(): void
+ {
+ $cache = $this->cache();
+
+ $cache->delete('non-existing');
+
+ static::assertFalse($cache->has('non-existing'));
+ }
+
+ abstract protected function cache(): Cache;
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheStreamingTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheStreamingTest.php
new file mode 100644
index 0000000000..87fefed3a3
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheStreamingTest.php
@@ -0,0 +1,120 @@
+ row(int_entry('id', $id)), range(1, 10)));
+
+ $bulkCache = new FilesystemCache($this->fs(), path(__DIR__ . '/var/filesystem-cache-bulk'));
+
+ $this->cache()->set('parity', $rows);
+ $bulkCache->set('parity', $rows);
+
+ $streamingFiles = glob(__DIR__ . '/var/filesystem-cache-streaming/*/*/*/*/parity.floe') ?: [];
+ $bulkFiles = glob(__DIR__ . '/var/filesystem-cache-bulk/*/*/*/*/parity.floe') ?: [];
+
+ static::assertNotEmpty($streamingFiles);
+ static::assertNotEmpty($bulkFiles);
+ static::assertSame(file_get_contents($bulkFiles[0]), file_get_contents($streamingFiles[0]));
+
+ $bulkCache->clear();
+ }
+
+ public function test_filesystem_cache_dsl_delegates_mode_and_batch_size(): void
+ {
+ $cache = filesystem_cache(path(__DIR__ . '/var/filesystem-cache-dsl'), serializer_batch_size: 2);
+
+ $cache->set('dsl', rows(...array_map(static fn(int $id): Row => row(int_entry('id', $id)), range(1, 5))));
+
+ static::assertCount(3, iterator_to_array($cache->read('dsl'), preserve_keys: false));
+
+ $cache->clear();
+ }
+
+ public function test_reading_yields_batches_of_batch_size(): void
+ {
+ $cache = $this->cache();
+
+ $cache->set('batched', rows(...array_map(static fn(int $id): Row => row(int_entry('id', $id)), range(1, 10))));
+
+ $batches = iterator_to_array($cache->read('batched'), preserve_keys: false);
+
+ static::assertCount(4, $batches);
+ static::assertSame([3, 3, 3, 1], array_map(static fn(Rows $batch): int => $batch->count(), $batches));
+ }
+
+ public function test_torn_entry_is_treated_as_a_cache_miss(): void
+ {
+ $cache = $this->cache();
+ $cache->set('torn', rows(row(int_entry('id', 1))));
+
+ $files = glob(__DIR__ . '/var/filesystem-cache-streaming/*/*/*/*/torn.floe') ?: [];
+ static::assertNotEmpty($files);
+ file_put_contents($files[0], 'not a valid floe file');
+
+ $this->expectException(KeyNotInCacheException::class);
+
+ $cache->get('torn');
+ }
+
+ public function test_corrupted_frame_with_intact_footer_is_treated_as_a_cache_miss(): void
+ {
+ $cache = $this->cache();
+ $cache->set('corrupt', rows(row(int_entry('id', 1))));
+
+ $files = glob(__DIR__ . '/var/filesystem-cache-streaming/*/*/*/*/corrupt.floe') ?: [];
+ static::assertNotEmpty($files);
+
+ // corrupt the first frame type (SCHEMA -> unknown 0x7F); the footer at the end stays
+ // intact, so the recovered row count no longer matches the footer's totalRows
+ $bytes = (string) file_get_contents($files[0]);
+ $bytes[6] = "\x7F";
+ file_put_contents($files[0], $bytes);
+
+ $this->expectException(KeyNotInCacheException::class);
+
+ $cache->get('corrupt');
+ }
+
+ public function test_torn_entry_read_is_treated_as_a_cache_miss(): void
+ {
+ $cache = $this->cache();
+ $cache->set('torn-read', rows(row(int_entry('id', 1))));
+
+ $files = glob(__DIR__ . '/var/filesystem-cache-streaming/*/*/*/*/torn-read.floe') ?: [];
+ static::assertNotEmpty($files);
+ file_put_contents($files[0], 'not a valid floe file');
+
+ $this->expectException(KeyNotInCacheException::class);
+
+ iterator_to_array($cache->read('torn-read'));
+ }
+
+ protected function cache(): Cache
+ {
+ return new FilesystemCache($this->fs(), path(__DIR__ . '/var/filesystem-cache-streaming'), 3);
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheTest.php
new file mode 100644
index 0000000000..690c701206
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheTest.php
@@ -0,0 +1,39 @@
+cache();
+ $cache->set('torn', rows(row(int_entry('id', 1))));
+
+ // simulate a crash mid-write / bit rot: overwrite the closed .floe file with garbage
+ $files = glob(__DIR__ . '/var/filesystem-cache/*/*/*/*/torn.floe') ?: [];
+ static::assertNotEmpty($files);
+ file_put_contents($files[0], 'not a valid floe file');
+
+ $this->expectException(KeyNotInCacheException::class);
+
+ $cache->get('torn');
+ }
+
+ protected function cache(): Cache
+ {
+ return new FilesystemCache($this->fs(), path(__DIR__ . '/var/filesystem-cache'));
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheTestSuite.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheTestSuite.php
deleted file mode 100644
index a5f4d1c803..0000000000
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheTestSuite.php
+++ /dev/null
@@ -1,18 +0,0 @@
-fs(), $this->serializer(), path(__DIR__ . '/var/filesystem-cache'));
- }
-}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/InMemoryCacheTestSuite.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/InMemoryCacheTest.php
similarity index 79%
rename from src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/InMemoryCacheTestSuite.php
rename to src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/InMemoryCacheTest.php
index fadfbf94fe..270a11da82 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/InMemoryCacheTestSuite.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/InMemoryCacheTest.php
@@ -7,7 +7,7 @@
use Flow\ETL\Cache;
use Flow\ETL\Cache\Implementation\InMemoryCache;
-final class InMemoryCacheTestSuite extends CacheBaseTestSuite
+final class InMemoryCacheTest extends CacheTestCase
{
protected function cache(): Cache
{
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleFilesystemCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleFilesystemCacheTest.php
new file mode 100644
index 0000000000..7f296592e6
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleFilesystemCacheTest.php
@@ -0,0 +1,38 @@
+set('torn', rows(row(int_entry('id', 1))));
+
+ $psr->set('torn', 'not a valid floe payload');
+
+ $this->expectException(KeyNotInCacheException::class);
+
+ $cache->get('torn');
+ }
+
+ protected function cache(): Cache
+ {
+ return new PSRSimpleCache(new Psr16Cache(
+ new FilesystemAdapter(directory: __DIR__ . '/var/psr-simple-file-cache'),
+ ));
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleFilesystemCacheTestSuite.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleFilesystemCacheTestSuite.php
deleted file mode 100644
index 4383d86dd0..0000000000
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleFilesystemCacheTestSuite.php
+++ /dev/null
@@ -1,20 +0,0 @@
-dsn(), ['retry_interval' => 1, 'timeout' => 1]);
+ $this->redisAvailable = true;
+ } catch (Exception) {
+ self::markTestSkipped('Redis server is not available.');
+ }
+
+ parent::setUp();
+ }
+
+ #[Override]
+ protected function tearDown(): void
+ {
+ // Skipped in setUp before the parent state was initialized - there is
+ // nothing to clean up and the parent tearDown would error.
+ if (!$this->redisAvailable) {
+ return;
+ }
+
+ parent::tearDown();
+ }
+
+ protected function cache(): Cache
+ {
+ return new PSRSimpleCache(new Psr16Cache(new RedisAdapter(RedisAdapter::createConnection($this->dsn(), [
+ 'retry_interval' => 2,
+ 'timeout' => 5,
+ ]))));
+ }
+
+ protected function dsn(): string
+ {
+ $host = getenv('REDIS_HOST');
+ $port = getenv('REDIS_PORT');
+
+ return 'redis://' . ($host === false ? 'localhost' : $host) . ':' . ($port === false ? '6379' : $port) . '/0';
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleRedisCacheTestSuite.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleRedisCacheTestSuite.php
deleted file mode 100644
index 74d2fd3405..0000000000
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleRedisCacheTestSuite.php
+++ /dev/null
@@ -1,29 +0,0 @@
- 2,
- 'timeout' => 5,
- ],
- ))));
- }
-}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/TraceableCacheTestSuite.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/TraceableCacheTest.php
similarity index 96%
rename from src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/TraceableCacheTestSuite.php
rename to src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/TraceableCacheTest.php
index ac30eb4579..0caf323001 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/TraceableCacheTestSuite.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/TraceableCacheTest.php
@@ -20,7 +20,7 @@
use Flow\Telemetry\Tracer\TracerProvider;
use Override;
-final class TraceableCacheTestSuite extends CacheBaseTestSuite
+final class TraceableCacheTest extends CacheTestCase
{
private InMemoryCache $innerCache;
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/CacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/CacheTest.php
index eb6e6b32ca..d661cbf869 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/CacheTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/CacheTest.php
@@ -5,6 +5,7 @@
namespace Flow\ETL\Tests\Integration\DataFrame;
use Flow\ETL\Cache\CacheIndex;
+use Flow\ETL\Cache\Implementation\FilesystemCache;
use Flow\ETL\Cache\Implementation\InMemoryCache;
use Flow\ETL\Extractor;
use Flow\ETL\FlowContext;
@@ -31,6 +32,7 @@
use function Flow\ETL\DSL\from_array;
use function Flow\ETL\DSL\from_cache;
use function Flow\ETL\DSL\telemetry_options;
+use function Flow\Filesystem\DSL\path;
use function range;
final class CacheTest extends FlowIntegrationTestCase
@@ -63,7 +65,7 @@ public function extract(FlowContext $context): Generator
->run();
static::assertEquals(1, $spyExtractor->extractions);
- static::assertInstanceOf(CacheIndex::class, $cache->get('test_etl_cache'));
+ static::assertInstanceOf(Rows::class, $cache->get('test_etl_cache'));
df(config_builder()->cache($cache))->read(from_cache('test_etl_cache', $spyExtractor, clear: true))->run();
@@ -81,8 +83,11 @@ public function test_cache_with_previously_set_batch_size(): void
->cache('test')
->run();
- /** @var CacheIndex $cacheIndex */
- $cacheIndex = $cache->get('test');
+ $indexRows = $cache->get('test');
+
+ static::assertInstanceOf(Rows::class, $indexRows);
+
+ $cacheIndex = CacheIndex::fromRows('test', $indexRows);
static::assertCount(5, $cacheIndex->values());
@@ -93,6 +98,26 @@ public function test_cache_with_previously_set_batch_size(): void
}
}
+ public function test_cache_streaming_serializer_mode_end_to_end(): void
+ {
+ $input = array_map(static fn(int $i) => ['id' => $i], range(1, 25));
+
+ $bulkCache = new FilesystemCache($this->fs(), path(__DIR__ . '/var/cache-mode-bulk'));
+ $streamingCache = new FilesystemCache($this->fs(), path(__DIR__ . '/var/cache-mode-streaming'), 4);
+
+ df(config_builder()->cache($bulkCache))->read(from_array($input))->batchSize(10)->cache('parity')->run();
+ df(config_builder()->cache($streamingCache))->read(from_array($input))->batchSize(10)->cache('parity')->run();
+
+ $bulkRows = df(config_builder()->cache($bulkCache))->read(from_cache('parity'))->fetch();
+ $streamingRows = df(config_builder()->cache($streamingCache))->read(from_cache('parity'))->fetch();
+
+ static::assertSame($input, $bulkRows->toArray());
+ static::assertSame($input, $streamingRows->toArray());
+
+ $bulkCache->clear();
+ $streamingCache->clear();
+ }
+
public function test_cache_with_telemetry_collects_spans_and_metrics(): void
{
$clock = new SystemClock();
@@ -139,8 +164,11 @@ public function test_cache_without_previously_set_batch_size(): void
->cache('test')
->run();
- /** @var CacheIndex $cacheIndex */
- $cacheIndex = $cache->get('test');
+ $indexRows = $cache->get('test');
+
+ static::assertInstanceOf(Rows::class, $indexRows);
+
+ $cacheIndex = CacheIndex::fromRows('test', $indexRows);
static::assertCount(100, $cacheIndex->values());
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ConfigBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ConfigBuilderTest.php
index e7ab49eebe..420ebf864e 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ConfigBuilderTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ConfigBuilderTest.php
@@ -4,6 +4,7 @@
namespace Flow\ETL\Tests\Integration\DataFrame;
+use Flow\ETL\Cache\Implementation\InMemoryCache;
use Flow\ETL\Config\Cache\CacheConfig;
use Flow\ETL\Sort\SortAlgorithms;
use Flow\ETL\Tests\FlowIntegrationTestCase;
@@ -16,6 +17,9 @@
use function Flow\ETL\DSL\analyze;
use function Flow\ETL\DSL\config_builder;
+use function Flow\ETL\DSL\int_entry;
+use function Flow\ETL\DSL\row;
+use function Flow\ETL\DSL\rows;
use function Flow\ETL\DSL\telemetry_options;
use function Flow\Filesystem\DSL\filesystem_telemetry_options;
use function Flow\Filesystem\DSL\native_local_filesystem;
@@ -29,6 +33,7 @@
use function Flow\Telemetry\DSL\telemetry;
use function Flow\Telemetry\DSL\tracer_provider;
use function Flow\Telemetry\DSL\void_exporter;
+use function iterator_to_array;
use function str_replace;
final class ConfigBuilderTest extends FlowIntegrationTestCase
@@ -48,7 +53,33 @@ public function test_cache_filesystem_protocol_override(): void
->cacheFilesystem('custom-cache')
->build();
- static::assertSame('custom-cache', $config->cache->filesystemProtocol);
+ static::assertSame('custom-cache', $config->cache->filesystemMount);
+ }
+
+ public function test_cache_serializer_mode_reaches_the_default_cache(): void
+ {
+ putenv(CacheConfig::CACHE_DIR_ENV . '=' . __DIR__ . '/var/cache-serializer-mode');
+
+ $config = config_builder()->cacheSerializerBatchSize(2)->build();
+
+ $config->cache->cache->set('key', rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2)),
+ row(int_entry('id', 3)),
+ ));
+
+ static::assertCount(2, iterator_to_array($config->cache->cache->read('key')));
+
+ $config->cache->cache->clear();
+ }
+
+ public function test_custom_cache_wins_over_serializer_mode(): void
+ {
+ $custom = new InMemoryCache();
+
+ $config = config_builder()->cache($custom)->cacheSerializerBatchSize(500)->build();
+
+ static::assertSame($custom, $config->cache->cache);
}
public function test_config_builder_with_analyze(): void
@@ -93,7 +124,7 @@ public function test_default_cache_filesystem_protocol_is_file(): void
{
$config = config_builder()->build();
- static::assertSame('file', $config->cache->filesystemProtocol);
+ static::assertSame('file', $config->cache->filesystemMount);
}
public function test_default_external_sort_filesystem_protocol_is_file(): void
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/CacheExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/CacheExtractorTest.php
index b5d5191581..abcd35e673 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/CacheExtractorTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/CacheExtractorTest.php
@@ -5,9 +5,13 @@
namespace Flow\ETL\Tests\Integration\Extractor;
use Flow\ETL\Cache\CacheIndex;
+use Flow\ETL\Cache\Implementation\FilesystemCache;
use Flow\ETL\Cache\Implementation\InMemoryCache;
+use Flow\ETL\Extractor\Signal;
+use Flow\ETL\Rows;
use Flow\ETL\Tests\FlowIntegrationTestCase;
+use function array_map;
use function array_merge;
use function Flow\ETL\DSL\array_to_rows;
use function Flow\ETL\DSL\config;
@@ -15,6 +19,7 @@
use function Flow\ETL\DSL\flow_context;
use function Flow\ETL\DSL\from_array;
use function Flow\ETL\DSL\from_cache;
+use function Flow\Filesystem\DSL\path;
use function iterator_to_array;
final class CacheExtractorTest extends FlowIntegrationTestCase
@@ -32,7 +37,7 @@ public function test_extracting_rows_from_cache(): void
$cache->set('rows_02', array_to_rows([['id' => 3], ['id' => 4]], flow_context(config())->entryFactory()));
$cache->set('rows_03', array_to_rows([['id' => 5]], flow_context(config())->entryFactory()));
- $cache->set('key', $index);
+ $cache->set('key', $index->toRows());
$extractor = from_cache($cacheKey);
@@ -58,7 +63,7 @@ public function test_extracting_rows_from_cache_with_clearing_cache_afterwards()
$cache->set('rows_02', array_to_rows([['id' => 3], ['id' => 4]], flow_context(config())->entryFactory()));
$cache->set('rows_03', array_to_rows([['id' => 5]], flow_context(config())->entryFactory()));
- $cache->set('key', $index);
+ $cache->set('key', $index->toRows());
$extractor = from_cache($cacheKey)->withClearOnFinish(true);
@@ -71,6 +76,81 @@ public function test_extracting_rows_from_cache_with_clearing_cache_afterwards()
static::assertFalse($cache->has('key'));
}
+ public function test_extracting_rows_from_streaming_cache_in_batches(): void
+ {
+ $cache = new FilesystemCache($this->fs(), path(__DIR__ . '/var/cache-extractor-streaming'), 2);
+ $cache->clear();
+
+ $index = new CacheIndex($cacheKey = 'key');
+ $index->add('rows_01');
+
+ $cache->set('rows_01', array_to_rows([
+ ['id' => 1],
+ ['id' => 2],
+ ['id' => 3],
+ ['id' => 4],
+ ['id' => 5],
+ ], flow_context(config())->entryFactory()));
+ $cache->set('key', $index->toRows());
+
+ $extractor = from_cache($cacheKey);
+
+ $rows = iterator_to_array($extractor->extract(flow_context(config_builder()->cache($cache)->build())));
+
+ static::assertCount(3, $rows);
+ static::assertEquals(
+ [['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4], ['id' => 5]],
+ array_merge(...array_map(static fn(Rows $batch): array => $batch->toArray(), $rows)),
+ );
+
+ $cache->clear();
+ }
+
+ public function test_stop_signal_stops_streaming_batches_and_skips_clearing(): void
+ {
+ $cache = new FilesystemCache($this->fs(), path(__DIR__ . '/var/cache-extractor-streaming-stop'), 2);
+ $cache->clear();
+
+ $index = new CacheIndex($cacheKey = 'key');
+ $index->add('rows_01');
+
+ $cache->set('rows_01', array_to_rows([
+ ['id' => 1],
+ ['id' => 2],
+ ['id' => 3],
+ ['id' => 4],
+ ['id' => 5],
+ ], flow_context(config())->entryFactory()));
+ $cache->set('key', $index->toRows());
+
+ $generator = from_cache($cacheKey)
+ ->withClearOnFinish(true)
+ ->extract(flow_context(config_builder()->cache($cache)->build()));
+
+ static::assertTrue($generator->valid());
+
+ $generator->send(Signal::STOP);
+
+ static::assertFalse($generator->valid());
+ static::assertTrue($cache->has('rows_01'));
+ static::assertTrue($cache->has('key'));
+
+ $cache->clear();
+ }
+
+ public function test_stop_signal_stops_fallback_extractor(): void
+ {
+ $generator = from_cache('missing')
+ ->withFallbackExtractor(from_array([['id' => 1], ['id' => 2]]))
+ ->extract(flow_context(config_builder()->cache(new InMemoryCache())->build()));
+
+ static::assertTrue($generator->valid());
+
+ $generator->send(Signal::STOP);
+
+ static::assertFalse($generator->valid());
+ }
+
public function test_fallback_extractor(): void
{
$cache = new InMemoryCache();
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortTest.php
index f7d1e0ae36..286a8695a2 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortTest.php
@@ -41,7 +41,7 @@ public function test_memory_implementation_of_external_sort_algorithm(): void
$randomizedInput = $input;
shuffle($randomizedInput);
- $sort = new ExternalSort(new FilesystemBucketsCache($this->fs(), $this->serializer(), 100, $cacheDir));
+ $sort = new ExternalSort(new FilesystemBucketsCache($this->fs(), cacheDir: $cacheDir));
$context = flow_context();
$pipeline = new Pipeline(from_array($randomizedInput));
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/FilesystemBucketsCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/FilesystemBucketsCacheTest.php
new file mode 100644
index 0000000000..b80bf1a42b
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/FilesystemBucketsCacheTest.php
@@ -0,0 +1,81 @@
+fs()->rm($cacheDir);
+
+ $cache = new FilesystemBucketsCache($this->fs(), cacheDir: $cacheDir);
+
+ static::assertSame([], iterator_to_array($cache->get('nope')));
+ }
+
+ public function test_remove_deletes_the_bucket(): void
+ {
+ $cacheDir = path(__DIR__ . '/var/buckets_remove');
+ $this->fs()->rm($cacheDir);
+
+ $cache = new FilesystemBucketsCache($this->fs(), cacheDir: $cacheDir);
+ $cache->set('bucket', [row(int_entry('id', 1))]);
+ $cache->remove('bucket');
+
+ static::assertSame([], iterator_to_array($cache->get('bucket')));
+
+ $this->fs()->rm($cacheDir);
+ }
+
+ public function test_round_trips_rows_across_the_write_batch_boundary(): void
+ {
+ $cacheDir = path(__DIR__ . '/var/buckets_batch');
+ $this->fs()->rm($cacheDir);
+
+ $cache = new FilesystemBucketsCache($this->fs(), cacheDir: $cacheDir);
+
+ $input = [];
+
+ for ($i = 0; $i < 1500; $i++) {
+ $input[] = row(int_entry('id', $i));
+ }
+
+ $cache->set('bucket', $input);
+
+ static::assertEquals($input, iterator_to_array($cache->get('bucket'), false));
+
+ $this->fs()->rm($cacheDir);
+ }
+
+ public function test_round_trips_schema_changing_rows(): void
+ {
+ $cacheDir = path(__DIR__ . '/var/buckets_heterogeneous');
+ $this->fs()->rm($cacheDir);
+
+ $cache = new FilesystemBucketsCache($this->fs(), cacheDir: $cacheDir);
+
+ $input = [
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2), str_entry('name', 'John')),
+ row(str_entry('name', 'Jane')),
+ ];
+
+ $cache->set('bucket', $input);
+
+ static::assertEquals($input, iterator_to_array($cache->get('bucket'), false));
+
+ $this->fs()->rm($cacheDir);
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/CacheIndexTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/CacheIndexTest.php
new file mode 100644
index 0000000000..79deb2bcb0
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/CacheIndexTest.php
@@ -0,0 +1,78 @@
+toRows();
+
+ static::assertCount(0, $indexRows);
+ static::assertEquals($index, CacheIndex::fromRows('dataset-id', $indexRows));
+ }
+
+ public function test_from_rows_key_comes_from_the_argument(): void
+ {
+ $index = new CacheIndex('original-key');
+ $index->add('chunk-1');
+
+ $reconstructed = CacheIndex::fromRows('different-key', $index->toRows());
+
+ static::assertSame('different-key', $reconstructed->key);
+ static::assertSame(['chunk-1'], $reconstructed->values());
+ }
+
+ public function test_from_rows_with_missing_key_entry_throws(): void
+ {
+ $this->expectException(InvalidArgumentException::class);
+
+ CacheIndex::fromRows('dataset-id', rows(row(int_entry('id', 1))));
+ }
+
+ public function test_from_rows_with_non_string_key_entry_throws(): void
+ {
+ $this->expectException(InvalidArgumentException::class);
+ $this->expectExceptionMessage('CacheIndex expects rows with a string "key" entry, got: null');
+
+ CacheIndex::fromRows('dataset-id', rows(row(str_entry('key', null))));
+ }
+
+ public function test_to_rows_from_rows_round_trip_preserves_order(): void
+ {
+ $index = new CacheIndex('dataset-id');
+ $index->add('chunk-b');
+ $index->add('chunk-a');
+ $index->add('chunk-c');
+
+ $reconstructed = CacheIndex::fromRows('dataset-id', $index->toRows());
+
+ static::assertEquals($index, $reconstructed);
+ static::assertSame(['chunk-b', 'chunk-a', 'chunk-c'], $reconstructed->values());
+ }
+
+ public function test_to_rows_stores_one_key_entry_per_row(): void
+ {
+ $index = new CacheIndex('dataset-id');
+ $index->add('chunk-1');
+ $index->add('chunk-2');
+
+ static::assertEquals(
+ rows(row(str_entry('key', 'chunk-1')), row(str_entry('key', 'chunk-2'))),
+ $index->toRows(),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/Implementation/TraceableCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/Implementation/TraceableCacheTest.php
index 5b948162d5..6cc4e3d636 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/Implementation/TraceableCacheTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/Implementation/TraceableCacheTest.php
@@ -5,7 +5,6 @@
namespace Flow\ETL\Tests\Unit\Cache\Implementation;
use Flow\ETL\Cache;
-use Flow\ETL\Cache\CacheIndex;
use Flow\ETL\Cache\Implementation\InMemoryCache;
use Flow\ETL\Cache\Implementation\TraceableCache;
use Flow\ETL\Exception\KeyNotInCacheException;
@@ -27,6 +26,7 @@
use function Flow\ETL\DSL\row;
use function Flow\ETL\DSL\rows;
+use function iterator_to_array;
#[CoversClass(TraceableCache::class)]
final class TraceableCacheTest extends FlowTestCase
@@ -100,7 +100,7 @@ public function test_get_increments_hit_counter_on_existing_key(): void
$innerCache = new InMemoryCache();
$cache = new TraceableCache($innerCache, $this->telemetry, 'test_dataframe');
- $innerCache->set('existing-key', row());
+ $innerCache->set('existing-key', rows(row()));
$cache->get('existing-key');
$this->telemetry->flush();
@@ -135,15 +135,14 @@ public function test_get_increments_miss_counter_and_throws_on_non_existing_key(
}
}
- public function test_has_increments_hit_counter_when_key_exists(): void
+ public function test_read_increments_hit_counter_on_existing_key(): void
{
$innerCache = new InMemoryCache();
$cache = new TraceableCache($innerCache, $this->telemetry, 'test_dataframe');
- $innerCache->set('existing-key', row());
- $exists = $cache->has('existing-key');
+ $innerCache->set('existing-key', $rows = rows(row()));
- static::assertTrue($exists);
+ static::assertEquals([$rows], iterator_to_array($cache->read('existing-key'), preserve_keys: false));
$this->telemetry->flush();
$hitMetrics = $this->metricProcessor->metricsWithName('flow.cache.hits');
@@ -152,71 +151,68 @@ public function test_has_increments_hit_counter_when_key_exists(): void
static::assertCount(1, $hitMetrics);
static::assertCount(0, $missMetrics);
static::assertSame(1, $hitMetrics[0]->value);
- static::assertSame('{operation}', $hitMetrics[0]->unit);
static::assertSame('test_dataframe', $hitMetrics[0]->attributes->get('flow.etl.dataframe.name'));
}
- public function test_has_increments_miss_counter_when_key_does_not_exist(): void
+ public function test_read_increments_miss_counter_and_throws_on_non_existing_key(): void
{
$innerCache = new InMemoryCache();
$cache = new TraceableCache($innerCache, $this->telemetry, 'test_dataframe');
- $exists = $cache->has('non-existing-key');
-
- static::assertFalse($exists);
+ $this->expectException(KeyNotInCacheException::class);
- $this->telemetry->flush();
- $hitMetrics = $this->metricProcessor->metricsWithName('flow.cache.hits');
- $missMetrics = $this->metricProcessor->metricsWithName('flow.cache.misses');
+ try {
+ iterator_to_array($cache->read('non-existing-key'));
+ } finally {
+ $this->telemetry->flush();
+ $hitMetrics = $this->metricProcessor->metricsWithName('flow.cache.hits');
+ $missMetrics = $this->metricProcessor->metricsWithName('flow.cache.misses');
- static::assertCount(0, $hitMetrics);
- static::assertCount(1, $missMetrics);
- static::assertSame(1, $missMetrics[0]->value);
- static::assertSame('test_dataframe', $missMetrics[0]->attributes->get('flow.etl.dataframe.name'));
+ static::assertCount(0, $hitMetrics);
+ static::assertCount(1, $missMetrics);
+ static::assertSame(1, $missMetrics[0]->value);
+ static::assertSame('test_dataframe', $missMetrics[0]->attributes->get('flow.etl.dataframe.name'));
+ }
}
- public function test_set_creates_span_with_cache_index_value_type(): void
+ public function test_has_increments_hit_counter_when_key_exists(): void
{
$innerCache = new InMemoryCache();
- $cache = new TraceableCache($innerCache, $this->telemetry);
-
- $cache->set('test-key', new CacheIndex('test-id'));
+ $cache = new TraceableCache($innerCache, $this->telemetry, 'test_dataframe');
- $this->telemetry->flush();
- $spans = $this->spanProcessor->endedSpans();
+ $innerCache->set('existing-key', rows(row()));
+ $exists = $cache->has('existing-key');
- static::assertCount(1, $spans);
+ static::assertTrue($exists);
- $span = $spans[0];
- static::assertSame('cache.set', $span->name());
- static::assertSame(SpanKind::CLIENT, $span->kind());
+ $this->telemetry->flush();
+ $hitMetrics = $this->metricProcessor->metricsWithName('flow.cache.hits');
+ $missMetrics = $this->metricProcessor->metricsWithName('flow.cache.misses');
- $attributes = $span->attributes();
- static::assertSame('set', $attributes['cache.operation']);
- static::assertSame('test-key', $attributes['cache.key']);
- static::assertSame('CacheIndex', $attributes['cache.value_type']);
+ static::assertCount(1, $hitMetrics);
+ static::assertCount(0, $missMetrics);
+ static::assertSame(1, $hitMetrics[0]->value);
+ static::assertSame('{operation}', $hitMetrics[0]->unit);
+ static::assertSame('test_dataframe', $hitMetrics[0]->attributes->get('flow.etl.dataframe.name'));
}
- public function test_set_creates_span_with_correct_name_and_attributes_for_row(): void
+ public function test_has_increments_miss_counter_when_key_does_not_exist(): void
{
$innerCache = new InMemoryCache();
- $cache = new TraceableCache($innerCache, $this->telemetry);
-
- $cache->set('test-key', row());
+ $cache = new TraceableCache($innerCache, $this->telemetry, 'test_dataframe');
- $this->telemetry->flush();
- $spans = $this->spanProcessor->endedSpans();
+ $exists = $cache->has('non-existing-key');
- static::assertCount(1, $spans);
+ static::assertFalse($exists);
- $span = $spans[0];
- static::assertSame('cache.set', $span->name());
- static::assertSame(SpanKind::CLIENT, $span->kind());
+ $this->telemetry->flush();
+ $hitMetrics = $this->metricProcessor->metricsWithName('flow.cache.hits');
+ $missMetrics = $this->metricProcessor->metricsWithName('flow.cache.misses');
- $attributes = $span->attributes();
- static::assertSame('set', $attributes['cache.operation']);
- static::assertSame('test-key', $attributes['cache.key']);
- static::assertSame('Row', $attributes['cache.value_type']);
+ static::assertCount(0, $hitMetrics);
+ static::assertCount(1, $missMetrics);
+ static::assertSame(1, $missMetrics[0]->value);
+ static::assertSame('test_dataframe', $missMetrics[0]->attributes->get('flow.etl.dataframe.name'));
}
public function test_set_creates_span_with_rows_value_type(): void
@@ -296,7 +292,7 @@ public function test_set_records_exception_on_error(): void
$this->expectExceptionMessage('Test error');
try {
- $cache->set('test-key', row());
+ $cache->set('test-key', rows(row()));
} finally {
$this->telemetry->flush();
$spans = $this->spanProcessor->endedSpans();
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CachingProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CachingProcessorTest.php
index c1be8ec6f8..e207a96673 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CachingProcessorTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CachingProcessorTest.php
@@ -7,6 +7,7 @@
use Flow\ETL\Cache\CacheIndex;
use Flow\ETL\Cache\Implementation\InMemoryCache;
use Flow\ETL\Processor\CachingProcessor;
+use Flow\ETL\Rows;
use Flow\ETL\Tests\FlowTestCase;
use function Flow\ETL\DSL\config_builder;
@@ -33,6 +34,18 @@ public function test_caches_batches_and_yields_them(): void
static::assertCount(2, $result);
static::assertTrue($cache->has('test-cache'));
+
+ $indexRows = $cache->get('test-cache');
+
+ static::assertInstanceOf(Rows::class, $indexRows);
+
+ $index = CacheIndex::fromRows('test-cache', $indexRows);
+
+ static::assertCount(2, $index->values());
+
+ foreach ($index->values() as $cacheKey) {
+ static::assertTrue($cache->has($cacheKey));
+ }
}
public function test_handles_empty_input(): void
@@ -57,7 +70,7 @@ public function test_passes_through_when_cache_exists(): void
$cache = new InMemoryCache();
$context = flow_context(config_builder()->cache($cache)->build());
- $cache->set('test-cache', new CacheIndex('test-cache'));
+ $cache->set('test-cache', (new CacheIndex('test-cache'))->toRows());
$processor = new CachingProcessor('test-cache');
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/EntriesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/EntriesTest.php
index c31caaede9..b3d2d5e005 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/EntriesTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/EntriesTest.php
@@ -204,6 +204,22 @@ public function test_merge_entries(): void
);
}
+ public function test_names(): void
+ {
+ $entries = new Entries(
+ integer_entry('integer', 100),
+ string_entry('string', 'new string entry'),
+ boolean_entry('bool', true),
+ );
+
+ static::assertSame(['integer', 'string', 'bool'], $entries->names());
+ }
+
+ public function test_names_of_empty_entries(): void
+ {
+ static::assertSame([], (new Entries())->names());
+ }
+
public function test_order_entries(): void
{
$entries = new Entries(
@@ -279,6 +295,23 @@ public function test_prevents_from_getting_unknown_entry(): void
$entries->get('unknown');
}
+ public function test_recreate_from_name_keyed_entries(): void
+ {
+ $integerEntry = integer_entry('integer-entry', 100);
+ $stringEntry = string_entry('string-entry', 'just a string');
+
+ $entries = Entries::recreate(['integer-entry' => $integerEntry, 'string-entry' => $stringEntry]);
+
+ static::assertEquals(new Entries($integerEntry, $stringEntry), $entries);
+ static::assertSame(['integer-entry', 'string-entry'], $entries->names());
+ static::assertSame([$integerEntry, $stringEntry], $entries->all());
+ }
+
+ public function test_recreate_with_empty_array(): void
+ {
+ static::assertEquals(new Entries(), Entries::recreate([]));
+ }
+
public function test_remove_entry(): void
{
$entries = new Entries(
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SerializeTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SerializeTransformerTest.php
index a950b5c20a..343370d761 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SerializeTransformerTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SerializeTransformerTest.php
@@ -6,8 +6,8 @@
use Flow\ETL\Tests\FlowTestCase;
use Flow\ETL\Transformer\SerializeTransformer;
+use Flow\Floe\FloeSerializer;
use Flow\Serializer\Base64Serializer;
-use Flow\Serializer\NativePHPSerializer;
use function Flow\ETL\DSL\bool_entry;
use function Flow\ETL\DSL\flow_context;
@@ -31,7 +31,7 @@ public function test_serializing_empty_row_under_one_entry(): void
static::assertEquals(
[
[
- 'serialized' => (new Base64Serializer(new NativePHPSerializer()))->serialize($row1),
+ 'serialized' => (new Base64Serializer(new FloeSerializer()))->serialize($row1),
],
],
$transformedRows->toArray(),
@@ -66,14 +66,14 @@ public function test_serializing_row_under_one_entry(): void
'name' => 'John',
'active' => true,
'tags' => ['tag1', 'tag2'],
- 'serialized' => (new Base64Serializer(new NativePHPSerializer()))->serialize($row1),
+ 'serialized' => (new Base64Serializer(new FloeSerializer()))->serialize($row1),
],
[
'id' => 2,
'name' => 'Jane',
'active' => false,
'tags' => ['tag3', 'tag4'],
- 'serialized' => (new Base64Serializer(new NativePHPSerializer()))->serialize($row2),
+ 'serialized' => (new Base64Serializer(new FloeSerializer()))->serialize($row2),
],
],
$transformedRows->toArray(),
@@ -104,10 +104,10 @@ public function test_serializing_row_under_standalone_entry(): void
static::assertEquals(
[
[
- 'serialized' => (new Base64Serializer(new NativePHPSerializer()))->serialize($row1),
+ 'serialized' => (new Base64Serializer(new FloeSerializer()))->serialize($row1),
],
[
- 'serialized' => (new Base64Serializer(new NativePHPSerializer()))->serialize($row2),
+ 'serialized' => (new Base64Serializer(new FloeSerializer()))->serialize($row2),
],
],
$transformedRows->toArray(),
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UnserializeTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UnserializeTransformerTest.php
index 0b91474ca8..3cf16ad327 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UnserializeTransformerTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UnserializeTransformerTest.php
@@ -6,8 +6,8 @@
use Flow\ETL\Tests\FlowTestCase;
use Flow\ETL\Transformer\UnserializeTransformer;
+use Flow\Floe\FloeSerializer;
use Flow\Serializer\Base64Serializer;
-use Flow\Serializer\NativePHPSerializer;
use function Flow\ETL\DSL\bool_entry;
use function Flow\ETL\DSL\flow_context;
@@ -37,8 +37,8 @@ public function test_unserializing_row_from_entry(): void
);
$rows = rows(
- row(str_entry('serialized', (new Base64Serializer(new NativePHPSerializer()))->serialize($row1))),
- row(str_entry('serialized', (new Base64Serializer(new NativePHPSerializer()))->serialize($row2))),
+ row(str_entry('serialized', (new Base64Serializer(new FloeSerializer()))->serialize($row1))),
+ row(str_entry('serialized', (new Base64Serializer(new FloeSerializer()))->serialize($row2))),
);
$transformer = new UnserializeTransformer('serialized');
@@ -48,14 +48,14 @@ public function test_unserializing_row_from_entry(): void
static::assertEquals(
[
[
- 'serialized' => (new Base64Serializer(new NativePHPSerializer()))->serialize($row1),
+ 'serialized' => (new Base64Serializer(new FloeSerializer()))->serialize($row1),
'id' => 1,
'name' => 'John',
'active' => true,
'tags' => ['tag1', 'tag2'],
],
[
- 'serialized' => (new Base64Serializer(new NativePHPSerializer()))->serialize($row2),
+ 'serialized' => (new Base64Serializer(new FloeSerializer()))->serialize($row2),
'id' => 2,
'name' => 'Jane',
'active' => false,
@@ -93,8 +93,8 @@ public function test_unserializing_without_merge(): void
);
$rows = rows(
- row(str_entry('serialized', (new Base64Serializer(new NativePHPSerializer()))->serialize($row1))),
- row(str_entry('serialized', (new Base64Serializer(new NativePHPSerializer()))->serialize($row2))),
+ row(str_entry('serialized', (new Base64Serializer(new FloeSerializer()))->serialize($row1))),
+ row(str_entry('serialized', (new Base64Serializer(new FloeSerializer()))->serialize($row2))),
);
$transformer = new UnserializeTransformer('serialized', false);
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Context/FloeFileContext.php b/src/core/etl/tests/Flow/Floe/Tests/Context/FloeFileContext.php
new file mode 100644
index 0000000000..4d995c264b
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Context/FloeFileContext.php
@@ -0,0 +1,151 @@
+readFrom($path);
+ $size = (int) $source->size();
+ /** @var int<1, max> $footerLength */
+ $footerLength = Format::parseTrailer($source->read(Format::TRAILER_LENGTH, $size - Format::TRAILER_LENGTH));
+
+ return Footer::fromJson($source->read($footerLength, $size - Format::TRAILER_LENGTH - $footerLength));
+ }
+
+ /**
+ * @return array frame type and body, in file order
+ */
+ public static function frames(Filesystem $filesystem, Path $path): array
+ {
+ return iterator_to_array((new FrameReader($filesystem->readFrom($path), 0x00))->frames());
+ }
+
+ /**
+ * Writes rows to a fresh in-memory Floe file and reads them back as a
+ * single batch.
+ */
+ public static function roundTrip(Rows $rows): Rows
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://round-trip.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write($rows);
+ $writer->close();
+
+ $batches = iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(),
+ );
+
+ return $batches === [] ? rows() : $batches[0];
+ }
+
+ /**
+ * @return array frame types in file order
+ */
+ public static function frameTypes(Filesystem $filesystem, Path $path): array
+ {
+ $types = [];
+
+ foreach (self::frames($filesystem, $path) as [$type]) {
+ $types[] = $type;
+ }
+
+ return $types;
+ }
+
+ /**
+ * Reads a written Floe file back into the original Row|Rows the cache stored.
+ */
+ public static function reconstruct(Filesystem $filesystem, Path $path): Row|Rows
+ {
+ $file = (new FloeReader($filesystem))->read($path);
+ $rows = [];
+
+ foreach ($file->rows() as $batch) {
+ foreach ($batch->all() as $row) {
+ $rows[] = $row;
+ }
+ }
+
+ return RowsValueMapper::reconstructFrom($rows, $file->footer());
+ }
+
+ /**
+ * Reads every row of a Floe file merged into a single Rows.
+ */
+ public static function readAll(Filesystem $filesystem, Path $path): Rows
+ {
+ $merged = new Rows();
+
+ foreach ((new FloeReader($filesystem))
+ ->read($path)
+ ->rows() as $batch) {
+ $merged = $merged->merge($batch);
+ }
+
+ return $merged;
+ }
+
+ /**
+ * @param array $metadata
+ */
+ public static function write(Filesystem $filesystem, Path $path, Rows $rows, array $metadata = []): void
+ {
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path, Metadata::fromArray($metadata));
+ $writer->write($rows);
+ $writer->close();
+ }
+
+ /**
+ * Writes a complete file then rewrites it with the FOOTER frame stripped -
+ * a crashed writer that flushed complete frames but never closed. recover()
+ * salvages such a file; strict rows() throws on the missing trailer.
+ */
+ public static function writeWithoutFooter(Filesystem $filesystem, Path $path, Rows $rows): void
+ {
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write($rows);
+ $writer->close();
+
+ $content = $filesystem->readFrom($path)->content();
+ /** @var int<1, max> $footerLength */
+ $footerLength = Format::parseTrailer(substr($content, -Format::TRAILER_LENGTH));
+ // FOOTER frame = 5-byte frame header + footer JSON + 8-byte trailer.
+ $torn = substr($content, 0, strlen($content) - (5 + $footerLength + Format::TRAILER_LENGTH));
+
+ $filesystem->writeTo($path)->append($torn)->close();
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Double/CodecStub.php b/src/core/etl/tests/Flow/Floe/Tests/Double/CodecStub.php
new file mode 100644
index 0000000000..02a28c65ac
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Double/CodecStub.php
@@ -0,0 +1,29 @@
+id;
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Double/PrefixingCodecStub.php b/src/core/etl/tests/Flow/Floe/Tests/Double/PrefixingCodecStub.php
new file mode 100644
index 0000000000..0dfe8c415e
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Double/PrefixingCodecStub.php
@@ -0,0 +1,34 @@
+id;
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Double/UnsizedFilesystem.php b/src/core/etl/tests/Flow/Floe/Tests/Double/UnsizedFilesystem.php
new file mode 100644
index 0000000000..73c81dc118
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Double/UnsizedFilesystem.php
@@ -0,0 +1,67 @@
+filesystem->appendTo($path);
+ }
+
+ public function getSystemTmpDir(): Path
+ {
+ return $this->filesystem->getSystemTmpDir();
+ }
+
+ public function list(Path $path, Filter $pathFilter = new KeepAll()): Generator
+ {
+ return $this->filesystem->list($path, $pathFilter);
+ }
+
+ public function mount(): Mount
+ {
+ return $this->filesystem->mount();
+ }
+
+ public function mv(Path $from, Path $to): bool
+ {
+ return $this->filesystem->mv($from, $to);
+ }
+
+ public function readFrom(Path $path): SourceStream
+ {
+ return new UnsizedSourceStream($this->filesystem->readFrom($path));
+ }
+
+ public function rm(Path $path): bool
+ {
+ return $this->filesystem->rm($path);
+ }
+
+ public function status(Path $path): ?FileStatus
+ {
+ return $this->filesystem->status($path);
+ }
+
+ public function writeTo(Path $path): DestinationStream
+ {
+ return $this->filesystem->writeTo($path);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Double/UnsizedSourceStream.php b/src/core/etl/tests/Flow/Floe/Tests/Double/UnsizedSourceStream.php
new file mode 100644
index 0000000000..986cdd7c5e
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Double/UnsizedSourceStream.php
@@ -0,0 +1,56 @@
+stream->close();
+ }
+
+ public function content(): string
+ {
+ return $this->stream->content();
+ }
+
+ public function isOpen(): bool
+ {
+ return $this->stream->isOpen();
+ }
+
+ public function iterate(int $length = 1): Generator
+ {
+ return $this->stream->iterate($length);
+ }
+
+ public function path(): Path
+ {
+ return $this->stream->path();
+ }
+
+ public function read(int $length, int $offset): string
+ {
+ return $this->stream->read($length, $offset);
+ }
+
+ public function readLines(string $separator = "\n", ?int $length = null): Generator
+ {
+ return $this->stream->readLines($separator, $length);
+ }
+
+ public function size(): ?int
+ {
+ return null;
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeDataFrameTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeDataFrameTest.php
new file mode 100644
index 0000000000..0b7e9d6762
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeDataFrameTest.php
@@ -0,0 +1,123 @@
+cacheDir->path() . '/appended';
+
+ data_frame()
+ ->read(from_array([['id' => 1]]))
+ ->saveMode(overwrite())
+ ->write(to_floe($dir . '/data.floe'))
+ ->run();
+
+ data_frame()
+ ->read(from_array([['id' => 2]]))
+ ->saveMode(append())
+ ->write(to_floe($dir . '/data.floe'))
+ ->run();
+
+ static::assertSame(2, data_frame()->read(from_floe($dir . '/*.floe'))->count());
+ }
+
+ public function test_input_file_uri_is_added_when_configured(): void
+ {
+ $path = $this->cacheDir->suffix('input-uri.floe');
+
+ data_frame()
+ ->read(from_array([['id' => 1]]))
+ ->saveMode(overwrite())
+ ->write(to_floe($path))
+ ->run();
+
+ $rows = data_frame(config_builder()->putInputIntoRows())->read(from_floe($path))->fetch();
+
+ static::assertTrue($rows->first()->entries()->has('_input_file_uri'));
+ }
+
+ public function test_offset_and_limit_pushdown(): void
+ {
+ $path = $this->cacheDir->suffix('pushdown.floe');
+
+ data_frame()
+ ->read(from_array([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4], ['id' => 5]]))
+ ->saveMode(overwrite())
+ ->write(to_floe($path))
+ ->run();
+
+ static::assertSame(2, data_frame()->read(from_floe($path))->limit(2)->fetch()->count());
+ static::assertSame(2, data_frame()->read(from_floe($path)->withOffset(3))->fetch()->count());
+ }
+
+ public function test_overwrite_replaces_the_dataset(): void
+ {
+ $path = $this->cacheDir->suffix('overwrite.floe');
+
+ data_frame()
+ ->read(from_array([['id' => 1], ['id' => 2], ['id' => 3]]))
+ ->saveMode(overwrite())
+ ->write(to_floe($path))
+ ->run();
+
+ data_frame()
+ ->read(from_array([['id' => 9]]))
+ ->saveMode(overwrite())
+ ->write(to_floe($path))
+ ->run();
+
+ static::assertSame(1, data_frame()->read(from_floe($path))->count());
+ }
+
+ public function test_partitioned_round_trip_with_pruning(): void
+ {
+ $dir = $this->cacheDir->path() . '/parts';
+
+ data_frame()
+ ->read(from_array([
+ ['id' => 1, 'country' => 'PL'],
+ ['id' => 2, 'country' => 'US'],
+ ['id' => 3, 'country' => 'PL'],
+ ]))
+ ->partitionBy('country')
+ ->saveMode(overwrite())
+ ->write(to_floe($dir . '/data.floe'))
+ ->run();
+
+ static::assertFileExists($dir . '/country=PL/data.floe');
+ static::assertFileExists($dir . '/country=US/data.floe');
+
+ static::assertSame(3, data_frame()->read(from_floe($dir . '/country=*/data.floe'))->count());
+ static::assertSame(2, data_frame()->read(from_floe($dir . '/country=PL/data.floe'))->count());
+ }
+
+ public function test_round_trip(): void
+ {
+ $path = $this->cacheDir->suffix('roundtrip.floe');
+
+ data_frame()
+ ->read(from_array([['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => 'b']]))
+ ->saveMode(overwrite())
+ ->write(to_floe($path))
+ ->run();
+
+ $result = data_frame()->read(from_floe($path))->fetch();
+
+ static::assertSame(2, $result->count());
+ static::assertSame(['id', 'name'], $result->first()->entries()->names());
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeFileTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeFileTest.php
new file mode 100644
index 0000000000..afbd7a4a13
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeFileTest.php
@@ -0,0 +1,158 @@
+cacheDir->suffix('evolving.floe');
+
+ $writer = new FloeWriter($this->fs());
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1))));
+ $writer->close();
+
+ $writer = new FloeWriter($this->fs());
+ $writer->append($path);
+ $writer->write(rows(row(int_entry('id', 2), str_entry('email', null))));
+ $writer->close();
+
+ $writer = new FloeWriter($this->fs());
+ $writer->append($path);
+ $writer->write(rows(row(int_entry('id', 3), str_entry('email', 'third@flow.php'))));
+ $writer->close();
+
+ $reader = (new FloeReader($this->fs()))->read($path);
+
+ static::assertSame(3, $reader->totalRows());
+
+ $read = [];
+
+ foreach ($reader->rows() as $batch) {
+ foreach ($batch->all() as $row) {
+ static::assertSame(['id', 'email'], $row->entries()->names());
+ $read[] = [$row->valueOf('id'), $row->valueOf('email')];
+ }
+ }
+
+ static::assertSame([[1, null], [2, null], [3, 'third@flow.php']], $read);
+ }
+
+ public function test_file_larger_than_compaction_threshold_round_trips(): void
+ {
+ $path = $this->cacheDir->suffix('large.floe');
+
+ $writer = new FloeWriter($this->fs());
+ $writer->create($path);
+ $written = 0;
+
+ for ($i = 0; $i < 4000; $i++) {
+ $writer->write(rows(row(int_entry('id', $i), str_entry('payload', str_pad('row_' . $i, 300, 'x')))));
+ $written++;
+ }
+
+ $writer->close();
+
+ // The file must cross FrameReader's 1 MiB buffer-compaction threshold,
+ // otherwise this test silently stops covering it.
+ static::assertGreaterThan(1_048_576, $this->fs()->readFrom($path)->size());
+
+ $read = 0;
+
+ foreach ((new FloeReader($this->fs(), chunkSize: 4096))
+ ->read($path)
+ ->rows(500) as $batch) {
+ foreach ($batch->all() as $row) {
+ static::assertSame($read, $row->valueOf('id'));
+ $read++;
+ }
+ }
+
+ static::assertSame($written, $read);
+ }
+
+ public function test_recover_salvages_truncated_copy_of_real_file(): void
+ {
+ $path = $this->cacheDir->suffix('original.floe');
+
+ $writer = new FloeWriter($this->fs());
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1)), row(int_entry('id', 2)), row(int_entry('id', 3))));
+ $writer->close();
+
+ $content = $this->fs()->readFrom($path)->content();
+ $truncatedPath = $this->cacheDir->suffix('truncated.floe');
+ $stream = $this->fs()->writeTo($truncatedPath);
+ $stream->append(substr($content, 0, strlen($content) - 250));
+ $stream->close();
+
+ $salvaged = 0;
+
+ foreach ((new FloeReader($this->fs()))
+ ->read($truncatedPath)
+ ->recover() as $batch) {
+ $salvaged += $batch->count();
+ }
+
+ static::assertGreaterThan(0, $salvaged);
+ static::assertLessThanOrEqual(3, $salvaged);
+ }
+
+ public function test_round_trip_of_all_entry_types_through_local_filesystem(): void
+ {
+ $rows = RowsMother::withAllEntryTypes();
+ $path = $this->cacheDir->suffix('all-types.floe');
+
+ $writer = new FloeWriter($this->fs());
+ $writer->create($path);
+ $writer->write($rows);
+ $writer->close();
+
+ $batches = iterator_to_array(
+ (new FloeReader($this->fs()))
+ ->read($path)
+ ->rows(),
+ );
+
+ static::assertCount(1, $batches);
+ static::assertEquals($rows->all(), $batches[0]->all());
+ }
+
+ public function test_round_trip_of_heterogeneous_rows_through_local_filesystem(): void
+ {
+ $rows = RowsMother::heterogeneous();
+ $path = $this->cacheDir->suffix('heterogeneous.floe');
+
+ $writer = new FloeWriter($this->fs());
+ $writer->create($path);
+ $writer->write($rows);
+ $writer->close();
+
+ $read = 0;
+
+ foreach ((new FloeReader($this->fs()))
+ ->read($path)
+ ->rows() as $batch) {
+ $read += $batch->count();
+ }
+
+ static::assertSame($rows->count(), $read);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeMergeDSLTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeMergeDSLTest.php
new file mode 100644
index 0000000000..44174458ff
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeMergeDSLTest.php
@@ -0,0 +1,50 @@
+cacheDir->suffix('mc-a.floe');
+ $b = $this->cacheDir->suffix('mc-b.floe');
+ $out = $this->cacheDir->suffix('mc-out.floe');
+
+ FloeFileContext::write($this->fs(), $a, rows(row(int_entry('id', 1))));
+ FloeFileContext::write($this->fs(), $b, rows(row(int_entry('id', 2))));
+
+ merge_floe([$a, $b], $out, compact: true);
+
+ static::assertEquals(
+ rows(row(int_entry('id', 1)), row(int_entry('id', 2))),
+ FloeFileContext::readAll($this->fs(), $out),
+ );
+ }
+
+ public function test_merge_floe_splices_local_files_from_string_paths(): void
+ {
+ $a = $this->cacheDir->suffix('ms-a.floe');
+ $b = $this->cacheDir->suffix('ms-b.floe');
+ $out = $this->cacheDir->suffix('ms-out.floe');
+
+ FloeFileContext::write($this->fs(), $a, rows(row(int_entry('id', 1))));
+ FloeFileContext::write($this->fs(), $b, rows(row(int_entry('id', 2))));
+
+ merge_floe([$a->path(), $b->path()], $out->path());
+
+ static::assertEquals(
+ rows(row(int_entry('id', 1)), row(int_entry('id', 2))),
+ FloeFileContext::readAll($this->fs(), $out),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeReaderExtensionParityTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeReaderExtensionParityTest.php
new file mode 100644
index 0000000000..f72729befd
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeReaderExtensionParityTest.php
@@ -0,0 +1,253 @@
+ [RowsMother::withAllEntryTypes()],
+ 'heterogeneous' => [RowsMother::heterogeneous()],
+ 'partitioned' => [RowsMother::partitioned()],
+ 'empty' => [rows()],
+ ];
+ }
+
+ #[Override]
+ protected function setUp(): void
+ {
+ parent::setUp();
+
+ if (!extension_loaded('flow_php')) {
+ self::markTestSkipped('flow_php extension is not loaded.');
+ }
+ }
+
+ #[DataProvider('rows_datasets')]
+ public function test_extension_and_pure_php_hydrate_identical_rows(Rows $rows): void
+ {
+ $path = $this->cacheDir->suffix('parity.floe');
+
+ $writer = new FloeWriter($this->fs());
+ $writer->create($path);
+ $writer->write($rows);
+ $writer->close();
+
+ static::assertEquals(
+ iterator_to_array(
+ (new FloeReader($this->fs(), useExtension: false))
+ ->read($path)
+ ->rows(),
+ ),
+ iterator_to_array(
+ (new FloeReader($this->fs(), useExtension: true))
+ ->read($path)
+ ->rows(),
+ ),
+ );
+ }
+
+ public function test_extension_and_pure_php_seek_offset_identically(): void
+ {
+ $path = $this->cacheDir->suffix('parity-offset.floe');
+
+ $writer = new FloeWriter($this->fs());
+ $writer->create($path);
+ $writer->write(rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2)),
+ row(int_entry('id', 3)),
+ row(int_entry('id', 4), str_entry('email', null)),
+ row(int_entry('id', 5), str_entry('email', 'x')),
+ row(int_entry('id', 6)),
+ ));
+ $writer->close();
+
+ static::assertEquals(
+ iterator_to_array(
+ (new FloeReader($this->fs(), useExtension: false))
+ ->read($path)
+ ->rows(1000, 2, 3),
+ ),
+ iterator_to_array(
+ (new FloeReader($this->fs(), useExtension: true))
+ ->read($path)
+ ->rows(1000, 2, 3),
+ ),
+ );
+ }
+
+ #[DataProvider('rows_datasets')]
+ public function test_extension_and_pure_php_recover_identical_rows(Rows $rows): void
+ {
+ $path = $this->cacheDir->suffix('parity-recover.floe');
+
+ $writer = new FloeWriter($this->fs());
+ $writer->create($path);
+ $writer->write($rows);
+ $writer->close();
+
+ static::assertEquals(
+ iterator_to_array(
+ (new FloeReader($this->fs(), useExtension: false))
+ ->read($path)
+ ->recover(),
+ ),
+ iterator_to_array(
+ (new FloeReader($this->fs(), useExtension: true))
+ ->read($path)
+ ->recover(),
+ ),
+ );
+ }
+
+ public function test_extension_and_pure_php_recover_a_torn_file_identically(): void
+ {
+ $path = $this->cacheDir->suffix('parity-recover-torn.floe');
+
+ FloeFileContext::writeWithoutFooter(
+ $this->fs(),
+ $path,
+ rows(row(int_entry('id', 1)), row(int_entry('id', 2), str_entry('email', 'x')), row(int_entry('id', 3))),
+ );
+
+ static::assertEquals(
+ iterator_to_array(
+ (new FloeReader($this->fs(), useExtension: false))
+ ->read($path)
+ ->recover(),
+ ),
+ iterator_to_array(
+ (new FloeReader($this->fs(), useExtension: true))
+ ->read($path)
+ ->recover(),
+ ),
+ );
+ }
+
+ public function test_extension_and_pure_php_salvage_rows_before_a_corrupt_row_identically(): void
+ {
+ $path = $this->cacheDir->suffix('parity-recover-corrupt.floe');
+
+ $plan = FloeWriter::growSectionPlan(null, row(int_entry('id', 1)));
+ $rowEncoder = new RowEncoder();
+
+ $stream = $this->fs()->writeTo($path);
+ $stream->append(
+ Format::header(0x00)
+ . Format::frame(Format::FRAME_SCHEMA, $plan->schemaBody)
+ . Format::frame(Format::FRAME_ROW, $rowEncoder->encode($plan, row(int_entry('id', 1))))
+ . Format::frame(Format::FRAME_ROW, $rowEncoder->encode($plan, row(int_entry('id', 2))))
+ . Format::frame(Format::FRAME_ROW, "\xEE"),
+ );
+ $stream->close();
+
+ $pure = iterator_to_array(
+ (new FloeReader($this->fs(), useExtension: false))
+ ->read($path)
+ ->recover(),
+ );
+
+ static::assertEquals(
+ $pure,
+ iterator_to_array(
+ (new FloeReader($this->fs(), useExtension: true))
+ ->read($path)
+ ->recover(),
+ ),
+ );
+ static::assertCount(1, $pure);
+ static::assertCount(2, $pure[0]->all());
+ }
+
+ public function test_extension_and_pure_php_recover_rows_around_a_late_partitions_frame_identically(): void
+ {
+ $path = $this->cacheDir->suffix('parity-recover-late-partitions.floe');
+
+ $plan = FloeWriter::growSectionPlan(null, row(int_entry('id', 1)));
+ $rowEncoder = new RowEncoder();
+ $partitionsBody = pack('V', 1) . pack('V', 1) . 'g' . pack('V', 1) . 'a';
+
+ $stream = $this->fs()->writeTo($path);
+ $stream->append(
+ Format::header(0x00)
+ . Format::frame(Format::FRAME_SCHEMA, $plan->schemaBody)
+ . Format::frame(Format::FRAME_ROW, $rowEncoder->encode($plan, row(int_entry('id', 1))))
+ . Format::frame(Format::FRAME_PARTITIONS, $partitionsBody)
+ . Format::frame(Format::FRAME_ROW, $rowEncoder->encode($plan, row(int_entry('id', 2)))),
+ );
+ $stream->close();
+
+ $pure = iterator_to_array(
+ (new FloeReader($this->fs(), useExtension: false))
+ ->read($path)
+ ->recover(),
+ );
+
+ static::assertEquals(
+ $pure,
+ iterator_to_array(
+ (new FloeReader($this->fs(), useExtension: true))
+ ->read($path)
+ ->recover(),
+ ),
+ );
+ static::assertCount(1, $pure);
+ static::assertCount(2, $pure[0]->all());
+ }
+
+ public function test_extension_and_pure_php_pad_evolved_sections_identically(): void
+ {
+ $path = $this->cacheDir->suffix('parity-evolved.floe');
+
+ $writer = new FloeWriter($this->fs());
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1))));
+ $writer->close();
+
+ $writer = new FloeWriter($this->fs());
+ $writer->append($path);
+ $writer->write(rows(row(int_entry('id', 2), str_entry('email', null))));
+ $writer->close();
+
+ static::assertEquals(
+ iterator_to_array(
+ (new FloeReader($this->fs(), useExtension: false))
+ ->read($path)
+ ->rows(),
+ ),
+ iterator_to_array(
+ (new FloeReader($this->fs(), useExtension: true))
+ ->read($path)
+ ->rows(),
+ ),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeSerializerExtensionParityTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeSerializerExtensionParityTest.php
new file mode 100644
index 0000000000..aa890b3227
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeSerializerExtensionParityTest.php
@@ -0,0 +1,58 @@
+ [RowsMother::withAllEntryTypes()],
+ 'heterogeneous' => [RowsMother::heterogeneous()],
+ 'partitioned' => [RowsMother::partitioned()],
+ 'empty' => [rows()],
+ 'single row' => [row(int_entry('id', 1))],
+ ];
+ }
+
+ #[Override]
+ protected function setUp(): void
+ {
+ parent::setUp();
+
+ if (!extension_loaded('flow_php')) {
+ self::markTestSkipped('flow_php extension is not loaded.');
+ }
+ }
+
+ #[DataProvider('value_datasets')]
+ public function test_extension_and_pure_php_unserialize_identical_value(Row|Rows $value): void
+ {
+ $serialized = (new FloeSerializer())->serialize($value);
+
+ static::assertEquals(
+ (new FloeSerializer(useExtension: false))->unserialize($serialized, [Row::class, Rows::class]),
+ (new FloeSerializer(useExtension: true))->unserialize($serialized, [Row::class, Rows::class]),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeValueSerializerExtensionParityTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeValueSerializerExtensionParityTest.php
new file mode 100644
index 0000000000..ee4a3469b0
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeValueSerializerExtensionParityTest.php
@@ -0,0 +1,111 @@
+ [RowsMother::withAllEntryTypes()],
+ 'heterogeneous' => [RowsMother::heterogeneous()],
+ 'partitioned' => [RowsMother::partitioned()],
+ 'empty' => [rows()],
+ 'single row' => [row(int_entry('id', 1))],
+ ];
+ }
+
+ #[Override]
+ protected function setUp(): void
+ {
+ parent::setUp();
+
+ if (!extension_loaded('flow_php')) {
+ self::markTestSkipped('flow_php extension is not loaded.');
+ }
+ }
+
+ #[DataProvider('value_datasets')]
+ public function test_extension_and_pure_php_decode_identically(Row|Rows $value): void
+ {
+ $bytes = (new FloeValueSerializer(useExtension: false))->encode($value);
+
+ static::assertEquals(
+ (new FloeValueSerializer(useExtension: false))->decode($bytes),
+ (new FloeValueSerializer(useExtension: true))->decode($bytes),
+ );
+ }
+
+ #[DataProvider('value_datasets')]
+ public function test_extension_encodes_byte_identical_to_pure_php(Row|Rows $value): void
+ {
+ static::assertSame(
+ (new FloeValueSerializer(useExtension: false))->encode($value),
+ (new FloeValueSerializer(useExtension: true))->encode($value),
+ );
+ }
+
+ public function test_extension_decode_failure_is_wrapped_as_floe_exception(): void
+ {
+ $bytes = (new FloeValueSerializer(useExtension: false))->encode(rows(row(int_entry('id', 1))));
+ // corrupt the first frame type (SCHEMA -> unknown 0x7F); the footer at the end stays intact
+ $bytes[6] = "\x7F";
+
+ $this->expectException(FloeException::class);
+
+ (new FloeValueSerializer(useExtension: true))->decode($bytes);
+ }
+
+ public function test_extension_encode_failure_is_wrapped_as_floe_exception(): void
+ {
+ $this->expectException(FloeException::class);
+
+ (new FloeValueSerializer(useExtension: true))->encode(rows(row(datetime_entry(
+ 'd',
+ new CustomDateTime('2025-01-01 00:00:00'),
+ ))));
+ }
+
+ #[DataProvider('value_datasets')]
+ public function test_streaming_mode_decodes_identically_on_both_engines(Row|Rows $value): void
+ {
+ $bytes = (new FloeValueSerializer(useExtension: false))->encode($value);
+
+ static::assertEquals(
+ (new FloeValueSerializer(2, useExtension: false))->decode($bytes),
+ (new FloeValueSerializer(2, useExtension: true))->decode($bytes),
+ );
+ }
+
+ #[DataProvider('value_datasets')]
+ public function test_streaming_mode_encodes_byte_identical_on_both_engines_and_to_bulk(Row|Rows $value): void
+ {
+ $bulk = (new FloeValueSerializer(useExtension: true))->encode($value);
+
+ static::assertSame($bulk, (new FloeValueSerializer(2, useExtension: false))->encode($value));
+ static::assertSame($bulk, (new FloeValueSerializer(2, useExtension: true))->encode($value));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeWriterExtensionParityTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeWriterExtensionParityTest.php
new file mode 100644
index 0000000000..26127cc1c7
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeWriterExtensionParityTest.php
@@ -0,0 +1,130 @@
+ [RowsMother::withAllEntryTypes()],
+ 'heterogeneous' => [RowsMother::heterogeneous()],
+ 'partitioned' => [RowsMother::partitioned()],
+ 'empty' => [rows()],
+ ];
+ }
+
+ #[Override]
+ protected function setUp(): void
+ {
+ parent::setUp();
+
+ if (!extension_loaded('flow_php')) {
+ self::markTestSkipped('flow_php extension is not loaded.');
+ }
+ }
+
+ public function test_extension_and_pure_php_reject_datetime_subclass_identically(): void
+ {
+ $rows = rows(row(datetime_entry('at', new CustomDateTime('2025-01-01 00:00:00 UTC'))));
+
+ foreach ([false, true] as $useExtension) {
+ $writer = new FloeWriter($this->fs(), useExtension: $useExtension);
+ $writer->create($this->cacheDir->suffix('subclass-' . ($useExtension ? 'ext' : 'php') . '.floe'));
+
+ try {
+ $writer->write($rows);
+ static::fail('Expected a FloeException for a custom datetime subclass');
+ } catch (FloeException $e) {
+ static::assertStringContainsString('DateTime and DateTimeImmutable', $e->getMessage());
+ }
+ }
+ }
+
+ #[DataProvider('rows_datasets')]
+ public function test_extension_and_pure_php_write_byte_identical_files(Rows $rows): void
+ {
+ $purePath = $this->cacheDir->suffix('write-pure.floe');
+ $extPath = $this->cacheDir->suffix('write-ext.floe');
+
+ $pure = new FloeWriter($this->fs(), useExtension: false);
+ $pure->create($purePath);
+ $pure->write($rows);
+ $pure->close();
+
+ $ext = new FloeWriter($this->fs(), useExtension: true);
+ $ext->create($extPath);
+ $ext->write($rows);
+ $ext->close();
+
+ static::assertSame($this->fs()->readFrom($purePath)->content(), $this->fs()->readFrom($extPath)->content());
+ }
+
+ public function test_extension_and_pure_php_write_byte_identical_files_across_multiple_writes(): void
+ {
+ $batches = [
+ rows(row(int_entry('id', 1)), row(int_entry('id', 2))),
+ rows(row(int_entry('id', 3))),
+ rows(row(int_entry('id', 4), str_entry('name', 'x')), row(int_entry('id', 5))),
+ ];
+
+ $purePath = $this->cacheDir->suffix('multi-write-pure.floe');
+ $extPath = $this->cacheDir->suffix('multi-write-ext.floe');
+
+ foreach ([[$purePath, false], [$extPath, true]] as [$path, $useExtension]) {
+ $writer = new FloeWriter($this->fs(), useExtension: $useExtension);
+ $writer->create($path);
+
+ foreach ($batches as $batch) {
+ $writer->write($batch);
+ }
+
+ $writer->close();
+ }
+
+ static::assertSame($this->fs()->readFrom($purePath)->content(), $this->fs()->readFrom($extPath)->content());
+ }
+
+ public function test_extension_and_pure_php_append_byte_identically(): void
+ {
+ $purePath = $this->cacheDir->suffix('append-pure.floe');
+ $extPath = $this->cacheDir->suffix('append-ext.floe');
+
+ foreach ([[$purePath, false], [$extPath, true]] as [$path, $useExtension]) {
+ $writer = new FloeWriter($this->fs(), useExtension: $useExtension);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1), str_entry('email', null))));
+ $writer->close();
+
+ $writer = new FloeWriter($this->fs(), useExtension: $useExtension);
+ $writer->append($path);
+ $writer->write(rows(row(int_entry('id', 2), str_entry('email', 'x'))));
+ $writer->close();
+ }
+
+ static::assertSame($this->fs()->readFrom($purePath)->content(), $this->fs()->readFrom($extPath)->content());
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Mother/DateTimeDecoderMother.php b/src/core/etl/tests/Flow/Floe/Tests/Mother/DateTimeDecoderMother.php
new file mode 100644
index 0000000000..6faf04d9e2
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Mother/DateTimeDecoderMother.php
@@ -0,0 +1,16 @@
+ $schemas raw normalized schema lists, e.g. [$schema->normalize()]
+ * @param array $fileSchema raw normalized schema, e.g. $schema->normalize()
+ * @param array $sections
+ * @param array $partitions
+ * @param array|bool|float|int|string> $metadata
+ */
+ public static function footer(
+ array $schemas = [],
+ array $fileSchema = [],
+ array $sections = [],
+ array $partitions = [],
+ int $totalRows = 0,
+ array $metadata = [],
+ ): Footer {
+ /**
+ * @var array>> $schemas
+ * @var array> $fileSchema
+ */
+ return new Footer(
+ 1,
+ 'test-writer',
+ $schemas,
+ $fileSchema,
+ $sections,
+ $partitions,
+ $totalRows,
+ Metadata::fromArray($metadata),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Mother/FrameReaderMother.php b/src/core/etl/tests/Flow/Floe/Tests/Mother/FrameReaderMother.php
new file mode 100644
index 0000000000..b3858ef341
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Mother/FrameReaderMother.php
@@ -0,0 +1,23 @@
+writeTo(path('memory://frames.floe'));
+ $stream->append($bytes);
+ $stream->close();
+
+ return new FrameReader($filesystem->readFrom(path('memory://frames.floe')), $expectedFlags, $chunkSize);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Mother/RowsMother.php b/src/core/etl/tests/Flow/Floe/Tests/Mother/RowsMother.php
new file mode 100644
index 0000000000..3422baf533
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Mother/RowsMother.php
@@ -0,0 +1,111 @@
+ 1, 'b' => 2], type_map(type_string(), type_integer())),
+ structure_entry(
+ 'structure',
+ ['street' => 'Main', 'nested' => ['count' => 5, 'tags' => ['a']]],
+ type_structure([
+ 'street' => type_string(),
+ 'nested' => type_structure(['count' => type_integer(), 'tags' => type_list(type_string())]),
+ ]),
+ ),
+ xml_entry('xml', 'text & entity'),
+ xml_element_entry('xml_element', '- value
'),
+ ));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Codec/NoopCodecTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Codec/NoopCodecTest.php
new file mode 100644
index 0000000000..cb68dd32fd
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Codec/NoopCodecTest.php
@@ -0,0 +1,26 @@
+decode("raw\x00bytes"));
+ }
+
+ public function test_encode_is_passthrough(): void
+ {
+ static::assertSame("raw\x00bytes", (new NoopCodec())->encode("raw\x00bytes"));
+ }
+
+ public function test_id_is_zero(): void
+ {
+ static::assertSame(0x00, (new NoopCodec())->id());
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/BooleanDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/BooleanDecoderTest.php
new file mode 100644
index 0000000000..a324590d1f
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/BooleanDecoderTest.php
@@ -0,0 +1,21 @@
+decode("\x01\x00", $position));
+ static::assertFalse($decoder->decode("\x01\x00", $position));
+ static::assertSame(2, $position);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/DateTimeDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/DateTimeDecoderTest.php
new file mode 100644
index 0000000000..be92b92f04
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/DateTimeDecoderTest.php
@@ -0,0 +1,68 @@
+encode($value);
+ $position = 0;
+
+ $decoded = DateTimeDecoderMother::create()->decode($encoded, $position);
+
+ static::assertInstanceOf(DateTimeImmutable::class, $decoded);
+ static::assertEquals($value, $decoded);
+ static::assertSame('Australia/Eucla', $decoded->getTimezone()->getName());
+ static::assertSame('987654', $decoded->format('u'));
+ static::assertSame(strlen($encoded), $position);
+ }
+
+ public function test_round_trip_of_mutable_datetime(): void
+ {
+ $value = new DateTime('2025-01-01 00:00:00 UTC');
+ $position = 0;
+
+ static::assertInstanceOf(DateTime::class, DateTimeDecoderMother::create()->decode(
+ (new DateTimeEncoder())->encode($value),
+ $position,
+ ));
+ }
+
+ public function test_round_trip_before_epoch(): void
+ {
+ $value = new DateTimeImmutable('1969-07-20 20:17:00.500000 UTC');
+ $position = 0;
+
+ static::assertEquals($value, DateTimeDecoderMother::create()->decode(
+ (new DateTimeEncoder())->encode($value),
+ $position,
+ ));
+ }
+
+ public function test_unknown_class_flag_throws(): void
+ {
+ $encoded = chr(0x02) . pack('V', 11) . 'NoSuchClass' . pack('P', 0) . pack('V', 0) . pack('V', 3) . 'UTC';
+ $position = 0;
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('Floe found unknown datetime flag 0x02');
+
+ DateTimeDecoderMother::create()->decode($encoded, $position);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/DynamicDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/DynamicDecoderTest.php
new file mode 100644
index 0000000000..8e08560595
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/DynamicDecoderTest.php
@@ -0,0 +1,71 @@
+ null,
+ 'int' => 42,
+ 'float' => 3.5,
+ 'bool' => false,
+ 'string' => 'text',
+ 'array' => ['a' => 1, 0 => 'zero', 'nested' => ['x' => [1, 2]]],
+ ];
+
+ foreach ($values as $label => $value) {
+ $position = 0;
+
+ static::assertSame($value, $decoder->decode($encoder->encode($value), $position), $label);
+ }
+ }
+
+ public function test_round_trip_of_tagged_objects(): void
+ {
+ $encoder = new DynamicEncoder();
+ $decoder = DynamicDecoderMother::create();
+
+ $position = 0;
+ $datetime = new DateTimeImmutable('2025-01-01 00:00:00.123456 UTC');
+ static::assertEquals($datetime, $decoder->decode($encoder->encode($datetime), $position));
+
+ $position = 0;
+ $uuid = new Uuid('0196aecb-b568-7e57-a381-8ec8d3e4a531');
+ // @mago-ignore analysis:mixed-assignment
+ $decodedUuid = $decoder->decode($encoder->encode($uuid), $position);
+ static::assertInstanceOf(Uuid::class, $decodedUuid);
+ static::assertSame($uuid->toString(), $decodedUuid->toString());
+
+ $position = 0;
+ $json = new Json('{"a":true}');
+ // @mago-ignore analysis:mixed-assignment
+ $decodedJson = $decoder->decode($encoder->encode($json), $position);
+ static::assertInstanceOf(Json::class, $decodedJson);
+ static::assertSame($json->toString(), $decodedJson->toString());
+ }
+
+ public function test_unknown_tag_throws(): void
+ {
+ $position = 0;
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('unknown dynamic value tag');
+
+ DynamicDecoderMother::create()->decode("\xEE", $position);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/EnumDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/EnumDecoderTest.php
new file mode 100644
index 0000000000..29f645da80
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/EnumDecoderTest.php
@@ -0,0 +1,50 @@
+decode(
+ (new EnumEncoder())->encode(BackedStringEnum::one),
+ $position,
+ ));
+ }
+
+ public function test_unknown_case_throws(): void
+ {
+ $class = BackedStringEnum::class;
+ $encoded = pack('V', strlen($class)) . $class . pack('V', 4) . 'nope';
+ $position = 0;
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('cannot restore enum case');
+
+ (new EnumDecoder())->decode($encoded, $position);
+ }
+
+ public function test_unknown_class_throws(): void
+ {
+ $encoded = pack('V', 10) . 'NoSuchEnum' . pack('V', 3) . 'one';
+ $position = 0;
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('enum not found');
+
+ (new EnumDecoder())->decode($encoded, $position);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/Float64DecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/Float64DecoderTest.php
new file mode 100644
index 0000000000..947ac6de25
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/Float64DecoderTest.php
@@ -0,0 +1,21 @@
+decode(pack('e', 3.5), $position));
+ static::assertSame(8, $position);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/HtmlDocumentDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/HtmlDocumentDecoderTest.php
new file mode 100644
index 0000000000..ba75c519d2
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/HtmlDocumentDecoderTest.php
@@ -0,0 +1,32 @@
+encode(ValueDecoder::htmlDocumentFromString('x
'));
+ $position = 0;
+
+ $decoded = (new HtmlDocumentDecoder())->decode($encoded, $position);
+
+ static::assertSame($encoded, $encoder->encode($decoded));
+ static::assertSame(strlen($encoded), $position);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/HtmlElementDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/HtmlElementDecoderTest.php
new file mode 100644
index 0000000000..6c6bfde5a4
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/HtmlElementDecoderTest.php
@@ -0,0 +1,30 @@
+x
');
+ $position = 0;
+
+ $decoded = (new HtmlElementDecoder())->decode((new HtmlElementEncoder())->encode($element), $position);
+
+ static::assertSame(ValueEncoder::htmlElementToString($element), ValueEncoder::htmlElementToString($decoded));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/Int64DecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/Int64DecoderTest.php
new file mode 100644
index 0000000000..d69cfc92ab
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/Int64DecoderTest.php
@@ -0,0 +1,28 @@
+decode(pack('P', $value), $position));
+ static::assertSame(8, $position);
+ }
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/IntervalDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/IntervalDecoderTest.php
new file mode 100644
index 0000000000..9daad8038b
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/IntervalDecoderTest.php
@@ -0,0 +1,34 @@
+invert = 1;
+ // @mago-ignore analysis:invalid-property-write
+ $value->f = 0.5;
+
+ $position = 0;
+ $decoded = (new IntervalDecoder())->decode((new IntervalEncoder())->encode($value), $position);
+
+ static::assertSame([1, 2, 3, 4, 0.5, 1], [
+ $decoded->d,
+ $decoded->h,
+ $decoded->i,
+ $decoded->s,
+ $decoded->f,
+ $decoded->invert,
+ ]);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/JsonDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/JsonDecoderTest.php
new file mode 100644
index 0000000000..ad9c3f63f1
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/JsonDecoderTest.php
@@ -0,0 +1,26 @@
+decode((new JsonEncoder())->encode($value), $position);
+
+ static::assertSame($value->toString(), $decoded->toString());
+ static::assertSame($value->isObject(), $decoded->isObject());
+ }
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/ListDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/ListDecoderTest.php
new file mode 100644
index 0000000000..e06d8b8501
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/ListDecoderTest.php
@@ -0,0 +1,25 @@
+decode((new ListEncoder(new StringEncoder()))->encode(['a', 'bc']), $position),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/MapDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/MapDecoderTest.php
new file mode 100644
index 0000000000..d192171d5e
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/MapDecoderTest.php
@@ -0,0 +1,42 @@
+ 1, 'b' => 2],
+ $decoder->decode((new MapEncoder(new StringKeyEncoder(), new Int64Encoder()))->encode([
+ 'a' => 1,
+ 'b' => 2,
+ ]), $position),
+ );
+ }
+
+ public function test_round_trip_of_integer_keyed_map(): void
+ {
+ $decoder = new MapDecoder(new Int64Decoder(), new StringDecoder());
+ $position = 0;
+
+ static::assertSame(
+ [7 => 'x'],
+ $decoder->decode((new MapEncoder(new Int64Encoder(), new StringEncoder()))->encode([7 => 'x']), $position),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/NullDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/NullDecoderTest.php
new file mode 100644
index 0000000000..2ce8d77948
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/NullDecoderTest.php
@@ -0,0 +1,19 @@
+decode('', $position));
+ static::assertSame(0, $position);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/OptionalDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/OptionalDecoderTest.php
new file mode 100644
index 0000000000..66e7405167
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/OptionalDecoderTest.php
@@ -0,0 +1,26 @@
+decode($encoder->encode($value), $position));
+ }
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/PackedListDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/PackedListDecoderTest.php
new file mode 100644
index 0000000000..bd1ffcaad2
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/PackedListDecoderTest.php
@@ -0,0 +1,43 @@
+decode((new PackedListEncoder('P'))->encode([10, -20, 30]), $position),
+ );
+ }
+
+ public function test_round_trip_of_float64_list(): void
+ {
+ $position = 0;
+
+ static::assertSame(
+ [1.5, -2.5],
+ (new PackedListDecoder('e'))->decode((new PackedListEncoder('e'))->encode([1.5, -2.5]), $position),
+ );
+ }
+
+ public function test_empty_list(): void
+ {
+ $position = 0;
+
+ static::assertSame(
+ [],
+ (new PackedListDecoder('P'))->decode((new PackedListEncoder('P'))->encode([]), $position),
+ );
+ static::assertSame(4, $position);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/StringDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/StringDecoderTest.php
new file mode 100644
index 0000000000..a8cf3c0b22
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/StringDecoderTest.php
@@ -0,0 +1,26 @@
+decode(pack('V', 3) . 'abc', $position));
+ static::assertSame(7, $position);
+
+ $position = 0;
+ static::assertSame('', $decoder->decode(pack('V', 0), $position));
+ static::assertSame(4, $position);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/StructureDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/StructureDecoderTest.php
new file mode 100644
index 0000000000..17c23fbabf
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/StructureDecoderTest.php
@@ -0,0 +1,64 @@
+ new Int64Encoder(), 'name' => new StringEncoder(), 'missing' => new Int64Encoder()],
+ false,
+ new DynamicEncoder(),
+ );
+ $decoder = new StructureDecoder(
+ ['id' => new Int64Decoder(), 'name' => new StringDecoder(), 'missing' => new Int64Decoder()],
+ false,
+ DynamicDecoderMother::create(),
+ );
+
+ $position = 0;
+
+ static::assertSame(
+ ['id' => 1, 'name' => null],
+ $decoder->decode($encoder->encode(['id' => 1, 'name' => null]), $position),
+ );
+ }
+
+ public function test_round_trip_with_extras(): void
+ {
+ $encoder = new StructureEncoder(['id' => new Int64Encoder()], true, new DynamicEncoder());
+ $decoder = new StructureDecoder(['id' => new Int64Decoder()], true, DynamicDecoderMother::create());
+
+ $position = 0;
+
+ static::assertSame(
+ ['id' => 1, 'extra' => 2],
+ $decoder->decode($encoder->encode(['id' => 1, 'extra' => 2]), $position),
+ );
+ }
+
+ public function test_unknown_element_flag_throws(): void
+ {
+ $decoder = new StructureDecoder(['name' => new StringDecoder()], false, DynamicDecoderMother::create());
+ $position = 0;
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('unknown structure element flag');
+
+ $decoder->decode("\xEE", $position);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/TimeZoneDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/TimeZoneDecoderTest.php
new file mode 100644
index 0000000000..96b264e4a1
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/TimeZoneDecoderTest.php
@@ -0,0 +1,26 @@
+decode(pack('V', 15) . 'Australia/Eucla', $position)->getName(),
+ );
+ static::assertSame(19, $position);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/TimeZonesTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/TimeZonesTest.php
new file mode 100644
index 0000000000..090a154f9a
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/TimeZonesTest.php
@@ -0,0 +1,19 @@
+get('UTC')->getName());
+ static::assertSame($timeZones->get('UTC'), $timeZones->get('UTC'));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/UuidDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/UuidDecoderTest.php
new file mode 100644
index 0000000000..6868419dc7
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/UuidDecoderTest.php
@@ -0,0 +1,24 @@
+decode('0196aecb-b568-7e57-a381-8ec8d3e4a531', $position);
+
+ static::assertInstanceOf(Uuid::class, $decoded);
+ static::assertSame('0196aecb-b568-7e57-a381-8ec8d3e4a531', $decoded->toString());
+ static::assertSame(36, $position);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/XmlDocumentDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/XmlDocumentDecoderTest.php
new file mode 100644
index 0000000000..59fa09db07
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/XmlDocumentDecoderTest.php
@@ -0,0 +1,37 @@
+loadXML('- value
');
+
+ $position = 0;
+ $decoded = (new XmlDocumentDecoder())->decode((new XmlDocumentEncoder())->encode($document), $position);
+
+ static::assertSame('item', $decoded->documentElement?->tagName);
+ }
+
+ public function test_invalid_xml_throws(): void
+ {
+ $position = 0;
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('failed to restore DOMDocument');
+
+ (new XmlDocumentDecoder())->decode(pack('V', 9) . 'not < xml', $position);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/XmlElementDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/XmlElementDecoderTest.php
new file mode 100644
index 0000000000..d89baff105
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/XmlElementDecoderTest.php
@@ -0,0 +1,28 @@
+loadXML('- value
');
+
+ $position = 0;
+ $decoded = (new XmlElementDecoder())->decode(
+ (new XmlElementEncoder())->encode($document->documentElement),
+ $position,
+ );
+
+ static::assertSame('item', $decoded->tagName);
+ static::assertSame('5', $decoded->getAttribute('id'));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/BooleanEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/BooleanEncoderTest.php
new file mode 100644
index 0000000000..81adc6792a
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/BooleanEncoderTest.php
@@ -0,0 +1,19 @@
+encode(true));
+ static::assertSame("\x00", $encoder->encode(false));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/DateTimeEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/DateTimeEncoderTest.php
new file mode 100644
index 0000000000..c2bf8d0744
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/DateTimeEncoderTest.php
@@ -0,0 +1,49 @@
+encode(new DateTimeImmutable('2025-01-01 00:00:00 UTC'))[0]),
+ );
+ static::assertSame(Format::DATETIME_MUTABLE, ord($encoder->encode(new DateTime('2025-01-01 00:00:00 UTC'))[0]));
+ }
+
+ public function test_encodes_timestamp_microseconds_and_timezone(): void
+ {
+ $value = new DateTimeImmutable('2025-01-01 00:00:00.123456 UTC');
+
+ static::assertSame(
+ "\x00" . pack('P', $value->getTimestamp()) . pack('V', 123456) . pack('V', 3) . 'UTC',
+ (new DateTimeEncoder())->encode($value),
+ );
+ }
+
+ public function test_custom_datetime_class_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('Floe supports only DateTime and DateTimeImmutable, got '
+ . CustomDateTime::class);
+
+ (new DateTimeEncoder())->encode(new CustomDateTime('2025-01-01 00:00:00 UTC'));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/DynamicEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/DynamicEncoderTest.php
new file mode 100644
index 0000000000..e498c1b365
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/DynamicEncoderTest.php
@@ -0,0 +1,69 @@
+ null,
+ 'int' => 42,
+ 'negative_int' => PHP_INT_MIN,
+ 'float' => 3.5,
+ 'bool' => true,
+ 'string' => 'text',
+ 'array' => ['a' => 1, 0 => 'zero', 'nested' => ['x' => [1, 2]], -7 => 'negative key'],
+ ];
+
+ foreach ($values as $label => $value) {
+ $position = 0;
+
+ static::assertSame($value, $decoder->decode($encoder->encode($value), $position), $label);
+ }
+ }
+
+ public function test_encodes_tagged_scalars(): void
+ {
+ $encoder = new DynamicEncoder();
+
+ static::assertSame(chr(Format::TAG_NULL), $encoder->encode(null));
+ static::assertSame(chr(Format::TAG_INTEGER) . pack('P', 42), $encoder->encode(42));
+ static::assertSame(chr(Format::TAG_BOOLEAN) . "\x01", $encoder->encode(true));
+ static::assertSame(chr(Format::TAG_STRING) . pack('V', 4) . 'text', $encoder->encode('text'));
+ }
+
+ public function test_unsupported_object_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('does not support values of type "stdClass" in mixed/union context');
+
+ (new DynamicEncoder())->encode(new stdClass());
+ }
+
+ public function test_custom_datetime_nested_in_array_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('Floe supports only DateTime and DateTimeImmutable');
+
+ (new DynamicEncoder())->encode(['created_at' => new CustomDateTime('2025-01-01 00:00:00 UTC')]);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/EnumEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/EnumEncoderTest.php
new file mode 100644
index 0000000000..c7c8b9d4e9
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/EnumEncoderTest.php
@@ -0,0 +1,25 @@
+encode(BackedStringEnum::one),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/Float64EncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/Float64EncoderTest.php
new file mode 100644
index 0000000000..2ce9f18af0
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/Float64EncoderTest.php
@@ -0,0 +1,21 @@
+encode(3.5));
+ static::assertSame(pack('e', -0.25), $encoder->encode(-0.25));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/HtmlDocumentEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/HtmlDocumentEncoderTest.php
new file mode 100644
index 0000000000..cef5bf04e2
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/HtmlDocumentEncoderTest.php
@@ -0,0 +1,30 @@
+encode(ValueDecoder::htmlDocumentFromString('x
'));
+
+ static::assertSame(pack('V', strlen($encoded) - 4), substr($encoded, 0, 4));
+ static::assertTrue(str_contains($encoded, 'x
'));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/HtmlElementEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/HtmlElementEncoderTest.php
new file mode 100644
index 0000000000..cdaa7d1e62
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/HtmlElementEncoderTest.php
@@ -0,0 +1,29 @@
+x');
+ $html = ValueEncoder::htmlElementToString($element);
+
+ static::assertSame(pack('V', strlen($html)) . $html, (new HtmlElementEncoder())->encode($element));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/Int64EncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/Int64EncoderTest.php
new file mode 100644
index 0000000000..432d9efa56
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/Int64EncoderTest.php
@@ -0,0 +1,26 @@
+encode(PHP_INT_MIN));
+ static::assertSame(pack('P', PHP_INT_MAX), $encoder->encode(PHP_INT_MAX));
+ static::assertSame("\xFB\xFF\xFF\xFF\xFF\xFF\xFF\xFF", $encoder->encode(-5));
+ static::assertSame("\x2A\x00\x00\x00\x00\x00\x00\x00", $encoder->encode(42));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/IntervalEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/IntervalEncoderTest.php
new file mode 100644
index 0000000000..dccd16a929
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/IntervalEncoderTest.php
@@ -0,0 +1,28 @@
+invert = 1;
+ // @mago-ignore analysis:invalid-property-write
+ $value->f = 0.5;
+
+ static::assertSame(
+ "\x01" . pack('VVVVVV', 1, 2, 3, 4, 5, 6) . pack('e', 0.5),
+ (new IntervalEncoder())->encode($value),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/JsonEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/JsonEncoderTest.php
new file mode 100644
index 0000000000..785328de95
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/JsonEncoderTest.php
@@ -0,0 +1,22 @@
+encode(new Json('[1,2,3]')));
+ static::assertSame(pack('V', 10) . '{"a":true}' . "\x01", $encoder->encode(new Json('{"a":true}')));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/ListEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/ListEncoderTest.php
new file mode 100644
index 0000000000..8fc99e6262
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/ListEncoderTest.php
@@ -0,0 +1,22 @@
+encode(['a', 'bc']));
+ static::assertSame(pack('V', 0), $encoder->encode([]));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/MapEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/MapEncoderTest.php
new file mode 100644
index 0000000000..1fae624359
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/MapEncoderTest.php
@@ -0,0 +1,30 @@
+encode(['a' => 42]));
+ }
+
+ public function test_encodes_integer_keys(): void
+ {
+ $encoder = new MapEncoder(new Int64Encoder(), new StringEncoder());
+
+ static::assertSame(pack('V', 1) . pack('P', 7) . pack('V', 1) . 'x', $encoder->encode([7 => 'x']));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/NullEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/NullEncoderTest.php
new file mode 100644
index 0000000000..6a143bab36
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/NullEncoderTest.php
@@ -0,0 +1,16 @@
+encode(null));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/OptionalEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/OptionalEncoderTest.php
new file mode 100644
index 0000000000..25fe70185c
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/OptionalEncoderTest.php
@@ -0,0 +1,23 @@
+encode(null));
+ static::assertSame(Format::VALUE_PRESENT_BYTE . pack('P', 42), $encoder->encode(42));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/PackedListEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/PackedListEncoderTest.php
new file mode 100644
index 0000000000..81fafd3499
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/PackedListEncoderTest.php
@@ -0,0 +1,28 @@
+encode([10, -20, 30]));
+ static::assertSame(pack('V', 0), $encoder->encode([]));
+ }
+
+ public function test_encodes_float64_list_in_bulk(): void
+ {
+ $encoder = new PackedListEncoder('e');
+
+ static::assertSame(pack('V', 2) . pack('e*', 1.5, -2.5), $encoder->encode([1.5, -2.5]));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StringEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StringEncoderTest.php
new file mode 100644
index 0000000000..adfacbf642
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StringEncoderTest.php
@@ -0,0 +1,21 @@
+encode('abc'));
+ static::assertSame(pack('V', 0), $encoder->encode(''));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StringKeyEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StringKeyEncoderTest.php
new file mode 100644
index 0000000000..6734fc8437
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StringKeyEncoderTest.php
@@ -0,0 +1,24 @@
+encode('abc'));
+ }
+
+ public function test_encodes_numeric_string_key_arriving_as_integer(): void
+ {
+ // PHP turns numeric-string array keys into integers
+ static::assertSame(pack('V', 3) . '123', (new StringKeyEncoder())->encode(123));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StructureEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StructureEncoderTest.php
new file mode 100644
index 0000000000..ae600a6639
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StructureEncoderTest.php
@@ -0,0 +1,43 @@
+ new Int64Encoder(), 'name' => new StringEncoder(), 'missing' => new Int64Encoder()],
+ false,
+ new DynamicEncoder(),
+ );
+
+ static::assertSame(
+ Format::VALUE_PRESENT_BYTE . pack('P', 1) . Format::VALUE_NULL_BYTE . Format::VALUE_ABSENT_BYTE,
+ $encoder->encode(['id' => 1, 'name' => null]),
+ );
+ }
+
+ public function test_encodes_extras_dynamically_when_allowed(): void
+ {
+ $encoder = new StructureEncoder(['id' => new Int64Encoder()], true, new DynamicEncoder());
+
+ static::assertSame(
+ Format::VALUE_PRESENT_BYTE . pack('P', 1) . pack('V', 1) . pack('V', 5) . 'extra' . chr(Format::TAG_INTEGER)
+ . pack('P', 2),
+ $encoder->encode(['id' => 1, 'extra' => 2]),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/TimeZoneEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/TimeZoneEncoderTest.php
new file mode 100644
index 0000000000..c96516c443
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/TimeZoneEncoderTest.php
@@ -0,0 +1,22 @@
+encode(new DateTimeZone('Australia/Eucla')),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/UuidEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/UuidEncoderTest.php
new file mode 100644
index 0000000000..6b737a138c
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/UuidEncoderTest.php
@@ -0,0 +1,20 @@
+encode(new Uuid('0196aecb-b568-7e57-a381-8ec8d3e4a531')),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/XmlDocumentEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/XmlDocumentEncoderTest.php
new file mode 100644
index 0000000000..300eaccc22
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/XmlDocumentEncoderTest.php
@@ -0,0 +1,26 @@
+loadXML('- value
');
+
+ $xml = ValueEncoder::xmlDocumentToString($document);
+
+ static::assertSame(pack('V', strlen($xml)) . $xml, (new XmlDocumentEncoder())->encode($document));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/XmlElementEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/XmlElementEncoderTest.php
new file mode 100644
index 0000000000..741e4782f9
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/XmlElementEncoderTest.php
@@ -0,0 +1,27 @@
+loadXML('- value
');
+ $element = $document->documentElement;
+
+ static::assertSame(
+ pack('V', strlen('- value
')) . '- value
',
+ (new XmlElementEncoder())->encode($element),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/EntryFactoryTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/EntryFactoryTest.php
new file mode 100644
index 0000000000..342c081aab
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/EntryFactoryTest.php
@@ -0,0 +1,38 @@
+create('id', 42, new IntegerDefinition('id'));
+
+ static::assertEquals(int_entry('id', 42), $entry);
+ static::assertSame(serialize(int_entry('id', 42)), serialize($entry));
+ }
+
+ public function test_creating_entry_with_null_value(): void
+ {
+ $entry = EntryFactory::forEntryClass(StringEntry::class)->create(
+ 'name',
+ null,
+ new StringDefinition('name', true),
+ );
+
+ static::assertNull($entry->value());
+ static::assertTrue($entry->definition()->isNullable());
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/EntryInstantiatorTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/EntryInstantiatorTest.php
new file mode 100644
index 0000000000..9d62dd52a2
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/EntryInstantiatorTest.php
@@ -0,0 +1,22 @@
+factoryFor(IntegerEntry::class),
+ $instantiator->factoryFor(IntegerEntry::class),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/ExtRowFrameEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/ExtRowFrameEncoderTest.php
new file mode 100644
index 0000000000..ccc4059ea5
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/ExtRowFrameEncoderTest.php
@@ -0,0 +1,66 @@
+encode($batch), (new ExtRowFrameEncoder())->encode($batch));
+ }
+
+ public function test_new_column_segments_are_identical_to_the_pure_php_engine(): void
+ {
+ $batch = rows(row(int_entry('id', 1)), row(int_entry('id', 2), str_entry('name', 'flow')));
+
+ static::assertEquals((new PhpRowFrameEncoder())->encode($batch), (new ExtRowFrameEncoder())->encode($batch));
+ }
+
+ public function test_narrower_row_segments_are_identical_to_the_pure_php_engine(): void
+ {
+ $batch = rows(row(int_entry('id', 1), str_entry('name', 'flow')), row(int_entry('id', 2)));
+
+ static::assertEquals((new PhpRowFrameEncoder())->encode($batch), (new ExtRowFrameEncoder())->encode($batch));
+ }
+
+ public function test_continuation_across_calls_is_identical_to_the_pure_php_engine(): void
+ {
+ $first = rows(row(int_entry('id', 1)));
+ $second = rows(row(int_entry('id', 2)));
+
+ $php = new PhpRowFrameEncoder();
+ $ext = new ExtRowFrameEncoder();
+
+ static::assertEquals($php->encode($first), $ext->encode($first));
+ static::assertEquals($php->encode($second), $ext->encode($second));
+ }
+
+ public function test_empty_rows_produce_no_segments(): void
+ {
+ static::assertSame([], (new ExtRowFrameEncoder())->encode(rows()));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeDSLTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeDSLTest.php
new file mode 100644
index 0000000000..134fe008bc
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeDSLTest.php
@@ -0,0 +1,48 @@
+source()->uri());
+ }
+
+ public function test_from_floe_resolves_string_path(): void
+ {
+ $extractor = from_floe(__DIR__ . '/example.floe');
+
+ static::assertInstanceOf(FloeExtractor::class, $extractor);
+ static::assertStringEndsWith('example.floe', $extractor->source()->path());
+ }
+
+ public function test_to_floe_builds_loader_from_path(): void
+ {
+ $loader = to_floe(path('memory://b.floe'));
+
+ static::assertInstanceOf(FloeLoader::class, $loader);
+ static::assertSame('memory://b.floe', $loader->destination()->uri());
+ }
+
+ public function test_to_floe_resolves_string_path(): void
+ {
+ $loader = to_floe(__DIR__ . '/out.floe');
+
+ static::assertInstanceOf(FloeLoader::class, $loader);
+ static::assertStringEndsWith('out.floe', $loader->destination()->path());
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php
new file mode 100644
index 0000000000..5dafb25e7e
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php
@@ -0,0 +1,215 @@
+expectException(InvalidArgumentException::class);
+
+ from_floe(path('memory://x.floe'))->changeLimit(0);
+ }
+
+ public function test_default_filter_keeps_only_files(): void
+ {
+ static::assertInstanceOf(OnlyFiles::class, from_floe(path('memory://x.floe'))->filter());
+ }
+
+ public function test_extract_adds_input_file_uri_when_configured(): void
+ {
+ $context = flow_context(config_builder()->putInputIntoRows()->build());
+ $path = path('memory://input-uri.floe');
+
+ $loader = to_floe($path);
+ $loader->load(rows(row(int_entry('id', 1))), $context);
+ $loader->closure($context);
+
+ $batches = iterator_to_array(from_floe($path)->extract($context));
+
+ static::assertSame($path->uri(), $batches[0]->first()->valueOf('_input_file_uri'));
+ }
+
+ public function test_extract_applies_offset_within_a_file(): void
+ {
+ $context = flow_context(config());
+ $path = path('memory://offset-within.floe');
+
+ $loader = to_floe($path);
+ $loader->load(rows(row(int_entry('id', 1)), row(int_entry('id', 2)), row(int_entry('id', 3))), $context);
+ $loader->closure($context);
+
+ $ids = [];
+
+ foreach (from_floe($path)->withOffset(1)->extract($context) as $batch) {
+ foreach ($batch->all() as $extractedRow) {
+ $ids[] = $extractedRow->valueOf('id');
+ }
+ }
+
+ static::assertSame([2, 3], $ids);
+ }
+
+ public function test_extract_honors_limit(): void
+ {
+ $context = flow_context(config());
+ $path = path('memory://limited.floe');
+
+ $loader = to_floe($path);
+ $loader->load(rows(row(int_entry('id', 1)), row(int_entry('id', 2)), row(int_entry('id', 3))), $context);
+ $loader->closure($context);
+
+ $extractor = from_floe($path);
+ $extractor->changeLimit(2);
+
+ $ids = [];
+
+ foreach ($extractor->extract($context) as $batch) {
+ foreach ($batch->all() as $extractedRow) {
+ $ids[] = $extractedRow->valueOf('id');
+ }
+ }
+
+ static::assertSame([1, 2], $ids);
+ }
+
+ public function test_extract_omits_input_file_uri_by_default(): void
+ {
+ $context = flow_context(config());
+ $path = path('memory://no-input-uri.floe');
+
+ $loader = to_floe($path);
+ $loader->load(rows(row(int_entry('id', 1))), $context);
+ $loader->closure($context);
+
+ $batches = iterator_to_array(from_floe($path)->extract($context));
+
+ static::assertSame(['id'], $batches[0]->first()->entries()->names());
+ }
+
+ public function test_extract_reads_rows(): void
+ {
+ $context = flow_context(config());
+ $path = path('memory://read.floe');
+
+ $loader = to_floe($path);
+ $loader->load(rows(row(int_entry('id', 1)), row(int_entry('id', 2))), $context);
+ $loader->closure($context);
+
+ $ids = [];
+
+ foreach (from_floe($path)->extract($context) as $batch) {
+ foreach ($batch->all() as $extractedRow) {
+ $ids[] = $extractedRow->valueOf('id');
+ }
+ }
+
+ static::assertSame([1, 2], $ids);
+ }
+
+ public function test_extract_skips_whole_files_with_offset(): void
+ {
+ $context = flow_context(config());
+
+ foreach (['a' => [1, 2], 'b' => [10, 11]] as $name => $values) {
+ $loader = to_floe(path('memory://skip-files/' . $name . '.floe'));
+ $loader->load(rows(row(int_entry('id', $values[0])), row(int_entry('id', $values[1]))), $context);
+ $loader->closure($context);
+ }
+
+ $ids = [];
+
+ foreach (from_floe(path('memory://skip-files/*.floe'))->withOffset(2)->extract($context) as $batch) {
+ foreach ($batch->all() as $extractedRow) {
+ $ids[] = $extractedRow->valueOf('id');
+ }
+ }
+
+ static::assertSame([10, 11], $ids);
+ }
+
+ public function test_extract_stops_on_stop_signal(): void
+ {
+ $context = flow_context(config());
+ $path = path('memory://stop.floe');
+
+ $loader = to_floe($path);
+ $loader->load(rows(row(int_entry('id', 1)), row(int_entry('id', 2))), $context);
+ $loader->closure($context);
+
+ $generator = from_floe($path)->extract($context);
+ $batches = 0;
+
+ foreach ($generator as $_batch) {
+ $batches++;
+ $generator->send(Signal::STOP);
+ }
+
+ static::assertSame(1, $batches);
+ }
+
+ public function test_is_limited_reflects_change_limit(): void
+ {
+ $extractor = from_floe(path('memory://x.floe'));
+
+ static::assertFalse($extractor->isLimited());
+
+ $extractor->changeLimit(5);
+
+ static::assertTrue($extractor->isLimited());
+ }
+
+ public function test_negative_offset_throws(): void
+ {
+ $this->expectException(InvalidArgumentException::class);
+
+ from_floe(path('memory://x.floe'))->withOffset(-1);
+ }
+
+ public function test_schema_reads_footer_only(): void
+ {
+ $context = flow_context(config());
+ $path = path('memory://schema.floe');
+
+ $loader = to_floe($path);
+ $loader->load(rows(row(int_entry('id', 1))), $context);
+ $loader->closure($context);
+
+ $schema = from_floe($path)->schema($context);
+
+ static::assertNotNull($schema->findDefinition('id'));
+ }
+
+ public function test_source_returns_path(): void
+ {
+ static::assertSame('memory://x.floe', from_floe(path('memory://x.floe'))->source()->uri());
+ }
+
+ public function test_with_path_filter_composes_filters(): void
+ {
+ $extractor = from_floe(path('memory://x.floe'))
+ ->withPathFilter(new OnlyFiles())
+ ->withPathFilter(new OnlyFiles());
+
+ static::assertInstanceOf(Filters::class, $extractor->filter());
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeLoaderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeLoaderTest.php
new file mode 100644
index 0000000000..f6977e2b58
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeLoaderTest.php
@@ -0,0 +1,112 @@
+load(rows(row(int_entry('id', 1)), row(int_entry('id', 2))), $context);
+ $loader->closure($context);
+
+ $ids = [];
+
+ foreach (from_floe($path)->extract($context) as $batch) {
+ foreach ($batch->all() as $extractedRow) {
+ $ids[] = $extractedRow->valueOf('id');
+ }
+ }
+
+ static::assertSame([1, 2], $ids);
+ }
+
+ public function test_destination_returns_path(): void
+ {
+ static::assertSame('memory://out.floe', to_floe(path('memory://out.floe'))->destination()->uri());
+ }
+
+ public function test_load_to_path_without_extension_reports_failure(): void
+ {
+ $this->expectException(RuntimeException::class);
+
+ to_floe(path('memory://no-extension'))->load(rows(row(int_entry('id', 1))), flow_context(config()));
+ }
+
+ public function test_partitioned_batches_write_one_file_per_partition(): void
+ {
+ $context = flow_context(config());
+ $base = path('memory://parts/data.floe');
+
+ $loader = to_floe($base);
+ $loader->load(
+ Rows::partitioned([row(int_entry('id', 1), str_entry('country', 'PL'))], [new Partition('country', 'PL')]),
+ $context,
+ );
+ $loader->load(
+ Rows::partitioned([row(int_entry('id', 2), str_entry('country', 'US'))], [new Partition('country', 'US')]),
+ $context,
+ );
+ $loader->closure($context);
+
+ $fs = $context->filesystem($base);
+
+ static::assertNotNull($fs->status(path('memory://parts/country=PL/data.floe')));
+ static::assertNotNull($fs->status(path('memory://parts/country=US/data.floe')));
+
+ $ids = [];
+
+ foreach (from_floe(path('memory://parts/**/*.floe'))->extract($context) as $batch) {
+ foreach ($batch->all() as $extractedRow) {
+ $ids[] = $extractedRow->valueOf('id');
+ }
+ }
+
+ sort($ids);
+
+ static::assertSame([1, 2], $ids);
+ }
+
+ public function test_repeated_loads_write_a_single_file(): void
+ {
+ $context = flow_context(config());
+ $path = path('memory://repeated.floe');
+
+ $loader = to_floe($path);
+ $loader->load(rows(row(int_entry('id', 1)), row(int_entry('id', 2))), $context);
+ $loader->load(rows(row(int_entry('id', 3))), $context);
+ $loader->closure($context);
+
+ $ids = [];
+
+ // reading the concrete (non-pattern) path proves all rows landed in one file
+ foreach (from_floe($path)->extract($context) as $batch) {
+ foreach ($batch->all() as $extractedRow) {
+ $ids[] = $extractedRow->valueOf('id');
+ }
+ }
+
+ static::assertSame([1, 2, 3], $ids);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeMergerTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeMergerTest.php
new file mode 100644
index 0000000000..bb5497bc72
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeMergerTest.php
@@ -0,0 +1,315 @@
+merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://splice.floe'));
+ (new FloeMerger($fs))->merge(
+ [path('memory://a.floe'), path('memory://b.floe')],
+ path('memory://compact.floe'),
+ compact: true,
+ );
+
+ static::assertEquals(
+ FloeFileContext::readAll($fs, path('memory://splice.floe')),
+ FloeFileContext::readAll($fs, path('memory://compact.floe')),
+ );
+ }
+
+ public function test_compact_coalesces_same_schema_sources_into_one_section(): void
+ {
+ $fs = memory_filesystem();
+ FloeFileContext::write($fs, path('memory://a.floe'), rows(row(int_entry('id', 1))));
+ FloeFileContext::write($fs, path('memory://b.floe'), rows(row(int_entry('id', 2))));
+
+ (new FloeMerger($fs))->merge(
+ [path('memory://a.floe'), path('memory://b.floe')],
+ path('memory://compact.floe'),
+ compact: true,
+ );
+
+ static::assertCount(1, (new FloeReader($fs))->read(path('memory://compact.floe'))->footer()->sections);
+ }
+
+ public function test_empty_source_contributes_no_rows(): void
+ {
+ $fs = memory_filesystem();
+ FloeFileContext::write($fs, path('memory://a.floe'), rows(row(int_entry('id', 1))));
+ FloeFileContext::write($fs, path('memory://empty.floe'), rows());
+ FloeFileContext::write($fs, path('memory://b.floe'), rows(row(int_entry('id', 2))));
+
+ (new FloeMerger($fs))->merge([
+ path('memory://a.floe'),
+ path('memory://empty.floe'),
+ path('memory://b.floe'),
+ ], path('memory://out.floe'));
+
+ static::assertEquals(
+ rows(row(int_entry('id', 1)), row(int_entry('id', 2))),
+ FloeFileContext::readAll($fs, path('memory://out.floe')),
+ );
+ }
+
+ public function test_empty_sources_list_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('at least one source');
+
+ (new FloeMerger(memory_filesystem()))->merge([], path('memory://out.floe'));
+ }
+
+ public function test_evolving_schema_pads_missing_columns(): void
+ {
+ $fs = memory_filesystem();
+ FloeFileContext::write($fs, path('memory://a.floe'), rows(row(int_entry('id', 1))));
+ FloeFileContext::write($fs, path('memory://b.floe'), rows(row(int_entry('id', 2), str_entry('email', 'x@y'))));
+
+ (new FloeMerger($fs))->merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://out.floe'));
+
+ // merging A then B must read back exactly like writing A's rows then B's rows to one file:
+ // the id-only section is padded with a from_null email, same as any heterogeneous Floe file.
+ FloeFileContext::write(
+ $fs,
+ path('memory://reference.floe'),
+ rows(row(int_entry('id', 1)), row(int_entry('id', 2), str_entry('email', 'x@y'))),
+ );
+
+ static::assertEquals(
+ FloeFileContext::readAll($fs, path('memory://reference.floe')),
+ FloeFileContext::readAll($fs, path('memory://out.floe')),
+ );
+ }
+
+ public function test_metadata_is_merged_with_new_keys_winning(): void
+ {
+ $fs = memory_filesystem();
+ FloeFileContext::write($fs, path('memory://a.floe'), rows(row(int_entry('id', 1))), [
+ 'a' => '1',
+ 'shared' => 'from-a',
+ ]);
+ FloeFileContext::write($fs, path('memory://b.floe'), rows(row(int_entry('id', 2))), [
+ 'b' => '2',
+ 'shared' => 'from-b',
+ ]);
+
+ (new FloeMerger($fs))->merge(
+ [path('memory://a.floe'), path('memory://b.floe')],
+ path('memory://out.floe'),
+ metadata: Metadata::fromArray(['shared' => 'override']),
+ );
+
+ static::assertSame(
+ ['a' => '1', 'shared' => 'override', 'b' => '2'],
+ (new FloeReader($fs))
+ ->read(path('memory://out.floe'))
+ ->metadata()
+ ->normalize(),
+ );
+ }
+
+ public function test_missing_source_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('does not exist');
+
+ (new FloeMerger(memory_filesystem()))->merge([path('memory://missing.floe')], path('memory://out.floe'));
+ }
+
+ public function test_offset_read_after_merge_seeks_across_sources(): void
+ {
+ $fs = memory_filesystem();
+ FloeFileContext::write($fs, path('memory://a.floe'), rows(row(int_entry('id', 1)), row(int_entry('id', 2))));
+ FloeFileContext::write($fs, path('memory://b.floe'), rows(row(int_entry('id', 3)), row(int_entry('id', 4))));
+
+ (new FloeMerger($fs))->merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://out.floe'));
+
+ $read = [];
+
+ foreach ((new FloeReader($fs))
+ ->read(path('memory://out.floe'))
+ ->rows(batchSize: 100, offset: 3) as $batch) {
+ foreach ($batch as $r) {
+ $read[] = $r->valueOf('id');
+ }
+ }
+
+ static::assertSame([4], $read);
+ }
+
+ public function test_partition_combination_mismatch_throws(): void
+ {
+ $fs = memory_filesystem();
+ FloeFileContext::write(
+ $fs,
+ path('memory://pl.floe'),
+ Rows::partitioned([row(int_entry('id', 1), str_entry('c', 'PL'))], [new Partition('c', 'PL')]),
+ );
+ FloeFileContext::write(
+ $fs,
+ path('memory://us.floe'),
+ Rows::partitioned([row(int_entry('id', 2), str_entry('c', 'US'))], [new Partition('c', 'US')]),
+ );
+
+ $this->expectException(IncompatibleSchemaException::class);
+ $this->expectExceptionMessage('one partition combination');
+
+ (new FloeMerger($fs))->merge([path('memory://pl.floe'), path('memory://us.floe')], path('memory://out.floe'));
+ }
+
+ public function test_partitioned_sources_merge_and_preserve_partition(): void
+ {
+ $fs = memory_filesystem();
+ FloeFileContext::write(
+ $fs,
+ path('memory://a.floe'),
+ Rows::partitioned([row(int_entry('id', 1), str_entry('c', 'PL'))], [new Partition('c', 'PL')]),
+ );
+ FloeFileContext::write(
+ $fs,
+ path('memory://b.floe'),
+ Rows::partitioned([row(int_entry('id', 2), str_entry('c', 'PL'))], [new Partition('c', 'PL')]),
+ );
+
+ (new FloeMerger($fs))->merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://out.floe'));
+
+ $file = (new FloeReader($fs))->read(path('memory://out.floe'));
+
+ static::assertSame(['c' => 'PL'], $file->footer()->partitions);
+ static::assertSame(2, $file->totalRows());
+ }
+
+ public function test_single_source_is_a_copy(): void
+ {
+ $fs = memory_filesystem();
+ $value = rows(row(int_entry('id', 1)), row(int_entry('id', 2)));
+ FloeFileContext::write($fs, path('memory://a.floe'), $value);
+
+ (new FloeMerger($fs))->merge([path('memory://a.floe')], path('memory://out.floe'));
+
+ static::assertEquals($value, FloeFileContext::readAll($fs, path('memory://out.floe')));
+ }
+
+ public function test_splices_three_files(): void
+ {
+ $fs = memory_filesystem();
+ FloeFileContext::write($fs, path('memory://a.floe'), rows(row(int_entry('id', 1))));
+ FloeFileContext::write($fs, path('memory://b.floe'), rows(row(int_entry('id', 2))));
+ FloeFileContext::write($fs, path('memory://c.floe'), rows(row(int_entry('id', 3))));
+
+ (new FloeMerger($fs))->merge([
+ path('memory://a.floe'),
+ path('memory://b.floe'),
+ path('memory://c.floe'),
+ ], path('memory://out.floe'));
+
+ static::assertEquals(
+ rows(row(int_entry('id', 1)), row(int_entry('id', 2)), row(int_entry('id', 3))),
+ FloeFileContext::readAll($fs, path('memory://out.floe')),
+ );
+ }
+
+ public function test_splices_two_files_with_the_same_schema(): void
+ {
+ $fs = memory_filesystem();
+ FloeFileContext::write($fs, path('memory://a.floe'), rows(row(int_entry('id', 1)), row(int_entry('id', 2))));
+ FloeFileContext::write($fs, path('memory://b.floe'), rows(row(int_entry('id', 3)), row(int_entry('id', 4))));
+
+ (new FloeMerger($fs))->merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://out.floe'));
+
+ static::assertEquals(
+ rows(row(int_entry('id', 1)), row(int_entry('id', 2)), row(int_entry('id', 3)), row(int_entry('id', 4))),
+ FloeFileContext::readAll($fs, path('memory://out.floe')),
+ );
+ }
+
+ public function test_type_conflict_throws(): void
+ {
+ $fs = memory_filesystem();
+ FloeFileContext::write($fs, path('memory://a.floe'), rows(row(int_entry('id', 1))));
+ FloeFileContext::write($fs, path('memory://b.floe'), rows(row(str_entry('id', 'x'))));
+
+ $this->expectException(IncompatibleSchemaException::class);
+ $this->expectExceptionMessage('not compatible');
+
+ (new FloeMerger($fs))->merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://out.floe'));
+ }
+
+ public function test_unsized_source_throws(): void
+ {
+ $fs = memory_filesystem();
+ FloeFileContext::write($fs, path('memory://a.floe'), rows(row(int_entry('id', 1))));
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('does not report its size');
+
+ (new FloeMerger(new UnsizedFilesystem($fs)))->merge([path('memory://a.floe')], path('memory://out.floe'));
+ }
+
+ public function test_source_smaller_than_header_and_trailer_throws(): void
+ {
+ $fs = memory_filesystem();
+ $fs->writeTo(path('memory://tiny.floe'))->append("FLOE\x01\x00")->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('too small');
+
+ (new FloeMerger($fs))->merge([path('memory://tiny.floe')], path('memory://out.floe'));
+ }
+
+ public function test_non_noop_codec_source_throws(): void
+ {
+ $fs = memory_filesystem();
+ $fs
+ ->writeTo(path('memory://codec.floe'))
+ ->append("FLOE\x01\x05" . str_repeat("\0", 10))
+ ->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('no-op codec');
+
+ (new FloeMerger($fs))->merge([path('memory://codec.floe')], path('memory://out.floe'));
+ }
+
+ public function test_source_with_footer_larger_than_file_throws(): void
+ {
+ $fs = memory_filesystem();
+ $fs
+ ->writeTo(path('memory://torn.floe'))
+ ->append("FLOE\x01\x00" . pack('V', 1_000_000) . 'FLOE')
+ ->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('footer does not fit');
+
+ (new FloeMerger($fs))->merge([path('memory://torn.floe')], path('memory://out.floe'));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeReaderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeReaderTest.php
new file mode 100644
index 0000000000..09b04cbb9b
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeReaderTest.php
@@ -0,0 +1,1288 @@
+create($path);
+ $writer->write($rows);
+ $writer->close();
+
+ $batches = iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(),
+ );
+
+ static::assertCount(1, $batches);
+ static::assertEquals($rows->all(), $batches[0]->all());
+ }
+
+ public function test_batches_follow_requested_size(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://batches.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2)),
+ row(int_entry('id', 3)),
+ row(int_entry('id', 4)),
+ row(int_entry('id', 5)),
+ ));
+ $writer->close();
+
+ $reader = (new FloeReader($filesystem))->read($path);
+
+ $sizes =
+ /**
+ * @param int<1, max> $batchSize
+ *
+ * @return array
+ */
+ static fn(int $batchSize): array => array_map(
+ static fn(Rows $batch): int => $batch->count(),
+ iterator_to_array($reader->rows($batchSize)),
+ );
+
+ static::assertSame([2, 2, 1], $sizes(2));
+ static::assertSame([5], $sizes(5));
+ static::assertSame([1, 1, 1, 1, 1], $sizes(1));
+ static::assertSame([5], $sizes(100));
+ }
+
+ public function test_limit_caps_returned_rows(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://limit.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2)),
+ row(int_entry('id', 3)),
+ row(int_entry('id', 4)),
+ ));
+ $writer->close();
+
+ $ids = [];
+
+ foreach ((new FloeReader($filesystem))
+ ->read($path)
+ ->rows(batchSize: 100, offset: 0, limit: 2) as $batch) {
+ foreach ($batch->all() as $row) {
+ $ids[] = $row->valueOf('id');
+ }
+ }
+
+ static::assertSame([1, 2], $ids);
+ }
+
+ public function test_limit_stops_within_a_batch(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://limit-batch.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1)), row(int_entry('id', 2)), row(int_entry('id', 3))));
+ $writer->close();
+
+ $batches = iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(batchSize: 100, limit: 2),
+ );
+
+ static::assertCount(1, $batches);
+ static::assertCount(2, $batches[0]->all());
+ }
+
+ public function test_negative_offset_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('offset must be greater or equal to 0');
+
+ iterator_to_array(
+ (new FloeReader(memory_filesystem()))
+ ->read(path('memory://neg-offset.floe'))
+ ->rows(offset: -1),
+ );
+ }
+
+ public function test_non_positive_limit_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('limit must be greater than 0');
+
+ iterator_to_array(
+ (new FloeReader(memory_filesystem()))
+ ->read(path('memory://zero-limit.floe'))
+ ->rows(limit: 0),
+ );
+ }
+
+ public function test_offset_skips_leading_rows_within_a_single_section(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://offset-single.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2)),
+ row(int_entry('id', 3)),
+ row(int_entry('id', 4)),
+ ));
+ $writer->close();
+
+ $ids = [];
+
+ foreach ((new FloeReader($filesystem))
+ ->read($path)
+ ->rows(offset: 2) as $batch) {
+ foreach ($batch->all() as $row) {
+ $ids[] = $row->valueOf('id');
+ }
+ }
+
+ static::assertSame([3, 4], $ids);
+ }
+
+ public function test_offset_skips_whole_sections(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://offset-sections.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2)),
+ row(int_entry('id', 3)),
+ row(int_entry('id', 4), str_entry('email', 'a')),
+ row(int_entry('id', 5), str_entry('email', 'b')),
+ row(int_entry('id', 6)),
+ row(int_entry('id', 7)),
+ ));
+ $writer->close();
+
+ $ids = [];
+ $names = [];
+ $emails = [];
+
+ foreach ((new FloeReader($filesystem))
+ ->read($path)
+ ->rows(offset: 4) as $batch) {
+ foreach ($batch->all() as $row) {
+ $ids[] = $row->valueOf('id');
+ $names[] = $row->entries()->names();
+ $emails[] = $row->valueOf('email');
+ }
+ }
+
+ static::assertSame([5, 6, 7], $ids);
+ static::assertSame(['id', 'email'], $names[0]);
+ static::assertSame('b', $emails[0]);
+ static::assertNull($emails[1]);
+ }
+
+ public function test_offset_beyond_total_rows_yields_nothing(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://offset-past-end.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1)), row(int_entry('id', 2))));
+ $writer->close();
+
+ static::assertSame(
+ [],
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(offset: 2),
+ ),
+ );
+ static::assertSame(
+ [],
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(offset: 5),
+ ),
+ );
+ }
+
+ public function test_offset_respects_batch_size(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://offset-batches.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2)),
+ row(int_entry('id', 3)),
+ row(int_entry('id', 4)),
+ ));
+ $writer->close();
+
+ $sizes = array_map(
+ static fn(Rows $batch): int => $batch->count(),
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(batchSize: 1, offset: 1),
+ ),
+ );
+
+ static::assertSame([1, 1, 1], $sizes);
+ }
+
+ public function test_offset_and_limit_combined(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://offset-limit.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2)),
+ row(int_entry('id', 3)),
+ row(int_entry('id', 4)),
+ row(int_entry('id', 5)),
+ ));
+ $writer->close();
+
+ $ids = [];
+
+ foreach ((new FloeReader($filesystem))
+ ->read($path)
+ ->rows(batchSize: 100, offset: 1, limit: 2) as $batch) {
+ foreach ($batch->all() as $row) {
+ $ids[] = $row->valueOf('id');
+ }
+ }
+
+ static::assertSame([2, 3], $ids);
+ }
+
+ public function test_offset_matches_tail_of_full_read(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://offset-tail.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2)),
+ row(int_entry('id', 3)),
+ row(int_entry('id', 4), str_entry('email', 'a')),
+ row(int_entry('id', 5), str_entry('email', 'b')),
+ row(int_entry('id', 6)),
+ row(int_entry('id', 7)),
+ ));
+ $writer->close();
+
+ $reader = (new FloeReader($filesystem))->read($path);
+ $full = [];
+
+ foreach ($reader->rows() as $batch) {
+ foreach ($batch->all() as $row) {
+ $full[] = $row->valueOf('id');
+ }
+ }
+
+ for ($offset = 0; $offset <= 7; $offset++) {
+ $tail = [];
+
+ foreach ($reader->rows(offset: $offset) as $batch) {
+ foreach ($batch->all() as $row) {
+ $tail[] = $row->valueOf('id');
+ }
+ }
+
+ static::assertSame(array_slice($full, $offset), $tail, "offset {$offset}");
+ }
+ }
+
+ public function test_offset_reads_through_a_custom_identity_codec(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://offset-codec.floe');
+
+ $writer = new FloeWriter($filesystem, new CodecStub(0x00));
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1)), row(int_entry('id', 2)), row(int_entry('id', 3))));
+ $writer->close();
+
+ $ids = [];
+
+ foreach ((new FloeReader($filesystem, new CodecStub(0x00)))
+ ->read($path)
+ ->rows(offset: 1) as $batch) {
+ foreach ($batch->all() as $extractedRow) {
+ $ids[] = $extractedRow->valueOf('id');
+ }
+ }
+
+ static::assertSame([2, 3], $ids);
+ }
+
+ public function test_offset_reattaches_partitions(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://offset-partitioned.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(Rows::partitioned([
+ row(int_entry('id', 1), str_entry('country', 'PL')),
+ row(int_entry('id', 2), str_entry('country', 'PL')),
+ row(int_entry('id', 3), str_entry('country', 'PL')),
+ ], [new Partition('country', 'PL')]));
+ $writer->close();
+
+ $batches = iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(offset: 1),
+ );
+
+ static::assertSame([2, 3], array_map(static fn($row) => $row->valueOf('id'), $batches[0]->all()));
+ static::assertEquals([new Partition('country', 'PL')], iterator_to_array($batches[0]->partitions()));
+ }
+
+ public function test_corrupt_row_body_throws_wrapped_exception(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://corrupt-row.floe');
+ $schemaBody = FloeWriter::growSectionPlan(null, row(int_entry('id', 1)))->schemaBody;
+ $footerJson = FooterMother::footer()->toJson();
+ $stream = $filesystem->writeTo($path);
+ $stream->append(
+ Format::header(0x00)
+ . Format::frame(Format::FRAME_SCHEMA, $schemaBody)
+ . Format::frame(Format::FRAME_ROW, "\xEE")
+ . Format::frame(Format::FRAME_FOOTER, $footerJson . Format::trailer(strlen($footerJson))),
+ );
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('unknown value flag');
+
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(),
+ );
+ }
+
+ public function test_corrupted_frame_chain_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://corrupted.floe');
+ $footerJson = FooterMother::footer()->toJson();
+ $stream = $filesystem->writeTo($path);
+ $stream->append(
+ Format::header(0x00) . "\x02" . pack('V', 10_000) . 'short'
+ . Format::frame(Format::FRAME_FOOTER, $footerJson . Format::trailer(strlen($footerJson))),
+ );
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('frame body is incomplete');
+
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(),
+ );
+ }
+
+ public function test_dead_footers_of_appended_files_are_skipped(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://appended.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1))));
+ $writer->close();
+
+ $writer = new FloeWriter($filesystem);
+ $writer->append($path);
+ $writer->write(rows(row(int_entry('id', 2))));
+ $writer->close();
+
+ $batches = iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(),
+ );
+
+ static::assertCount(1, $batches);
+ static::assertSame([1, 2], array_map(static fn($row) => $row->valueOf('id'), $batches[0]->all()));
+ }
+
+ public function test_empty_file_yields_no_batches(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://empty.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->close();
+
+ $reader = (new FloeReader($filesystem))->read($path);
+
+ static::assertSame([], iterator_to_array($reader->rows()));
+ static::assertSame([], iterator_to_array($reader->recover()));
+ static::assertSame(0, $reader->totalRows());
+ static::assertCount(0, $reader->schema()->definitions());
+ }
+
+ public function test_evolved_file_rows_conform_to_merged_schema(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://evolved.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1)), row(int_entry('id', 2))));
+ $writer->close();
+
+ $writer = new FloeWriter($filesystem);
+ $writer->append($path);
+ $writer->write(rows(
+ // new columns must be nullable on append - the first row of the section defines its schema
+ row(int_entry('id', 3), str_entry('email', null)),
+ row(int_entry('id', 4), str_entry('email', 'x@flow.php')),
+ ));
+ $writer->close();
+
+ $batches = iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(),
+ );
+ $rows = $batches[0]->all();
+
+ static::assertCount(4, $rows);
+
+ foreach ($rows as $row) {
+ static::assertSame(['id', 'email'], $row->entries()->names());
+ }
+
+ static::assertNull($rows[0]->valueOf('email'));
+ static::assertTrue($rows[0]->get('email')->definition()->metadata()->has(Metadata::FROM_NULL));
+ static::assertSame('x@flow.php', $rows[3]->valueOf('email'));
+ }
+
+ public function test_footer_length_exceeding_file_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://bad-footer-length.floe');
+ $stream = $filesystem->writeTo($path);
+ $stream->append(Format::header(0x00) . Format::trailer(9999));
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('footer does not fit inside the file');
+
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->footer();
+ }
+
+ public function test_footer_of_file_smaller_than_header_and_trailer_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://stub.floe');
+ $stream = $filesystem->writeTo($path);
+ $stream->append('FLOE');
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('too small');
+
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->footer();
+ }
+
+ public function test_footer_over_unsized_stream_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://unsized.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('requires a sized stream');
+
+ (new FloeReader(new UnsizedFilesystem($filesystem)))
+ ->read($path)
+ ->footer();
+ }
+
+ public function test_footer_accessors(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://accessors.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path, Metadata::fromArray(['source' => 'unit']));
+ $writer->write(rows(row(int_entry('id', 1), str_entry('name', 'x'))));
+ $writer->close();
+
+ $reader = (new FloeReader($filesystem))->read($path);
+
+ static::assertSame(Format::VERSION, $reader->footer()->version);
+ static::assertSame(['source' => 'unit'], $reader->metadata()->normalize());
+ static::assertSame(1, $reader->totalRows());
+ static::assertNotNull($reader->schema()->findDefinition('id'));
+ static::assertNotNull($reader->schema()->findDefinition('name'));
+ }
+
+ public function test_open_with_non_noop_codec_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('supports only the no-op codec, got codec 0x09');
+
+ (new FloeReader(memory_filesystem(), new CodecStub(0x09)))->read(path('memory://x.floe'));
+ }
+
+ public function test_partitions_are_reattached_to_batches(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://partitioned.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(Rows::partitioned([row(int_entry('id', 1), str_entry('country', 'PL'))], [new Partition(
+ 'country',
+ 'PL',
+ )]));
+ $writer->close();
+
+ $batches = iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(),
+ );
+
+ static::assertCount(1, $batches);
+ static::assertEquals([new Partition('country', 'PL')], iterator_to_array($batches[0]->partitions()));
+ }
+
+ public function test_reader_with_mismatched_codec_flags_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://flags.floe');
+ $stream = $filesystem->writeTo($path);
+ $stream->append(Format::header(0x07) . Format::trailer(0));
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('written with codec 0x07');
+
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->footer();
+ }
+
+ public function test_recover_chunks_batches(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://recover-batches.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1)), row(int_entry('id', 2)), row(int_entry('id', 3))));
+ $writer->close();
+
+ static::assertCount(
+ 2,
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->recover(2),
+ ),
+ );
+ }
+
+ public function test_recover_stops_at_corrupt_row_body(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://recover-corrupt-row.floe');
+ $schemaBody = FloeWriter::growSectionPlan(null, row(int_entry('id', 1)))->schemaBody;
+ $stream = $filesystem->writeTo($path);
+ $stream->append(
+ Format::header(0x00) . Format::frame(Format::FRAME_SCHEMA, $schemaBody)
+ . Format::frame(Format::FRAME_ROW, "\xEE"),
+ );
+ $stream->close();
+
+ static::assertSame(
+ [],
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->recover(),
+ ),
+ );
+ }
+
+ public function test_recover_stops_at_corrupt_schema_body(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://recover-corrupt-schema.floe');
+ $stream = $filesystem->writeTo($path);
+ $stream->append(Format::header(0x00) . Format::frame(Format::FRAME_SCHEMA, '{broken'));
+ $stream->close();
+
+ static::assertSame(
+ [],
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->recover(),
+ ),
+ );
+ }
+
+ public function test_recover_stops_at_unknown_frame_type(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://recover-unknown.floe');
+
+ FloeFileContext::writeWithoutFooter($filesystem, $path, rows(row(int_entry('id', 1))));
+
+ $filesystem->appendTo($path)->append(Format::frame(0x55, 'mystery'))->close();
+
+ $batches = iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->recover(),
+ );
+
+ static::assertCount(1, $batches);
+ static::assertCount(1, $batches[0]->all());
+ }
+
+ public function test_recover_reads_partitions_from_frame(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://recover-partitions.floe');
+
+ FloeFileContext::writeWithoutFooter(
+ $filesystem,
+ $path,
+ Rows::partitioned([row(int_entry('id', 1), str_entry('country', 'PL'))], [new Partition('country', 'PL')]),
+ );
+
+ $batches = iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->recover(),
+ );
+
+ static::assertCount(1, $batches);
+ static::assertEquals([new Partition('country', 'PL')], iterator_to_array($batches[0]->partitions()));
+ }
+
+ public function test_recover_salvages_rows_from_file_missing_close(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://torn.floe');
+
+ FloeFileContext::writeWithoutFooter($filesystem, $path, rows(row(int_entry('id', 1)), row(int_entry('id', 2))));
+
+ $reader = (new FloeReader($filesystem))->read($path);
+ $batches = iterator_to_array($reader->recover());
+
+ static::assertCount(1, $batches);
+ static::assertCount(2, $batches[0]->all());
+ }
+
+ public function test_recover_salvages_complete_frames_from_truncated_tail(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://complete.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1)), row(int_entry('id', 2)), row(int_entry('id', 3))));
+ $writer->close();
+
+ $content = $filesystem->readFrom($path)->content();
+ $truncated = path('memory://truncated.floe');
+ $stream = $filesystem->writeTo($truncated);
+ $stream->append(substr($content, 0, strlen($content) - 320));
+ $stream->close();
+
+ $salvaged = iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($truncated)
+ ->recover(),
+ );
+
+ static::assertNotSame([], $salvaged);
+ static::assertLessThanOrEqual(3, count($salvaged[0]->all()));
+ }
+
+ public function test_recover_yields_rows_as_written_without_padding(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://recover-evolved.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1))));
+ $writer->close();
+
+ $writer = new FloeWriter($filesystem);
+ $writer->append($path);
+ $writer->write(rows(row(int_entry('id', 2), str_entry('email', null))));
+ $writer->close();
+
+ $rows = [];
+
+ foreach ((new FloeReader($filesystem))
+ ->read($path)
+ ->recover() as $batch) {
+ foreach ($batch->all() as $row) {
+ $rows[] = $row->entries()->names();
+ }
+ }
+
+ static::assertSame([['id'], ['id', 'email']], $rows);
+ }
+
+ public function test_recover_stops_at_row_frame_before_schema_frame(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://recover-row-first.floe');
+ $stream = $filesystem->writeTo($path);
+ $stream->append(Format::header(0x00) . Format::frame(Format::FRAME_ROW, 'row-bytes'));
+ $stream->close();
+
+ static::assertSame(
+ [],
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->recover(),
+ ),
+ );
+ }
+
+ public function test_recover_stops_at_row_body_longer_than_hydrated_content(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://recover-long-row.floe');
+ $plan = FloeWriter::growSectionPlan(null, row(int_entry('id', 1)));
+ $rowBody = (new RowEncoder())->encode($plan, row(int_entry('id', 1)));
+ $stream = $filesystem->writeTo($path);
+ $stream->append(
+ Format::header(0x00) . Format::frame(Format::FRAME_SCHEMA, $plan->schemaBody)
+ . Format::frame(Format::FRAME_ROW, $rowBody . 'extra-bytes'),
+ );
+ $stream->close();
+
+ static::assertSame(
+ [],
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->recover(),
+ ),
+ );
+ }
+
+ public function test_rows_of_file_replaced_after_footer_load_with_different_codec_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://swapped-flags.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1))));
+ $writer->close();
+
+ $reader = (new FloeReader($filesystem))->read($path);
+ $reader->footer();
+
+ $stream = $filesystem->writeTo($path);
+ $stream->append(Format::header(0x05) . Format::frame(Format::FRAME_ROW, 'x'));
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('written with codec 0x05');
+
+ iterator_to_array($reader->rows());
+ }
+
+ public function test_rows_of_file_replaced_after_footer_load_with_torn_frame_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://swapped-torn.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1))));
+ $writer->close();
+
+ $reader = (new FloeReader($filesystem))->read($path);
+ $reader->footer();
+
+ $stream = $filesystem->writeTo($path);
+ $stream->append(Format::header(0x00) . "\x02\xFF");
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('frame header is incomplete');
+
+ iterator_to_array($reader->rows());
+ }
+
+ public function test_rows_of_file_truncated_below_header_after_footer_load_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://swapped-short.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1))));
+ $writer->close();
+
+ $reader = (new FloeReader($filesystem))->read($path);
+ $reader->footer();
+
+ $stream = $filesystem->writeTo($path);
+ $stream->append('FLO');
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('header is incomplete');
+
+ iterator_to_array($reader->rows());
+ }
+
+ public function test_rows_through_a_custom_identity_codec(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://custom-codec.floe');
+
+ $writer = new FloeWriter($filesystem, new CodecStub(0x00));
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1)), row(int_entry('id', 2))));
+ $writer->close();
+
+ $batches = iterator_to_array(
+ (new FloeReader($filesystem, new CodecStub(0x00)))
+ ->read($path)
+ ->rows(),
+ );
+
+ static::assertCount(1, $batches);
+ static::assertCount(2, $batches[0]->all());
+ }
+
+ public function test_rows_through_a_custom_identity_codec_with_long_row_body_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://custom-codec-long-row.floe');
+ $plan = FloeWriter::growSectionPlan(null, row(int_entry('id', 1)));
+ $rowBody = (new RowEncoder())->encode($plan, row(int_entry('id', 1)));
+ $footerJson = FooterMother::footer()->toJson();
+ $stream = $filesystem->writeTo($path);
+ $stream->append(
+ Format::header(0x00)
+ . Format::frame(Format::FRAME_SCHEMA, $plan->schemaBody)
+ . Format::frame(Format::FRAME_ROW, $rowBody . 'extra-bytes')
+ . Format::frame(Format::FRAME_FOOTER, $footerJson . Format::trailer(strlen($footerJson))),
+ );
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('row frame length does not match its content');
+
+ iterator_to_array(
+ (new FloeReader($filesystem, new CodecStub(0x00)))
+ ->read($path)
+ ->rows(),
+ );
+ }
+
+ public function test_row_frame_before_schema_frame_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://row-first.floe');
+ $footerJson = FooterMother::footer()->toJson();
+ $stream = $filesystem->writeTo($path);
+ $stream->append(
+ Format::header(0x00) . Format::frame(Format::FRAME_ROW, 'row-bytes')
+ . Format::frame(Format::FRAME_FOOTER, $footerJson . Format::trailer(strlen($footerJson))),
+ );
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('row frame before any schema frame');
+
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(),
+ );
+ }
+
+ public function test_row_body_longer_than_hydrated_content_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://long-row.floe');
+ $plan = FloeWriter::growSectionPlan(null, row(int_entry('id', 1)));
+ $rowBody = (new RowEncoder())->encode($plan, row(int_entry('id', 1)));
+ $footerJson = FooterMother::footer()->toJson();
+ $stream = $filesystem->writeTo($path);
+ $stream->append(
+ Format::header(0x00)
+ . Format::frame(Format::FRAME_SCHEMA, $plan->schemaBody)
+ . Format::frame(Format::FRAME_ROW, $rowBody . 'extra-bytes')
+ . Format::frame(Format::FRAME_FOOTER, $footerJson . Format::trailer(strlen($footerJson))),
+ );
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('row frame length does not match its content');
+
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(),
+ );
+ }
+
+ public function test_torn_file_without_footer_throws_in_strict_mode(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://strict-torn.floe');
+
+ FloeFileContext::writeWithoutFooter($filesystem, $path, rows(row(int_entry('id', 1))));
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('magic');
+
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(),
+ );
+ }
+
+ public function test_unknown_frame_type_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://unknown-frame.floe');
+ $footerJson = FooterMother::footer()->toJson();
+ $stream = $filesystem->writeTo($path);
+ $stream->append(
+ Format::header(0x00) . Format::frame(0x55, 'mystery')
+ . Format::frame(Format::FRAME_FOOTER, $footerJson . Format::trailer(strlen($footerJson))),
+ );
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('unknown frame type 0x55');
+
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->rows(),
+ );
+ }
+
+ public function test_head_returns_first_rows(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://head.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2)),
+ row(int_entry('id', 3)),
+ row(int_entry('id', 4)),
+ row(int_entry('id', 5)),
+ ));
+ $writer->close();
+
+ $ids = [];
+
+ foreach ((new FloeReader($filesystem))
+ ->read($path)
+ ->head(3) as $batch) {
+ foreach ($batch->all() as $row) {
+ $ids[] = $row->valueOf('id');
+ }
+ }
+
+ static::assertSame([1, 2, 3], $ids);
+ }
+
+ public function test_head_count_beyond_total_returns_all_rows(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://head-all.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1)), row(int_entry('id', 2))));
+ $writer->close();
+
+ $ids = [];
+
+ foreach ((new FloeReader($filesystem))
+ ->read($path)
+ ->head(5) as $batch) {
+ foreach ($batch->all() as $row) {
+ $ids[] = $row->valueOf('id');
+ }
+ }
+
+ static::assertSame([1, 2], $ids);
+ }
+
+ public function test_head_respects_batch_size(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://head-batches.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2)),
+ row(int_entry('id', 3)),
+ row(int_entry('id', 4)),
+ ));
+ $writer->close();
+
+ $sizes = array_map(
+ static fn(Rows $batch): int => count($batch->all()),
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->head(3, batchSize: 1),
+ ),
+ );
+
+ static::assertSame([1, 1, 1], $sizes);
+ }
+
+ public function test_head_non_positive_count_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://head-zero.floe');
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('head count must be greater than 0');
+
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->head(0),
+ );
+ }
+
+ public function test_tail_returns_last_rows_across_sections(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://tail-sections.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2)),
+ row(int_entry('id', 3)),
+ row(int_entry('id', 4), str_entry('email', 'a')),
+ row(int_entry('id', 5), str_entry('email', 'b')),
+ row(int_entry('id', 6)),
+ row(int_entry('id', 7)),
+ ));
+ $writer->close();
+
+ $ids = [];
+ $names = [];
+
+ foreach ((new FloeReader($filesystem))
+ ->read($path)
+ ->tail(3) as $batch) {
+ foreach ($batch->all() as $row) {
+ $ids[] = $row->valueOf('id');
+ $names[] = $row->entries()->names();
+ }
+ }
+
+ static::assertSame([5, 6, 7], $ids);
+ static::assertSame(['id', 'email'], $names[0]);
+ }
+
+ public function test_tail_count_beyond_total_returns_all_rows(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://tail-all.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1)), row(int_entry('id', 2))));
+ $writer->close();
+
+ $ids = [];
+
+ foreach ((new FloeReader($filesystem))
+ ->read($path)
+ ->tail(5) as $batch) {
+ foreach ($batch->all() as $row) {
+ $ids[] = $row->valueOf('id');
+ }
+ }
+
+ static::assertSame([1, 2], $ids);
+ }
+
+ public function test_tail_respects_batch_size(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://tail-batches.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2)),
+ row(int_entry('id', 3)),
+ row(int_entry('id', 4)),
+ ));
+ $writer->close();
+
+ $sizes = array_map(
+ static fn(Rows $batch): int => count($batch->all()),
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->tail(3, batchSize: 1),
+ ),
+ );
+
+ static::assertSame([1, 1, 1], $sizes);
+ }
+
+ public function test_tail_matches_slice_of_full_read(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://tail-slice.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2)),
+ row(int_entry('id', 3)),
+ row(int_entry('id', 4)),
+ row(int_entry('id', 5)),
+ row(int_entry('id', 6)),
+ row(int_entry('id', 7)),
+ ));
+ $writer->close();
+
+ $reader = (new FloeReader($filesystem))->read($path);
+
+ $full = [];
+
+ foreach ($reader->rows() as $batch) {
+ foreach ($batch->all() as $row) {
+ $full[] = $row->valueOf('id');
+ }
+ }
+
+ for ($count = 1; $count <= 9; $count++) {
+ $tail = [];
+
+ foreach ($reader->tail($count) as $batch) {
+ foreach ($batch->all() as $row) {
+ $tail[] = $row->valueOf('id');
+ }
+ }
+
+ static::assertSame(array_slice($full, -$count), $tail, "tail {$count}");
+ }
+ }
+
+ public function test_tail_non_positive_count_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://tail-zero.floe');
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('tail count must be greater than 0');
+
+ iterator_to_array(
+ (new FloeReader($filesystem))
+ ->read($path)
+ ->tail(0),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeSerializerTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeSerializerTest.php
new file mode 100644
index 0000000000..2ec7eaa2c9
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeSerializerTest.php
@@ -0,0 +1,118 @@
+unserialize(
+ $serializer->serialize($value),
+ [Row::class, Rows::class],
+ ));
+ }
+
+ public function test_streaming_serialization_is_byte_identical_to_bulk(): void
+ {
+ $value = RowsMother::heterogeneous();
+
+ static::assertSame((new FloeSerializer(1))->serialize($value), (new FloeSerializer())->serialize($value));
+ }
+
+ public function test_round_trip_all_entry_types(): void
+ {
+ $serializer = new FloeSerializer();
+ $value = RowsMother::withAllEntryTypes();
+
+ static::assertEquals($value, $serializer->unserialize(
+ $serializer->serialize($value),
+ [Row::class, Rows::class],
+ ));
+ }
+
+ public function test_round_trip_empty_rows(): void
+ {
+ $serializer = new FloeSerializer();
+
+ static::assertEquals(rows(), $serializer->unserialize(
+ $serializer->serialize(rows()),
+ [Row::class, Rows::class],
+ ));
+ }
+
+ public function test_round_trip_partitioned_rows(): void
+ {
+ $serializer = new FloeSerializer();
+ $value = RowsMother::partitioned();
+
+ static::assertEquals($value, $serializer->unserialize(
+ $serializer->serialize($value),
+ [Row::class, Rows::class],
+ ));
+ }
+
+ public function test_round_trip_row(): void
+ {
+ $serializer = new FloeSerializer();
+ $value = row(int_entry('id', 1), str_entry('name', 'John'));
+
+ $result = $serializer->unserialize($serializer->serialize($value), [Row::class, Rows::class]);
+
+ static::assertInstanceOf(Row::class, $result);
+ static::assertEquals($value, $result);
+ }
+
+ public function test_round_trip_rows(): void
+ {
+ $serializer = new FloeSerializer();
+ $value = rows(row(int_entry('id', 1)), row(int_entry('id', 2)));
+
+ static::assertEquals($value, $serializer->unserialize(
+ $serializer->serialize($value),
+ [Row::class, Rows::class],
+ ));
+ }
+
+ public function test_serialize_rejects_other_objects(): void
+ {
+ $this->expectException(SerializationException::class);
+ $this->expectExceptionMessage('FloeSerializer supports only Rows and Row');
+
+ (new FloeSerializer())->serialize(new stdClass());
+ }
+
+ public function test_unserialize_rejects_torn_bytes(): void
+ {
+ $this->expectException(SerializationException::class);
+
+ (new FloeSerializer())->unserialize('not a floe file', [Row::class, Rows::class]);
+ }
+
+ public function test_unserialize_rejects_unexpected_class(): void
+ {
+ $serializer = new FloeSerializer();
+
+ $this->expectException(SerializationException::class);
+ $this->expectExceptionMessage('must return instance of');
+
+ $serializer->unserialize($serializer->serialize(rows(row(int_entry('id', 1)))), [Row::class]);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeValueRoundTripTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeValueRoundTripTest.php
new file mode 100644
index 0000000000..197557c2ce
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeValueRoundTripTest.php
@@ -0,0 +1,288 @@
+= 8.4')]
+ public function test_html_entries(): void
+ {
+ $rows = rows(row(html_entry('html', 'hello
')));
+
+ static::assertSame(
+ $rows->first()->entries()['html']->toString(),
+ FloeFileContext::roundTrip($rows)->first()->entries()['html']->toString(),
+ );
+ }
+
+ public function test_interval_entries(): void
+ {
+ $negative = new DateInterval('PT5H30M');
+ // @mago-ignore analysis:invalid-property-write
+ $negative->invert = 1;
+
+ $fractional = new DateInterval('PT1S');
+ // @mago-ignore analysis:invalid-property-write
+ $fractional->f = 0.123456;
+
+ static::assertEquals(
+ $rows = rows(row(
+ time_entry('time', new DateInterval('PT2H30M15S')),
+ time_entry('negative', $negative),
+ time_entry('fractional', $fractional),
+ time_entry('days', new DateInterval('P3D')),
+ )),
+ FloeFileContext::roundTrip($rows),
+ );
+ }
+
+ public function test_json_entries(): void
+ {
+ static::assertEquals(
+ $rows = rows(row(
+ json_entry('list', '[1,2,3]'),
+ json_object_entry('object', '{"a":1,"b":[true,null]}'),
+ json_object_entry('empty_object', '{}'),
+ json_entry('empty_list', '[]'),
+ )),
+ FloeFileContext::roundTrip($rows),
+ );
+ }
+
+ public function test_list_entries(): void
+ {
+ static::assertEquals(
+ $rows = rows(row(
+ list_entry('int', [1, -2, PHP_INT_MAX], type_list(type_integer())),
+ list_entry('int_empty', [], type_list(type_integer())),
+ list_entry('float', [1.5, -2.25], type_list(type_float())),
+ list_entry('float_empty', [], type_list(type_float())),
+ list_entry('string', ['a', 'b', ''], type_list(type_string())),
+ list_entry('bool', [true, false], type_list(type_boolean())),
+ list_entry('nullable_int', [1, null, 3], type_list(type_optional(type_integer()))),
+ list_entry('mixed', [1, 'two', 3.5, null, true, ['nested' => 'array']], type_list(type_mixed())),
+ list_entry('list_of_lists', [[1, 2], [3]], type_list(type_list(type_integer()))),
+ )),
+ FloeFileContext::roundTrip($rows),
+ );
+ }
+
+ public function test_map_entries(): void
+ {
+ static::assertEquals(
+ $rows = rows(row(
+ map_entry('string_keys', ['a' => 1, 'b' => 2], type_map(type_string(), type_integer())),
+ map_entry('int_keys', [10 => 'x', 20 => 'y'], type_map(type_integer(), type_string())),
+ map_entry(
+ 'mixed_values',
+ ['a' => 1, 'b' => [1, 2, ['deep' => true]]],
+ type_map(type_string(), type_mixed()),
+ ),
+ map_entry('empty', [], type_map(type_string(), type_integer())),
+ )),
+ FloeFileContext::roundTrip($rows),
+ );
+ }
+
+ public function test_mixed_values_with_uuid_json_and_datetime(): void
+ {
+ static::assertEquals(
+ $rows = rows(row(list_entry(
+ 'mixed_objects',
+ [
+ new Uuid('0196aecb-b568-7e57-a381-8ec8d3e4a531'),
+ new Json('[1,2]'),
+ new DateTimeImmutable('2025-01-01 00:00:00 UTC'),
+ ],
+ type_list(type_mixed()),
+ ))),
+ FloeFileContext::roundTrip($rows),
+ );
+ }
+
+ public function test_scalar_entries(): void
+ {
+ static::assertEquals(
+ $rows = rows(row(
+ int_entry('int', 42),
+ int_entry('int_min', PHP_INT_MIN),
+ int_entry('int_max', PHP_INT_MAX),
+ float_entry('float', 3.14159),
+ float_entry('float_negative', -1.0E-10),
+ bool_entry('bool_true', true),
+ bool_entry('bool_false', false),
+ str_entry('string', 'hello'),
+ str_entry('string_empty', ''),
+ str_entry('string_binary', "line\nbreak\x00null\xFFbyte"),
+ str_entry('string_unicode', 'zażółć gęślą jaźń 🚀'),
+ )),
+ FloeFileContext::roundTrip($rows),
+ );
+ }
+
+ public function test_structure_entries(): void
+ {
+ $type = type_structure([
+ 'street' => type_string(),
+ 'nested' => type_structure(['count' => type_integer(), 'tags' => type_list(type_string())]),
+ ], ['optional_zip' => type_string()]);
+
+ static::assertEquals(
+ $rows = rows(row(
+ structure_entry(
+ 'full',
+ ['street' => 'Main', 'nested' => ['count' => 5, 'tags' => ['a']], 'optional_zip' => '00-001'],
+ $type,
+ ),
+ structure_entry(
+ 'without_optional',
+ ['street' => 'Side', 'nested' => ['count' => 0, 'tags' => []]],
+ $type,
+ ),
+ // @mago-ignore analysis:possibly-invalid-argument
+ structure_entry(
+ 'nullable_element',
+ ['street' => null, 'nested' => ['count' => 1, 'tags' => []]],
+ type_structure([
+ 'street' => type_optional(type_string()),
+ 'nested' => type_structure(['count' => type_integer(), 'tags' => type_list(type_string())]),
+ ]),
+ ),
+ structure_entry(
+ 'with_extra',
+ ['id' => 1, 'custom' => 'x', 'more' => [1, 2]],
+ type_structure(['id' => type_integer()], [], true),
+ ),
+ structure_entry('with_timezone', ['tz' => new DateTimeZone('Europe/Warsaw')], type_structure([
+ 'tz' => type_time_zone(),
+ ])),
+ structure_entry('with_null_type', ['nothing' => null], type_structure(['nothing' => type_null()])),
+ )),
+ FloeFileContext::roundTrip($rows),
+ );
+ }
+
+ public function test_uuid_entries(): void
+ {
+ static::assertEquals(
+ $rows = rows(row(uuid_entry('uuid', '0196aecb-b568-7e57-a381-8ec8d3e4a531'))),
+ FloeFileContext::roundTrip($rows),
+ );
+ }
+
+ public function test_xml_entries(): void
+ {
+ static::assertEquals(
+ $rows = rows(row(
+ xml_entry('xml', 'text & entity'),
+ xml_element_entry('element', '- value
'),
+ )),
+ FloeFileContext::roundTrip($rows),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeValueSerializerTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeValueSerializerTest.php
new file mode 100644
index 0000000000..50a233d913
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeValueSerializerTest.php
@@ -0,0 +1,163 @@
+
+ */
+ public static function values(): array
+ {
+ return [
+ 'row' => [row(int_entry('id', 1), str_entry('name', 'John'))],
+ 'rows' => [rows(row(int_entry('id', 1)), row(int_entry('id', 2)))],
+ 'partitioned rows' => [RowsMother::partitioned()],
+ 'heterogeneous rows' => [RowsMother::heterogeneous()],
+ 'all entry types' => [RowsMother::withAllEntryTypes()],
+ 'empty rows' => [rows()],
+ 'empty partitioned rows' => [Rows::partitioned([], [new Partition('country', 'PL')])],
+ ];
+ }
+
+ #[DataProvider('values')]
+ public function test_batch_size_does_not_change_bytes(Row|Rows $value): void
+ {
+ static::assertSame((new FloeValueSerializer(1))->encode($value), (new FloeValueSerializer())->encode($value));
+ }
+
+ #[DataProvider('values')]
+ public function test_batch_sizes_decode_each_others_bytes(Row|Rows $value): void
+ {
+ $small = new FloeValueSerializer(1);
+ $default = new FloeValueSerializer();
+
+ static::assertEquals($value, $default->decode($small->encode($value)));
+ static::assertEquals($value, $small->decode($default->encode($value)));
+ }
+
+ public function test_decode_rejects_torn_bytes(): void
+ {
+ $this->expectException(FloeException::class);
+
+ (new FloeValueSerializer())->decode('not a valid floe payload');
+ }
+
+ public function test_decode_rejects_payload_smaller_than_header_and_trailer(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('too small');
+
+ (new FloeValueSerializer())->decode('tiny');
+ }
+
+ public function test_decode_rejects_footer_that_does_not_fit(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('footer does not fit');
+
+ (new FloeValueSerializer())->decode(Format::header(0x00) . Format::trailer(1000));
+ }
+
+ public function test_batch_size_smaller_than_row_count(): void
+ {
+ $codec = new FloeValueSerializer(3);
+ $value = rows(...array_map(static fn(int $id): Row => row(int_entry('id', $id)), range(1, 10)));
+
+ static::assertSame((new FloeValueSerializer())->encode($value), $codec->encode($value));
+ static::assertEquals($value, $codec->decode($codec->encode($value)));
+ }
+
+ public function test_batch_size_larger_than_row_count(): void
+ {
+ $codec = new FloeValueSerializer(100);
+ $value = rows(row(int_entry('id', 1)), row(int_entry('id', 2)));
+
+ static::assertEquals($value, $codec->decode($codec->encode($value)));
+ }
+
+ #[DataProvider('values')]
+ public function test_round_trip(Row|Rows $value): void
+ {
+ $codec = new FloeValueSerializer();
+
+ static::assertEquals($value, $codec->decode($codec->encode($value)));
+ }
+
+ public function test_heterogeneous_rows_round_trip_exactly_unpadded(): void
+ {
+ $codec = new FloeValueSerializer();
+ // rows with different schemas must come back as written - NOT padded to a merged schema
+ $value = rows(
+ row(int_entry('id', 1)),
+ row(int_entry('id', 2), str_entry('name', 'John')),
+ row(str_entry('name', 'Jane')),
+ );
+
+ static::assertEquals($value, $codec->decode($codec->encode($value)));
+ }
+
+ public function test_round_trip_all_entry_types(): void
+ {
+ $codec = new FloeValueSerializer();
+ $value = RowsMother::withAllEntryTypes();
+
+ static::assertEquals($value, $codec->decode($codec->encode($value)));
+ }
+
+ public function test_round_trip_empty_rows(): void
+ {
+ $codec = new FloeValueSerializer();
+
+ static::assertEquals(rows(), $codec->decode($codec->encode(rows())));
+ }
+
+ public function test_round_trip_partitioned_rows(): void
+ {
+ $codec = new FloeValueSerializer();
+ $value = RowsMother::partitioned();
+
+ static::assertEquals($value, $codec->decode($codec->encode($value)));
+ }
+
+ public function test_round_trip_row(): void
+ {
+ $codec = new FloeValueSerializer();
+ $value = row(int_entry('id', 1), str_entry('name', 'John'));
+
+ $result = $codec->decode($codec->encode($value));
+
+ static::assertInstanceOf(Row::class, $result);
+ static::assertEquals($value, $result);
+ }
+
+ public function test_round_trip_rows(): void
+ {
+ $codec = new FloeValueSerializer();
+ $value = rows(row(int_entry('id', 1)), row(int_entry('id', 2)));
+
+ $result = $codec->decode($codec->encode($value));
+
+ static::assertInstanceOf(Rows::class, $result);
+ static::assertEquals($value, $result);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeWriterTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeWriterTest.php
new file mode 100644
index 0000000000..6b0172c9e8
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeWriterTest.php
@@ -0,0 +1,512 @@
+append($path);
+ $writer->write(rows(row(int_entry('id', 1))));
+ $writer->close();
+
+ static::assertSame(1, FloeFileContext::footer($filesystem, $path)->totalRows);
+ }
+
+ public function test_append_on_empty_file_behaves_like_create(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://zero-bytes.floe');
+ $filesystem->writeTo($path)->close();
+
+ $writer = new FloeWriter($filesystem);
+ $writer->append($path);
+ $writer->write(rows(row(int_entry('id', 1))));
+ $writer->close();
+
+ static::assertSame(1, FloeFileContext::footer($filesystem, $path)->totalRows);
+ }
+
+ public function test_append_over_unsized_stream_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://unsized.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('requires a sized stream');
+
+ (new FloeWriter(new UnsizedFilesystem($filesystem)))->append($path);
+ }
+
+ public function test_append_with_footer_length_exceeding_file_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://bad-footer-length.floe');
+ $stream = $filesystem->writeTo($path);
+ $stream->append(Format::header(0x00) . Format::trailer(9999));
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('footer does not fit inside the file');
+
+ (new FloeWriter($filesystem))->append($path);
+ }
+
+ public function test_append_metadata_merges_over_existing_footer_metadata(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://meta.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path, Metadata::fromArray(['source' => 'create', 'kept' => 'yes']));
+ $writer->close();
+ $writer = new FloeWriter($filesystem);
+ $writer->append($path, Metadata::fromArray(['source' => 'append']));
+ $writer->close();
+
+ static::assertSame(
+ ['source' => 'append', 'kept' => 'yes'],
+ FloeFileContext::footer($filesystem, $path)->metadata->normalize(),
+ );
+ }
+
+ public function test_append_to_file_with_different_codec_flags_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://flags.floe');
+ $stream = $filesystem->writeTo($path);
+ $stream->append(Format::header(0x05) . Format::trailer(0));
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('written with codec 0x05');
+
+ (new FloeWriter($filesystem))->append($path);
+ }
+
+ public function test_append_to_torn_file_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://torn.floe');
+ $stream = $filesystem->writeTo($path);
+ $stream->append(Format::header(0x00) . Format::frame(Format::FRAME_ROW, 'not closed'));
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('torn');
+
+ (new FloeWriter($filesystem))->append($path);
+ }
+
+ public function test_append_to_truncated_stub_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://stub.floe');
+ $stream = $filesystem->writeTo($path);
+ $stream->append('FLOE');
+ $stream->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('too small');
+
+ (new FloeWriter($filesystem))->append($path);
+ }
+
+ public function test_append_with_incompatible_schema_keeps_previous_rows_readable(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://safe.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1))));
+ $writer->close();
+
+ $writer = new FloeWriter($filesystem);
+ $writer->append($path);
+
+ try {
+ $writer->write(rows(row(str_entry('id', 'no longer an int'))));
+ static::fail('expected ' . IncompatibleSchemaException::class);
+ } catch (IncompatibleSchemaException) {
+ }
+
+ $writer->close();
+
+ $footer = FloeFileContext::footer($filesystem, $path);
+
+ static::assertSame(1, $footer->totalRows);
+ static::assertCount(1, $footer->sections);
+ }
+
+ public function test_append_with_new_non_nullable_column_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://evolve.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1))));
+ $writer->close();
+
+ $this->expectException(IncompatibleSchemaException::class);
+ $this->expectExceptionMessage('adds new column "email" which must be nullable');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->append($path);
+ $writer->write(rows(row(int_entry('id', 2), str_entry('email', 'x'))));
+ }
+
+ public function test_append_with_new_nullable_column_merges_file_schema(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://merge.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1))));
+ $writer->close();
+
+ $writer = new FloeWriter($filesystem);
+ $writer->append($path);
+ $writer->write(rows(row(int_entry('id', 2), str_entry('email', null))));
+ $writer->close();
+
+ $footer = FloeFileContext::footer($filesystem, $path);
+ $email = $footer->fileSchema()->findDefinition('email');
+
+ static::assertSame(2, $footer->totalRows);
+ static::assertCount(2, $footer->schemas);
+ static::assertCount(2, $footer->sections);
+ static::assertNotNull($email);
+ static::assertTrue($email->isNullable());
+ }
+
+ public function test_append_with_same_schema_starts_section_without_schema_frame(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://same-schema.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('id', 1))));
+ $writer->close();
+
+ $writer = new FloeWriter($filesystem);
+ $writer->append($path);
+ $writer->write(rows(row(int_entry('id', 2))));
+ $writer->close();
+
+ static::assertSame(
+ [Format::FRAME_SCHEMA, Format::FRAME_ROW, Format::FRAME_FOOTER, Format::FRAME_ROW, Format::FRAME_FOOTER],
+ FloeFileContext::frameTypes($filesystem, $path),
+ );
+
+ $footer = FloeFileContext::footer($filesystem, $path);
+
+ static::assertSame(2, $footer->totalRows);
+ static::assertCount(1, $footer->schemas);
+ static::assertCount(2, $footer->sections);
+ static::assertSame($footer->sections[0]->schemaId, $footer->sections[1]->schemaId);
+ }
+
+ public function test_close_twice_throws(): void
+ {
+ $writer = new FloeWriter(memory_filesystem());
+ $writer->create(path('memory://closed.floe'));
+ $writer->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('Floe writer session is not open');
+
+ $writer->close();
+ }
+
+ public function test_opening_a_second_session_throws(): void
+ {
+ $writer = new FloeWriter(memory_filesystem());
+ $writer->create(path('memory://first.floe'));
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('Floe writer session is already open');
+
+ $writer->create(path('memory://second.floe'));
+ }
+
+ public function test_closing_empty_file_writes_header_and_footer_only(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://empty.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->close();
+
+ static::assertSame([Format::FRAME_FOOTER], FloeFileContext::frameTypes($filesystem, $path));
+
+ $footer = FloeFileContext::footer($filesystem, $path);
+
+ static::assertSame(0, $footer->totalRows);
+ static::assertSame([], $footer->sections);
+ static::assertSame([], $footer->schemas);
+ static::assertSame([], $footer->fileSchema);
+ static::assertSame([], $footer->partitions);
+ }
+
+ public function test_create_with_non_noop_codec_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('supports only the no-op codec, got codec 0x05');
+
+ (new FloeWriter(memory_filesystem(), new CodecStub(0x05)))->create(path('memory://codec.floe'));
+ }
+
+ public function test_open_on_stream_is_byte_identical_to_create(): void
+ {
+ $filesystem = memory_filesystem();
+ $viaCreate = path('memory://via-create.floe');
+ $viaStream = path('memory://via-stream.floe');
+ $data = rows(row(int_entry('id', 1), str_entry('name', 'a')), row(int_entry('id', 2), str_entry('name', 'b')));
+
+ $create = new FloeWriter($filesystem);
+ $create->create($viaCreate);
+ $create->write($data);
+ $create->close();
+
+ $onStream = new FloeWriter($filesystem);
+ $onStream->createOnStream($filesystem->writeTo($viaStream));
+ $onStream->write($data);
+ $onStream->close();
+
+ static::assertSame($filesystem->readFrom($viaCreate)->content(), $filesystem->readFrom($viaStream)->content());
+ }
+
+ public function test_open_on_stream_round_trips(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://on-stream.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->createOnStream($filesystem->writeTo($path), Metadata::fromArray(['source' => 'stream']));
+ $writer->write(rows(row(int_entry('id', 1)), row(int_entry('id', 2)), row(int_entry('id', 3))));
+ $writer->close();
+
+ $footer = FloeFileContext::footer($filesystem, $path);
+
+ static::assertSame(3, $footer->totalRows);
+ static::assertSame(['source' => 'stream'], $footer->metadata->normalize());
+ static::assertSame(
+ [Format::FRAME_SCHEMA, Format::FRAME_ROW, Format::FRAME_ROW, Format::FRAME_ROW, Format::FRAME_FOOTER],
+ FloeFileContext::frameTypes($filesystem, $path),
+ );
+ }
+
+ public function test_open_on_stream_with_non_noop_codec_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('supports only the no-op codec, got codec 0x05');
+
+ (new FloeWriter(memory_filesystem(), new CodecStub(0x05)))->createOnStream(memory_filesystem()->writeTo(path(
+ 'memory://on-stream-codec.floe',
+ )));
+ }
+
+ public function test_nothing_reaches_the_stream_before_buffer_fills_or_close(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://buffered.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(str_entry('data', 'x'))));
+
+ static::assertSame(0, $filesystem->readFrom($path)->size());
+
+ $writer->write(rows(row(str_entry('data', str_repeat('x', 70_000)))));
+
+ static::assertGreaterThan(0, $filesystem->readFrom($path)->size());
+ }
+
+ public function test_partitioned_rows_write_partitions_frame_after_header(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://partitioned.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(Rows::partitioned([row(int_entry('id', 1), str_entry('country', 'PL'))], [new Partition(
+ 'country',
+ 'PL',
+ )]));
+ $writer->close();
+
+ static::assertSame(
+ [Format::FRAME_PARTITIONS, Format::FRAME_SCHEMA, Format::FRAME_ROW, Format::FRAME_FOOTER],
+ FloeFileContext::frameTypes($filesystem, $path),
+ );
+ static::assertSame(['country' => 'PL'], FloeFileContext::footer($filesystem, $path)->partitions);
+ }
+
+ public function test_partitions_fixed_empty_reject_partitioned_writes(): void
+ {
+ $writer = new FloeWriter(memory_filesystem());
+ $writer->create(path('memory://fixed-empty.floe'));
+ $writer->write(rows(row(int_entry('id', 1))));
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('one partition combination');
+
+ $writer->write(Rows::partitioned([row(int_entry('id', 2), str_entry('country', 'PL'))], [new Partition(
+ 'country',
+ 'PL',
+ )]));
+ }
+
+ public function test_partitions_mismatch_on_append_throws(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://append-partitions.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(Rows::partitioned([row(int_entry('id', 1), str_entry('country', 'PL'))], [new Partition(
+ 'country',
+ 'PL',
+ )]));
+ $writer->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('one partition combination');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->append($path);
+ $writer->write(Rows::partitioned([row(int_entry('id', 2), str_entry('country', 'US'))], [new Partition(
+ 'country',
+ 'US',
+ )]));
+ }
+
+ public function test_a_new_column_grows_a_section_and_narrower_rows_ride_it(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://sections.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(
+ // section {a,b}; then c is new -> grow to section {a,b,c}; then {a,b} fits it (c absent)
+ row(int_entry('a', 1), str_entry('b', 'x')),
+ row(int_entry('a', 2), str_entry('c', 'y')),
+ row(int_entry('a', 3), str_entry('b', 'z')),
+ ));
+ $writer->close();
+
+ $footer = FloeFileContext::footer($filesystem, $path);
+
+ static::assertCount(2, $footer->schemas);
+ static::assertCount(2, $footer->sections);
+ static::assertSame([0, 1], [
+ $footer->sections[0]->schemaId,
+ $footer->sections[1]->schemaId,
+ ]);
+ static::assertSame([1, 2], [
+ $footer->sections[0]->rowCount,
+ $footer->sections[1]->rowCount,
+ ]);
+ static::assertSame(3, $footer->totalRows);
+ }
+
+ public function test_section_offsets_point_at_frames(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://offsets.floe');
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path);
+ $writer->write(rows(row(int_entry('a', 1)), row(str_entry('b', 'x'))));
+ $writer->close();
+
+ $footer = FloeFileContext::footer($filesystem, $path);
+ $source = $filesystem->readFrom($path);
+
+ static::assertSame(Format::HEADER_LENGTH, $footer->sections[0]->offset);
+
+ foreach ($footer->sections as $section) {
+ static::assertSame(Format::FRAME_SCHEMA, ord($source->read(1, $section->offset)));
+ }
+ }
+
+ public function test_write_after_close_throws(): void
+ {
+ $writer = new FloeWriter(memory_filesystem());
+ $writer->create(path('memory://done.floe'));
+ $writer->close();
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('Floe writer session is not open');
+
+ $writer->write(rows(row(int_entry('id', 1))));
+ }
+
+ public function test_grow_section_plan_from_null_uses_the_row_schema(): void
+ {
+ $plan = FloeWriter::growSectionPlan(null, row(int_entry('a', 1), str_entry('b', 'x')));
+
+ static::assertSame(['a', 'b'], array_values(array_map(static fn($column) => $column->name, $plan->columns)));
+ }
+
+ public function test_grow_section_plan_grows_the_union_preserving_order(): void
+ {
+ $base = FloeWriter::growSectionPlan(null, row(int_entry('a', 1), str_entry('b', 'x')));
+
+ $grown = FloeWriter::growSectionPlan($base->schemaBody, row(int_entry('a', 2), float_entry('c', 1.5)));
+
+ static::assertSame(
+ ['a', 'b', 'c'],
+ array_values(array_map(static fn($column) => $column->name, $grown->columns)),
+ );
+ }
+
+ public function test_grow_section_plan_rides_a_narrower_row_without_growing(): void
+ {
+ $base = FloeWriter::growSectionPlan(null, row(int_entry('a', 1), str_entry('b', 'x')));
+
+ $grown = FloeWriter::growSectionPlan($base->schemaBody, row(int_entry('a', 2)));
+
+ static::assertSame($base->schemaBody, $grown->schemaBody);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FooterTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FooterTest.php
new file mode 100644
index 0000000000..b12b929f7f
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FooterTest.php
@@ -0,0 +1,157 @@
+toJson();
+
+ static::assertStringContainsString('"partitions":{}', $json);
+ static::assertStringContainsString('"metadata":{}', $json);
+ static::assertStringContainsString('"schemas":[]', $json);
+ static::assertStringContainsString('"sections":[]', $json);
+ }
+
+ public function test_file_schema_round_trips_through_schema_object(): void
+ {
+ $schema = schema(int_schema('id'), str_schema('name', nullable: true));
+ $footer = FooterMother::footer(fileSchema: $schema->normalize());
+
+ static::assertTrue($schema->isSame(Footer::fromJson($footer->toJson())->fileSchema()));
+ }
+
+ public function test_from_json_requires_all_fields(): void
+ {
+ $this->expectException(FloeException::class);
+
+ Footer::fromJson(
+ '{"version":1,"writer":"x","schemas":[],"fileSchema":[],"partitions":{},"totalRows":0,"metadata":{}}',
+ );
+ }
+
+ public function test_from_json_with_malformed_json_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('failed to decode footer JSON');
+
+ Footer::fromJson('{broken');
+ }
+
+ public function test_from_json_with_non_integer_version_throws(): void
+ {
+ $this->expectException(FloeException::class);
+
+ Footer::fromJson('{"version":"1"}');
+ }
+
+ public function test_from_json_with_non_integer_total_rows_throws(): void
+ {
+ $this->expectException(FloeException::class);
+
+ Footer::fromJson('{"version":1,"totalRows":"0"}');
+ }
+
+ public function test_from_json_with_non_object_footer_throws(): void
+ {
+ $this->expectException(FloeException::class);
+
+ Footer::fromJson('"scalar"');
+ }
+
+ public function test_from_json_with_non_object_section_throws(): void
+ {
+ /** @var array $footer */
+ $footer = json_decode(FooterMother::footer()->toJson(), true, 512, JSON_THROW_ON_ERROR);
+ $footer['sections'] = [1];
+
+ $this->expectException(FloeException::class);
+
+ Footer::fromJson(json_encode($footer, JSON_THROW_ON_ERROR));
+ }
+
+ public function test_from_json_with_non_string_writer_throws(): void
+ {
+ $this->expectException(FloeException::class);
+
+ Footer::fromJson('{"version":1,"totalRows":0,"writer":42}');
+ }
+
+ public function test_from_json_round_trips_typed_metadata(): void
+ {
+ $footer = FooterMother::footer(metadata: [
+ 'row_count' => 42,
+ 'ratio' => 1.5,
+ 'sorted' => true,
+ 'source' => 'bucket-7',
+ 'id_stats' => ['min' => 1, 'max' => 1000],
+ ]);
+
+ $decoded = Footer::fromJson($footer->toJson());
+
+ static::assertEquals($footer, $decoded);
+ static::assertSame(42, $decoded->metadata->getAs('row_count', type_integer()));
+ static::assertSame(['min' => 1, 'max' => 1000], $decoded->metadata->get('id_stats'));
+ }
+
+ public function test_from_json_with_non_scalar_metadata_value_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('metadata is malformed');
+
+ Footer::fromJson(
+ '{"version":1,"writer":"x","schemas":[],"fileSchema":[],"sections":[],"partitions":{},"totalRows":0,"metadata":{"bad":null}}',
+ );
+ }
+
+ public function test_schema_body_re_encodes_stored_schema(): void
+ {
+ $schema = schema(int_schema('id'));
+ $footer = FooterMother::footer(schemas: [$schema->normalize()]);
+
+ static::assertJsonStringEqualsJsonString(
+ json_encode($schema->normalize(), JSON_THROW_ON_ERROR),
+ Footer::fromJson($footer->toJson())->schemaBody(0),
+ );
+ }
+
+ public function test_schema_body_with_unknown_id_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('does not hold schema with id 3');
+
+ FooterMother::footer()->schemaBody(3);
+ }
+
+ public function test_to_json_round_trips_every_field(): void
+ {
+ $footer = FooterMother::footer(
+ schemas: [schema(int_schema('id'))->normalize()],
+ fileSchema: schema(int_schema('id'))->normalize(),
+ sections: [new Section(6, 0, 2), new Section(120, 0, 3)],
+ partitions: ['country' => 'PL'],
+ totalRows: 5,
+ metadata: ['source' => 'test'],
+ );
+
+ static::assertEquals($footer, Footer::fromJson($footer->toJson()));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FormatTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FormatTest.php
new file mode 100644
index 0000000000..bc82fc6118
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FormatTest.php
@@ -0,0 +1,101 @@
+expectException(FloeException::class);
+ $this->expectExceptionMessage('magic');
+
+ Format::parseTrailer(pack('V', 10) . 'NOPE');
+ }
+
+ public function test_parse_trailer_with_wrong_length_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('exactly 8 bytes');
+
+ Format::parseTrailer('FLOE');
+ }
+
+ public function test_trailer_is_footer_length_and_magic(): void
+ {
+ static::assertSame(pack('V', 42) . 'FLOE', Format::trailer(42));
+ static::assertSame(Format::TRAILER_LENGTH, strlen(Format::trailer(42)));
+ }
+
+ public function test_validate_codec_id_accepts_noop(): void
+ {
+ Format::validateCodecId(0x00);
+
+ $this->addToAssertionCount(1);
+ }
+
+ public function test_validate_codec_id_rejects_other_codecs(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('supports only the no-op codec');
+
+ Format::validateCodecId(0x01);
+ }
+
+ public function test_validate_header_returns_flags(): void
+ {
+ static::assertSame(0x7F, Format::validateHeader(Format::header(0x7F)));
+ }
+
+ public function test_validate_header_with_unknown_magic_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('magic');
+
+ Format::validateHeader('NOPE' . "\x01\x00");
+ }
+
+ public function test_validate_header_with_unsupported_version_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('version');
+
+ Format::validateHeader(Format::MAGIC . "\x7F\x00");
+ }
+
+ public function test_validate_truncated_header_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('header is incomplete');
+
+ Format::validateHeader('FLO');
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameReaderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameReaderTest.php
new file mode 100644
index 0000000000..5fbdee3a21
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameReaderTest.php
@@ -0,0 +1,127 @@
+frames(lenient: true)));
+ }
+
+ public function test_empty_stream_in_strict_mode_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('header is incomplete');
+
+ iterator_to_array(FrameReaderMother::overBytes('')->frames());
+ }
+
+ public function test_flags_mismatch_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('written with codec 0x05, expected 0x00');
+
+ iterator_to_array(FrameReaderMother::overBytes(Format::header(0x05))->frames());
+ }
+
+ public function test_frames_crossing_compaction_threshold_are_streamed(): void
+ {
+ $body = str_repeat('x', 8192);
+ $bytes = Format::header(0x00);
+
+ for ($i = 0; $i < 200; $i++) {
+ $bytes .= Format::frame(Format::FRAME_ROW, $body);
+ }
+
+ $read = 0;
+
+ foreach (FrameReaderMother::overBytes($bytes, chunkSize: 4096)->frames() as [$type, $frameBody]) {
+ static::assertSame(Format::FRAME_ROW, $type);
+ static::assertSame($body, $frameBody);
+ $read++;
+ }
+
+ static::assertSame(200, $read);
+ }
+
+ public function test_frames_are_yielded_in_order_with_bodies(): void
+ {
+ $bytes =
+ Format::header(0x00)
+ . Format::frame(Format::FRAME_SCHEMA, 'schema')
+ . Format::frame(Format::FRAME_ROW, 'row-1')
+ . Format::frame(Format::FRAME_FOOTER, 'footer');
+
+ static::assertSame(
+ [
+ [Format::FRAME_SCHEMA, 'schema'],
+ [Format::FRAME_ROW, 'row-1'],
+ [Format::FRAME_FOOTER, 'footer'],
+ ],
+ iterator_to_array(FrameReaderMother::overBytes($bytes)->frames()),
+ );
+ }
+
+ public function test_invalid_magic_throws_even_in_lenient_mode(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('magic');
+
+ iterator_to_array(FrameReaderMother::overBytes('NOPE' . "\x01\x00")->frames(lenient: true));
+ }
+
+ public function test_stream_ending_at_frame_boundary_ends_cleanly(): void
+ {
+ $bytes = Format::header(0x00) . Format::frame(Format::FRAME_ROW, 'row');
+
+ static::assertCount(1, iterator_to_array(FrameReaderMother::overBytes($bytes)->frames()));
+ }
+
+ public function test_truncated_frame_body_in_lenient_mode_salvages_previous_frames(): void
+ {
+ $bytes =
+ Format::header(0x00)
+ . Format::frame(Format::FRAME_ROW, 'complete')
+ . chr(Format::FRAME_ROW)
+ . pack('V', 100)
+ . 'short';
+
+ static::assertSame(
+ [[Format::FRAME_ROW, 'complete']],
+ iterator_to_array(FrameReaderMother::overBytes($bytes)->frames(lenient: true)),
+ );
+ }
+
+ public function test_truncated_frame_body_in_strict_mode_throws(): void
+ {
+ $bytes = Format::header(0x00) . chr(Format::FRAME_ROW) . pack('V', 100) . 'short';
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('frame body is incomplete');
+
+ iterator_to_array(FrameReaderMother::overBytes($bytes)->frames());
+ }
+
+ public function test_truncated_frame_header_in_strict_mode_throws(): void
+ {
+ $bytes = Format::header(0x00) . chr(Format::FRAME_ROW) . 'xy';
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('frame header is incomplete');
+
+ iterator_to_array(FrameReaderMother::overBytes($bytes)->frames());
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameSegmentTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameSegmentTest.php
new file mode 100644
index 0000000000..e8341301e2
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameSegmentTest.php
@@ -0,0 +1,29 @@
+schemaBody);
+ static::assertSame("\x02\x03\x00\x00\x00abc", $segment->frames);
+ static::assertSame(1, $segment->rowCount);
+ }
+
+ public function test_exposes_a_continuation_segment(): void
+ {
+ $segment = new FrameSegment(null, '', 0);
+
+ static::assertNull($segment->schemaBody);
+ static::assertSame('', $segment->frames);
+ static::assertSame(0, $segment->rowCount);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/PhpRowFrameEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/PhpRowFrameEncoderTest.php
new file mode 100644
index 0000000000..653a51adfc
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/PhpRowFrameEncoderTest.php
@@ -0,0 +1,131 @@
+encode($plan, $first);
+ $secondBody = (new RowEncoder())->encode($plan, $second);
+
+ $segments = (new PhpRowFrameEncoder())->encode(rows($first, $second));
+
+ static::assertCount(1, $segments);
+ static::assertSame($plan->schemaBody, $segments[0]->schemaBody);
+ static::assertSame(2, $segments[0]->rowCount);
+ static::assertSame(
+ chr(Format::FRAME_ROW)
+ . pack('V', strlen($firstBody))
+ . $firstBody
+ . chr(Format::FRAME_ROW)
+ . pack('V', strlen($secondBody))
+ . $secondBody,
+ $segments[0]->frames,
+ );
+ }
+
+ public function test_new_column_starts_a_new_segment_with_the_grown_union(): void
+ {
+ $first = row(int_entry('id', 1));
+ $second = row(int_entry('id', 2), str_entry('name', 'flow'));
+
+ $firstPlan = FloeWriter::growSectionPlan(null, $first);
+ $secondPlan = FloeWriter::growSectionPlan($firstPlan->schemaBody, $second);
+ $firstBody = (new RowEncoder())->encode($firstPlan, $first);
+ $secondBody = (new RowEncoder())->encode($secondPlan, $second);
+
+ $segments = (new PhpRowFrameEncoder())->encode(rows($first, $second));
+
+ static::assertCount(2, $segments);
+ static::assertSame($firstPlan->schemaBody, $segments[0]->schemaBody);
+ static::assertSame(1, $segments[0]->rowCount);
+ static::assertSame(chr(Format::FRAME_ROW) . pack('V', strlen($firstBody)) . $firstBody, $segments[0]->frames);
+ static::assertSame($secondPlan->schemaBody, $segments[1]->schemaBody);
+ static::assertSame(1, $segments[1]->rowCount);
+ static::assertSame(chr(Format::FRAME_ROW) . pack('V', strlen($secondBody)) . $secondBody, $segments[1]->frames);
+ }
+
+ public function test_narrower_row_rides_the_current_section(): void
+ {
+ $first = row(int_entry('id', 1), str_entry('name', 'flow'));
+ $second = row(int_entry('id', 2));
+
+ $plan = FloeWriter::growSectionPlan(null, $first);
+ $firstBody = (new RowEncoder())->encode($plan, $first);
+ $secondBody = (new RowEncoder())->encode($plan, $second);
+
+ $segments = (new PhpRowFrameEncoder())->encode(rows($first, $second));
+
+ static::assertCount(1, $segments);
+ static::assertSame($plan->schemaBody, $segments[0]->schemaBody);
+ static::assertSame(2, $segments[0]->rowCount);
+ static::assertSame(
+ chr(Format::FRAME_ROW)
+ . pack('V', strlen($firstBody))
+ . $firstBody
+ . chr(Format::FRAME_ROW)
+ . pack('V', strlen($secondBody))
+ . $secondBody,
+ $segments[0]->frames,
+ );
+ }
+
+ public function test_continuation_across_calls_yields_a_null_schema_body_segment(): void
+ {
+ $first = row(int_entry('id', 1));
+ $second = row(int_entry('id', 2));
+
+ $plan = FloeWriter::growSectionPlan(null, $first);
+ $secondBody = (new RowEncoder())->encode($plan, $second);
+
+ $encoder = new PhpRowFrameEncoder();
+ $encoder->encode(rows($first));
+
+ $segments = $encoder->encode(rows($second));
+
+ static::assertCount(1, $segments);
+ static::assertNull($segments[0]->schemaBody);
+ static::assertSame(1, $segments[0]->rowCount);
+ static::assertSame(chr(Format::FRAME_ROW) . pack('V', strlen($secondBody)) . $secondBody, $segments[0]->frames);
+ }
+
+ public function test_codec_is_applied_to_each_row_body(): void
+ {
+ $row = row(int_entry('id', 1));
+
+ $plan = FloeWriter::growSectionPlan(null, $row);
+ $body = "\xFF" . (new RowEncoder())->encode($plan, $row);
+
+ $segments = (new PhpRowFrameEncoder(new PrefixingCodecStub()))->encode(rows($row));
+
+ static::assertCount(1, $segments);
+ static::assertSame(chr(Format::FRAME_ROW) . pack('V', strlen($body)) . $body, $segments[0]->frames);
+ }
+
+ public function test_empty_rows_produce_no_segments(): void
+ {
+ static::assertSame([], (new PhpRowFrameEncoder())->encode(rows()));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/RowHydratorTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/RowHydratorTest.php
new file mode 100644
index 0000000000..d961193c82
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/RowHydratorTest.php
@@ -0,0 +1,76 @@
+encode($encoderPlan, $original);
+
+ $hydratorPlan = (new SchemaDecoder(
+ new ValueDecoder(),
+ new EntryInstantiator(),
+ ))->decode($encoderPlan->schemaBody);
+ $position = 0;
+ $hydrated = (new RowHydrator())->hydrate($hydratorPlan, $body, $position);
+
+ static::assertEquals($original, $hydrated);
+ static::assertSame(strlen($body), $position);
+ static::assertSame(serialize($original), serialize($hydrated));
+ }
+
+ public function test_hydrated_definitions_are_not_shared_between_rows(): void
+ {
+ $original = row(int_entry('id', 1));
+
+ $encoderPlan = FloeWriter::growSectionPlan(null, $original);
+ $body = (new RowEncoder())->encode($encoderPlan, $original);
+
+ $hydratorPlan = (new SchemaDecoder(
+ new ValueDecoder(),
+ new EntryInstantiator(),
+ ))->decode($encoderPlan->schemaBody);
+ $hydrator = new RowHydrator();
+
+ $position = 0;
+ $first = $hydrator->hydrate($hydratorPlan, $body, $position);
+ $position = 0;
+ $second = $hydrator->hydrate($hydratorPlan, $body, $position);
+
+ static::assertNotSame($first->entries()['id']->definition(), $second->entries()['id']->definition());
+ }
+
+ public function test_hydrating_row_with_unknown_value_flag_throws(): void
+ {
+ $original = row(int_entry('id', 1));
+ $schemaJson = FloeWriter::growSectionPlan(null, $original)->schemaBody;
+ $hydratorPlan = (new SchemaDecoder(new ValueDecoder(), new EntryInstantiator()))->decode($schemaJson);
+ $position = 0;
+
+ $this->expectException(SerializationException::class);
+ $this->expectExceptionMessage('unknown value flag');
+
+ (new RowHydrator())->hydrate($hydratorPlan, "\xEF", $position);
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/RowPaddingTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/RowPaddingTest.php
new file mode 100644
index 0000000000..59a35cc905
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/RowPaddingTest.php
@@ -0,0 +1,79 @@
+apply(row(int_entry('id', 1)));
+ $email = $padded->get('email');
+
+ static::assertSame(['id', 'email'], $padded->entries()->names());
+ static::assertSame(1, $padded->get('id')->value());
+ static::assertNull($email->value());
+ static::assertTrue($email->definition()->isNullable());
+ static::assertTrue($email->definition()->metadata()->has(Metadata::FROM_NULL));
+ }
+
+ public function test_padding_entries_are_shared_between_rows(): void
+ {
+ $padding = RowPadding::forFileSchema(
+ schema(int_schema('id'), str_schema('email', nullable: true)),
+ new SchemaDecoder(new ValueDecoder(), new EntryInstantiator()),
+ );
+
+ static::assertSame(
+ $padding->apply(row(int_entry('id', 1)))->get('email'),
+ $padding->apply(row(int_entry('id', 2)))->get('email'),
+ );
+ }
+
+ public function test_row_is_rebuilt_in_file_schema_order(): void
+ {
+ $padding = RowPadding::forFileSchema(
+ schema(int_schema('id'), str_schema('name')),
+ new SchemaDecoder(new ValueDecoder(), new EntryInstantiator()),
+ );
+
+ $padded = $padding->apply(row(str_entry('name', 'x'), int_entry('id', 1)));
+
+ static::assertSame(['id', 'name'], $padded->entries()->names());
+ static::assertSame(1, $padded->get('id')->value());
+ static::assertSame('x', $padded->get('name')->value());
+ }
+
+ public function test_present_columns_are_kept_as_written(): void
+ {
+ $padding = RowPadding::forFileSchema(
+ schema(int_schema('id'), str_schema('name')),
+ new SchemaDecoder(new ValueDecoder(), new EntryInstantiator()),
+ );
+
+ $original = row(int_entry('id', 1), str_entry('name', 'x'));
+ $padded = $padding->apply($original);
+
+ static::assertSame($original->get('id'), $padded->get('id'));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/RowsValueMapperTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/RowsValueMapperTest.php
new file mode 100644
index 0000000000..1e9f424653
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/RowsValueMapperTest.php
@@ -0,0 +1,195 @@
+ 'row'],
+ RowsValueMapper::metadataFor(row(int_entry('id', 1)))->normalize(),
+ );
+ }
+
+ public function test_metadata_tags_rows(): void
+ {
+ static::assertSame(
+ [RowsValueMapper::VALUE_TYPE_KEY => 'rows'],
+ RowsValueMapper::metadataFor(rows(row(int_entry('id', 1))))->normalize(),
+ );
+ }
+
+ public function test_partitions_from_follows_footer_order_without_order_metadata(): void
+ {
+ $footer = new Footer(1, 'test', [], [], [], ['country' => 'PL', 'year' => '2025'], 0, Metadata::empty());
+
+ static::assertEquals(
+ [new Partition('country', 'PL'), new Partition('year', '2025')],
+ RowsValueMapper::partitionsFrom($footer),
+ );
+ }
+
+ public function test_partitions_from_rejects_malformed_order_metadata(): void
+ {
+ $footer = new Footer(
+ 1,
+ 'test',
+ [],
+ [],
+ [],
+ ['country' => 'PL'],
+ 0,
+ Metadata::with('flow.cache.partition_order', ['not' => 'a-list']),
+ );
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('partition order metadata is malformed');
+
+ RowsValueMapper::partitionsFrom($footer);
+ }
+
+ public function test_reconstruct_defaults_to_rows_without_value_type_metadata(): void
+ {
+ $footer = new Footer(1, 'test', [], [], [], [], 1, Metadata::empty());
+
+ static::assertEquals(
+ rows(row(int_entry('id', 1))),
+ RowsValueMapper::reconstructFrom([row(int_entry('id', 1))], $footer),
+ );
+ }
+
+ public function test_reconstruct_rejects_row_tagged_payload_without_rows(): void
+ {
+ $footer = new Footer(1, 'test', [], [], [], [], 0, Metadata::with(RowsValueMapper::VALUE_TYPE_KEY, 'row'));
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('contained no rows');
+
+ RowsValueMapper::reconstructFrom([], $footer);
+ }
+
+ public function test_reconstruct_empty_rows(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://codec-empty.floe');
+ $value = rows();
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path, RowsValueMapper::metadataFor($value));
+ $writer->write(RowsValueMapper::wrap($value));
+ $writer->close();
+
+ static::assertEquals($value, FloeFileContext::reconstruct($filesystem, $path));
+ }
+
+ public function test_reconstruct_partitioned_rows(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://codec-partitioned.floe');
+ $value = Rows::partitioned([row(int_entry('id', 1), str_entry('country', 'PL'))], [new Partition(
+ 'country',
+ 'PL',
+ )]);
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path, RowsValueMapper::metadataFor($value));
+ $writer->write(RowsValueMapper::wrap($value));
+ $writer->close();
+
+ static::assertEquals($value, FloeFileContext::reconstruct($filesystem, $path));
+ }
+
+ public function test_reconstruct_preserves_non_alphabetical_partition_order(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://codec-partition-order.floe');
+ $value = Rows::partitioned([row(
+ int_entry('id', 1),
+ str_entry('year', '2020'),
+ str_entry('day', '15'),
+ str_entry('month', '03'),
+ )], [new Partition('year', '2020'), new Partition('day', '15'), new Partition('month', '03')]);
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path, RowsValueMapper::metadataFor($value));
+ $writer->write(RowsValueMapper::wrap($value));
+ $writer->close();
+
+ $result = FloeFileContext::reconstruct($filesystem, $path);
+
+ static::assertInstanceOf(Rows::class, $result);
+ static::assertSame(
+ ['year', 'day', 'month'],
+ array_map(static fn(Partition $p): string => $p->name, $result->partitions()->toArray()),
+ );
+ static::assertEquals($value, $result);
+ }
+
+ public function test_reconstruct_row(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://codec-row.floe');
+ $value = row(int_entry('id', 1), str_entry('name', 'John'));
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path, RowsValueMapper::metadataFor($value));
+ $writer->write(RowsValueMapper::wrap($value));
+ $writer->close();
+
+ $result = FloeFileContext::reconstruct($filesystem, $path);
+
+ static::assertInstanceOf(Row::class, $result);
+ static::assertEquals($value, $result);
+ }
+
+ public function test_reconstruct_rows(): void
+ {
+ $filesystem = memory_filesystem();
+ $path = path('memory://codec-rows.floe');
+ $value = rows(row(int_entry('id', 1)), row(int_entry('id', 2)));
+
+ $writer = new FloeWriter($filesystem);
+ $writer->create($path, RowsValueMapper::metadataFor($value));
+ $writer->write(RowsValueMapper::wrap($value));
+ $writer->close();
+
+ $result = FloeFileContext::reconstruct($filesystem, $path);
+
+ static::assertInstanceOf(Rows::class, $result);
+ static::assertEquals($value, $result);
+ }
+
+ public function test_wrap_keeps_rows(): void
+ {
+ $value = rows(row(int_entry('id', 1)));
+
+ static::assertSame($value, RowsValueMapper::wrap($value));
+ }
+
+ public function test_wrap_lifts_a_row_into_rows(): void
+ {
+ static::assertEquals(rows(row(int_entry('id', 1))), RowsValueMapper::wrap(row(int_entry('id', 1))));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaDecoderTest.php
new file mode 100644
index 0000000000..9fcc301f31
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaDecoderTest.php
@@ -0,0 +1,73 @@
+schemaBody;
+
+ $plan = (new SchemaDecoder(new ValueDecoder(), new EntryInstantiator()))->decode($schemaJson);
+
+ static::assertCount(1, $plan);
+ static::assertSame('name', $plan[0]->name);
+ static::assertFalse($plan[0]->definition->isNullable());
+ static::assertTrue($plan[0]->nullableDefinition->isNullable());
+ static::assertTrue($plan[0]->fromNullDefinition->isNullable());
+ static::assertTrue($plan[0]->fromNullDefinition->metadata()->has(Metadata::FROM_NULL));
+ static::assertFalse($plan[0]->nullableDefinition->metadata()->has(Metadata::FROM_NULL));
+ }
+
+ public function test_decoded_plan_strips_from_null_marker_from_base_definition(): void
+ {
+ $schemaJson = FloeWriter::growSectionPlan(null, row(StringEntry::fromNull('name')))->schemaBody;
+
+ $plan = (new SchemaDecoder(new ValueDecoder(), new EntryInstantiator()))->decode($schemaJson);
+
+ static::assertFalse($plan[0]->definition->metadata()->has(Metadata::FROM_NULL));
+ static::assertTrue($plan[0]->fromNullDefinition->metadata()->has(Metadata::FROM_NULL));
+ }
+
+ public function test_decoded_plan_preserves_custom_metadata(): void
+ {
+ $schemaJson = FloeWriter::growSectionPlan(null, row(int_entry('id', 1, Metadata::fromArray([
+ 'custom' => 'meta',
+ ]))))->schemaBody;
+
+ $plan = (new SchemaDecoder(new ValueDecoder(), new EntryInstantiator()))->decode($schemaJson);
+
+ static::assertSame('meta', $plan[0]->definition->metadata()->get('custom'));
+ }
+
+ public function test_decoding_invalid_json_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('failed to decode schema JSON');
+
+ (new SchemaDecoder(new ValueDecoder(), new EntryInstantiator()))->decode('{invalid');
+ }
+
+ public function test_decoding_non_list_json_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('expected schema JSON to be a list');
+
+ (new SchemaDecoder(new ValueDecoder(), new EntryInstantiator()))->decode('"scalar"');
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaEncoderTest.php
new file mode 100644
index 0000000000..ecfc0a6ba5
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaEncoderTest.php
@@ -0,0 +1,39 @@
+encodeSchema(schema(int_schema('id'), str_schema('name')));
+
+ static::assertSame(['id', 'name'], array_keys($plan->columns));
+ static::assertSame(
+ ['id', 'name'],
+ array_values(array_map(static fn($column) => $column->name, $plan->columns)),
+ );
+ }
+
+ public function test_column_name_with_invalid_utf8_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('failed to encode schema as JSON');
+
+ (new SchemaEncoder(new ValueEncoder()))->encodeSchema(schema(str_schema("bad\xFFname")));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaEvolutionTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaEvolutionTest.php
new file mode 100644
index 0000000000..56ae9d85c4
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaEvolutionTest.php
@@ -0,0 +1,88 @@
+expectNotToPerformAssertions();
+
+ (new SchemaEvolution())->validate(
+ schema(int_schema('id'), str_schema('name')),
+ schema(int_schema('id'), str_schema('name')),
+ );
+ }
+
+ public function test_narrowing_nullable_column_to_non_nullable_is_valid(): void
+ {
+ $this->expectNotToPerformAssertions();
+
+ (new SchemaEvolution())->validate(schema(str_schema('name', nullable: true)), schema(str_schema('name')));
+ }
+
+ public function test_new_non_nullable_column_throws(): void
+ {
+ $this->expectException(IncompatibleSchemaException::class);
+ $this->expectExceptionMessage('adds new column "email" which must be nullable');
+
+ (new SchemaEvolution())->validate(schema(int_schema('id')), schema(int_schema('id'), str_schema('email')));
+ }
+
+ public function test_new_nullable_column_is_valid(): void
+ {
+ $this->expectNotToPerformAssertions();
+
+ (new SchemaEvolution())->validate(
+ schema(int_schema('id')),
+ schema(int_schema('id'), str_schema('email', nullable: true)),
+ );
+ }
+
+ public function test_nullable_incoming_column_on_non_nullable_file_column_throws(): void
+ {
+ $this->expectException(IncompatibleSchemaException::class);
+ $this->expectExceptionMessage('changes column "id"');
+
+ (new SchemaEvolution())->validate(schema(int_schema('id')), schema(int_schema('id', nullable: true)));
+ }
+
+ public function test_omitting_non_nullable_column_throws(): void
+ {
+ $this->expectException(IncompatibleSchemaException::class);
+ $this->expectExceptionMessage('omits column "id" which is not nullable');
+
+ (new SchemaEvolution())->validate(
+ schema(int_schema('id'), str_schema('name', nullable: true)),
+ schema(str_schema('name', nullable: true)),
+ );
+ }
+
+ public function test_omitting_nullable_column_is_valid(): void
+ {
+ $this->expectNotToPerformAssertions();
+
+ (new SchemaEvolution())->validate(
+ schema(int_schema('id'), str_schema('name', nullable: true)),
+ schema(int_schema('id')),
+ );
+ }
+
+ public function test_type_change_throws(): void
+ {
+ $this->expectException(IncompatibleSchemaException::class);
+ $this->expectExceptionMessage('changes column "id" from integer to string');
+
+ (new SchemaEvolution())->validate(schema(int_schema('id')), schema(str_schema('id')));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaTrackerTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaTrackerTest.php
new file mode 100644
index 0000000000..cb6524ea6a
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaTrackerTest.php
@@ -0,0 +1,129 @@
+fits($plan, $row));
+ }
+
+ public function test_row_with_same_entries_fits(): void
+ {
+ $plan = FloeWriter::growSectionPlan(null, row(int_entry('a', 1), str_entry('b', 'x')));
+ $row = row(int_entry('a', 2), str_entry('b', 'y'));
+
+ static::assertTrue((new SchemaTracker())->fits($plan, $row));
+ }
+
+ public function test_row_with_a_new_column_does_not_fit(): void
+ {
+ $plan = FloeWriter::growSectionPlan(null, row(int_entry('a', 1)));
+ $row = row(int_entry('a', 2), int_entry('b', 3));
+
+ static::assertFalse((new SchemaTracker())->fits($plan, $row));
+ }
+
+ public function test_row_with_a_column_absent_from_the_plan_does_not_fit(): void
+ {
+ $plan = FloeWriter::growSectionPlan(null, row(int_entry('a', 1)));
+ $row = row(int_entry('b', 2));
+
+ static::assertFalse((new SchemaTracker())->fits($plan, $row));
+ }
+
+ public function test_row_with_an_incompatible_type_does_not_fit(): void
+ {
+ $plan = FloeWriter::growSectionPlan(null, row(int_entry('a', 1)));
+ $row = row(str_entry('a', 'x'));
+
+ static::assertFalse((new SchemaTracker())->fits($plan, $row));
+ }
+
+ public function test_row_with_a_different_list_element_type_does_not_fit(): void
+ {
+ $plan = FloeWriter::growSectionPlan(null, row(list_entry('a', [1], type_list(type_integer()))));
+ $row = row(list_entry('a', ['x'], type_list(type_string())));
+
+ static::assertFalse((new SchemaTracker())->fits($plan, $row));
+ }
+
+ public function test_row_with_equal_but_not_identical_list_type_fits(): void
+ {
+ $plan = FloeWriter::growSectionPlan(null, row(list_entry('a', [1], type_list(type_integer()))));
+
+ $tracker = new SchemaTracker();
+ $row = row(list_entry('a', [2, 3], type_list(type_integer())));
+
+ static::assertTrue($tracker->fits($plan, $row));
+ static::assertTrue($tracker->fits($plan, $row));
+ }
+
+ public function test_cached_scalar_fingerprint_does_not_mask_a_type_change(): void
+ {
+ $plan = FloeWriter::growSectionPlan(null, row(int_entry('a', 1)));
+
+ $tracker = new SchemaTracker();
+
+ static::assertTrue($tracker->fits($plan, row(int_entry('a', 2))));
+ static::assertTrue($tracker->fits($plan, row(int_entry('a', 3))));
+ static::assertFalse($tracker->fits($plan, row(str_entry('a', 'x'))));
+ }
+
+ public function test_row_with_custom_metadata_still_fits(): void
+ {
+ // documented limitation: column metadata is captured once per schema frame,
+ // per-row metadata drift does not force a new section
+ $plan = FloeWriter::growSectionPlan(null, row(int_entry('a', 1)));
+ $row = row(int_entry('a', 2, Metadata::fromArray(['custom' => 'meta'])));
+
+ static::assertTrue((new SchemaTracker())->fits($plan, $row));
+ }
+
+ public function test_row_with_numeric_column_name_fits(): void
+ {
+ // a numeric column name coerces to an int key in the name-keyed plan;
+ // the lookup key coerces identically, so the column still matches
+ $plan = FloeWriter::growSectionPlan(null, row(int_entry('123', 1)));
+
+ static::assertTrue((new SchemaTracker())->fits($plan, row(int_entry('123', 2))));
+ static::assertFalse((new SchemaTracker())->fits($plan, row(str_entry('123', 'x'))));
+ }
+
+ public function test_row_with_from_null_string_entry_fits_plain_string_plan(): void
+ {
+ $plan = FloeWriter::growSectionPlan(null, row(str_entry('a', 'value')));
+ $row = row(StringEntry::fromNull('a'));
+
+ static::assertTrue((new SchemaTracker())->fits($plan, $row));
+ }
+
+ public function test_row_with_null_value_still_fits(): void
+ {
+ $plan = FloeWriter::growSectionPlan(null, row(str_entry('a', 'value')));
+ $row = row(str_entry('a', null));
+
+ static::assertTrue((new SchemaTracker())->fits($plan, $row));
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/SectionTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/SectionTest.php
new file mode 100644
index 0000000000..c24510b808
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/SectionTest.php
@@ -0,0 +1,41 @@
+expectException(FloeException::class);
+
+ Section::fromArray(['offset' => 6, 'schemaId' => 0]);
+ }
+
+ public function test_from_array_with_non_integer_field_throws(): void
+ {
+ $this->expectException(FloeException::class);
+
+ Section::fromArray(['offset' => '6', 'schemaId' => 0, 'rowCount' => 1]);
+ }
+
+ public function test_from_array_with_non_integer_schema_id_throws(): void
+ {
+ $this->expectException(FloeException::class);
+
+ Section::fromArray(['offset' => 6, 'schemaId' => '0', 'rowCount' => 1]);
+ }
+
+ public function test_normalize_round_trips(): void
+ {
+ $section = new Section(6, 0, 100);
+
+ static::assertEquals($section, Section::fromArray($section->normalize()));
+ static::assertSame(['offset' => 6, 'schemaId' => 0, 'rowCount' => 100], $section->normalize());
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/ValueDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/ValueDecoderTest.php
new file mode 100644
index 0000000000..d0abb631e6
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/ValueDecoderTest.php
@@ -0,0 +1,244 @@
+expectException(FloeException::class);
+ $this->expectExceptionMessage('Floe found unknown datetime flag 0x02');
+
+ DateTimeDecoderMother::create()->decode($encoded, $position);
+ }
+
+ public function test_decoding_datetime_with_unknown_flag_throws(): void
+ {
+ $position = 0;
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('unknown datetime flag 0xEE');
+
+ DateTimeDecoderMother::create()->decode("\xEE", $position);
+ }
+
+ public function test_decoding_dynamic_value_with_unknown_tag_throws(): void
+ {
+ $position = 0;
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('unknown dynamic value tag');
+
+ DynamicDecoderMother::create()->decode("\xEE", $position);
+ }
+
+ public function test_decoding_enum_of_unknown_case_throws(): void
+ {
+ $class = BackedStringEnum::class;
+ $encoded = pack('V', strlen($class)) . $class . pack('V', 4) . 'nope';
+ $decoder = (new ValueDecoder())->decoderFor(type_enum(BackedStringEnum::class));
+ $position = 0;
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('cannot restore enum case');
+
+ $decoder->decode($encoded, $position);
+ }
+
+ public function test_decoding_enum_of_unknown_class_throws(): void
+ {
+ $encoded = pack('V', 10) . 'NoSuchEnum' . pack('V', 3) . 'one';
+ $decoder = (new ValueDecoder())->decoderFor(type_enum(BackedStringEnum::class));
+ $position = 0;
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('enum not found');
+
+ $decoder->decode($encoded, $position);
+ }
+
+ public function test_decoding_html_document_below_php84_throws(): void
+ {
+ if (class_exists('\Dom\HTMLDocument')) {
+ static::markTestSkipped('requires PHP < 8.4');
+ }
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('requires PHP 8.4+');
+
+ ValueDecoder::htmlDocumentFromString('x
');
+ }
+
+ public function test_decoding_html_element_below_php84_throws(): void
+ {
+ if (class_exists('\Dom\HTMLDocument')) {
+ static::markTestSkipped('requires PHP < 8.4');
+ }
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('requires PHP 8.4+');
+
+ ValueDecoder::htmlElementFromString('x
');
+ }
+
+ public function test_decoding_structure_with_unknown_element_flag_throws(): void
+ {
+ $decoder = (new ValueDecoder())->decoderFor(type_structure(['name' => type_string()]));
+ $position = 0;
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('unknown structure element flag');
+
+ $decoder->decode("\xEE", $position);
+ }
+
+ public function test_decoding_invalid_xml_throws(): void
+ {
+ $encoded = pack('V', 9) . 'not < xml';
+ $decoder = (new ValueDecoder())->decoderFor(type_xml());
+ $position = 0;
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('failed to restore DOMDocument');
+
+ $decoder->decode($encoded, $position);
+ }
+
+ public function test_decoding_value_of_unsupported_type_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('does not support values of type');
+
+ (new ValueDecoder())->decoderFor(type_callable());
+ }
+
+ public function test_round_trip_of_datetime_preserves_timezone_and_microseconds(): void
+ {
+ $value = new DateTimeImmutable('2025-06-15 12:30:45.987654', new DateTimeZone('Australia/Eucla'));
+ $encoded = (new ValueEncoder())
+ ->encoderFor(type_datetime())
+ ->encode($value);
+ $position = 0;
+
+ // @mago-ignore analysis:mixed-assignment
+ $decoded = (new ValueDecoder())
+ ->decoderFor(type_datetime())
+ ->decode($encoded, $position);
+
+ static::assertInstanceOf(DateTimeImmutable::class, $decoded);
+ static::assertEquals($value, $decoded);
+ static::assertSame('Australia/Eucla', $decoded->getTimezone()->getName());
+ static::assertSame('987654', $decoded->format('u'));
+ static::assertSame(strlen($encoded), $position);
+ }
+
+ public function test_round_trip_of_datetime_before_epoch(): void
+ {
+ $value = new DateTimeImmutable('1969-07-20 20:17:00.500000 UTC');
+ $encoded = (new ValueEncoder())
+ ->encoderFor(type_datetime())
+ ->encode($value);
+ $position = 0;
+
+ static::assertEquals($value, (new ValueDecoder())
+ ->decoderFor(type_datetime())
+ ->decode($encoded, $position));
+ }
+
+ public function test_round_trip_of_integer_signed_edges(): void
+ {
+ $encoder = (new ValueEncoder())->encoderFor(type_integer());
+ $decoder = (new ValueDecoder())->decoderFor(type_integer());
+
+ foreach ([PHP_INT_MIN, PHP_INT_MAX, -1, 0, 42] as $value) {
+ $position = 0;
+
+ static::assertSame($value, $decoder->decode($encoder->encode($value), $position));
+ }
+ }
+
+ public function test_round_trip_of_interval_preserves_all_fields(): void
+ {
+ $value = new DateInterval('P1DT2H3M4S');
+ // @mago-ignore analysis:invalid-property-write
+ $value->invert = 1;
+ // @mago-ignore analysis:invalid-property-write
+ $value->f = 0.5;
+
+ $encoded = (new ValueEncoder())
+ ->encoderFor(type_time())
+ ->encode($value);
+ $position = 0;
+
+ // @mago-ignore analysis:mixed-assignment
+ $decoded = (new ValueDecoder())
+ ->decoderFor(type_time())
+ ->decode($encoded, $position);
+
+ static::assertInstanceOf(DateInterval::class, $decoded);
+ static::assertSame([1, 2, 3, 4, 0.5, 1], [
+ $decoded->d,
+ $decoded->h,
+ $decoded->i,
+ $decoded->s,
+ $decoded->f,
+ $decoded->invert,
+ ]);
+ }
+
+ public function test_restoring_xml_element_from_string(): void
+ {
+ static::assertSame('item', ValueDecoder::xmlElementFromString('- value
')->tagName);
+ }
+
+ public function test_round_trip_of_uuid_skips_validation(): void
+ {
+ $uuid = new Uuid('0196aecb-b568-7e57-a381-8ec8d3e4a531');
+ $encoded = (new ValueEncoder())
+ ->encoderFor(type_uuid())
+ ->encode($uuid);
+ $position = 0;
+
+ // @mago-ignore analysis:mixed-assignment
+ $decoded = (new ValueDecoder())
+ ->decoderFor(type_uuid())
+ ->decode($encoded, $position);
+
+ static::assertInstanceOf(Uuid::class, $decoded);
+ static::assertSame($uuid->toString(), $decoded->toString());
+ }
+}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/ValueEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/ValueEncoderTest.php
new file mode 100644
index 0000000000..9ed67a0f0c
--- /dev/null
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/ValueEncoderTest.php
@@ -0,0 +1,134 @@
+encode(new DateTimeImmutable('2025-01-01 00:00:00 UTC'))[0]),
+ );
+ static::assertSame(Format::DATETIME_MUTABLE, ord($encoder->encode(new DateTime('2025-01-01 00:00:00 UTC'))[0]));
+ }
+
+ public function test_encoding_datetime_of_custom_class_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('Floe supports only DateTime and DateTimeImmutable, got '
+ . CustomDateTime::class);
+
+ (new DateTimeEncoder())->encode(new CustomDateTime('2025-01-01 00:00:00 UTC'));
+ }
+
+ public function test_encoding_datetime_of_custom_class_nested_in_dynamic_value_throws(): void
+ {
+ $encoder = (new ValueEncoder())->encoderFor(type_list(type_mixed()));
+
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('Floe supports only DateTime and DateTimeImmutable');
+
+ $encoder->encode([['created_at' => new CustomDateTime('2025-01-01 00:00:00 UTC')]]);
+ }
+
+ public function test_encoding_dynamic_value_of_unsupported_type_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('does not support values of type "stdClass" in mixed/union context');
+
+ (new DynamicEncoder())->encode(new stdClass());
+ }
+
+ public function test_encoding_list_of_mixed_values_with_object_throws(): void
+ {
+ $encoder = (new ValueEncoder())->encoderFor(type_list(type_mixed()));
+
+ $this->expectException(FloeException::class);
+
+ $encoder->encode([new stdClass()]);
+ }
+
+ public function test_encoding_value_of_unsupported_type_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('does not support values of type');
+
+ (new ValueEncoder())->encoderFor(type_callable());
+ }
+
+ public function test_encoding_xml_element_without_owner_document_throws(): void
+ {
+ $this->expectException(FloeException::class);
+ $this->expectExceptionMessage('failed to convert DOMElement to XML string');
+
+ ValueEncoder::xmlElementToString(new DOMElement('detached'));
+ }
+
+ public function test_integer_encoding_is_little_endian_for_signed_edges(): void
+ {
+ $encoder = (new ValueEncoder())->encoderFor(type_integer());
+
+ static::assertSame(pack('P', PHP_INT_MIN), $encoder->encode(PHP_INT_MIN));
+ static::assertSame(pack('P', PHP_INT_MAX), $encoder->encode(PHP_INT_MAX));
+ static::assertSame("\xFB\xFF\xFF\xFF\xFF\xFF\xFF\xFF", $encoder->encode(-5));
+ static::assertSame("\x2A\x00\x00\x00\x00\x00\x00\x00", $encoder->encode(42));
+ }
+
+ public function test_integer_list_fast_path_is_bulk_packed(): void
+ {
+ $encoder = (new ValueEncoder())->encoderFor(type_list(type_integer()));
+
+ static::assertSame(pack('V', 3) . pack('P*', 10, -20, 30), $encoder->encode([10, -20, 30]));
+ static::assertSame(pack('V', 0), $encoder->encode([]));
+ }
+
+ public function test_round_trip_of_dynamic_values(): void
+ {
+ $encoder = new DynamicEncoder();
+ $decoder = DynamicDecoderMother::create();
+
+ $values = [
+ 'null' => null,
+ 'int' => 42,
+ 'negative_int' => PHP_INT_MIN,
+ 'float' => 3.5,
+ 'bool' => true,
+ 'string' => 'text',
+ 'array' => ['a' => 1, 0 => 'zero', 'nested' => ['x' => [1, 2]], -7 => 'negative key'],
+ ];
+
+ foreach ($values as $label => $value) {
+ $position = 0;
+
+ static::assertSame($value, $decoder->decode($encoder->encode($value), $position), $label);
+ }
+ }
+}
diff --git a/src/extension/flow-php-ext/.cargo/config.toml b/src/extension/flow-php-ext/.cargo/config.toml
new file mode 100644
index 0000000000..074e32eac4
--- /dev/null
+++ b/src/extension/flow-php-ext/.cargo/config.toml
@@ -0,0 +1,11 @@
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
+
+[target.x86_64-apple-darwin]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
+
+[target.x86_64-unknown-linux-gnu]
+rustflags = ["-C", "link-arg=-Wl,--allow-shlib-undefined"]
+
+[target.aarch64-unknown-linux-gnu]
+rustflags = ["-C", "link-arg=-Wl,--allow-shlib-undefined"]
diff --git a/src/extension/flow-php-ext/.gitattributes b/src/extension/flow-php-ext/.gitattributes
new file mode 100644
index 0000000000..1b17167edd
--- /dev/null
+++ b/src/extension/flow-php-ext/.gitattributes
@@ -0,0 +1,13 @@
+# Exclude development files from distribution archives
+/.github export-ignore
+/tests export-ignore
+/target export-ignore
+/vendor export-ignore
+/.gitattributes export-ignore
+/.gitignore export-ignore
+
+# Ensure consistent line endings
+*.php text eol=lf
+*.rs text eol=lf
+*.toml text eol=lf
+*.m4 text eol=lf
diff --git a/src/extension/flow-php-ext/.github/workflows/readonly.yaml b/src/extension/flow-php-ext/.github/workflows/readonly.yaml
new file mode 100644
index 0000000000..cb11ce36e1
--- /dev/null
+++ b/src/extension/flow-php-ext/.github/workflows/readonly.yaml
@@ -0,0 +1,28 @@
+name: Readonly
+
+on:
+ pull_request_target:
+ types: [opened]
+
+permissions: {}
+
+jobs:
+ run:
+ runs-on: ubuntu-latest
+ permissions:
+ pull-requests: write # Required to comment on and close the PR.
+ steps:
+ - name: Close pull request
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GH_REPO: ${{ github.repository }}
+ PR_NUMBER: ${{ github.event.pull_request.number }}
+ COMMENT: |
+ Thanks for your contribution! We love it & really appreciate your effort!
+
+ However, you should instead open your PR on the main monorepo repository:
+ https://github.com/flow-php/flow
+
+ This repository is what we call a "subtree split": a read-only subset of that main monorepo repository.
+ We're looking forward to your PR there!
+ run: gh pr close "$PR_NUMBER" --comment "$COMMENT"
diff --git a/src/extension/flow-php-ext/.github/workflows/release.yml b/src/extension/flow-php-ext/.github/workflows/release.yml
new file mode 100644
index 0000000000..1cbd327f7f
--- /dev/null
+++ b/src/extension/flow-php-ext/.github/workflows/release.yml
@@ -0,0 +1,250 @@
+name: Release Precompiled Binaries
+
+on:
+ push:
+ tags:
+ - '*'
+ workflow_dispatch:
+ inputs:
+ tag:
+ description: 'Existing tag to (re)build and publish precompiled binaries for'
+ required: true
+
+permissions:
+ contents: read
+
+env:
+ EXTENSION_NAME: flow_php
+ RUST_BACKTRACE: 1
+
+jobs:
+ create-release:
+ name: Create Release
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ outputs:
+ tag: ${{ steps.tag.outputs.tag }}
+ steps:
+ - name: Get tag
+ id: tag
+ env:
+ EVENT_NAME: ${{ github.event_name }}
+ INPUT_TAG: ${{ inputs.tag }}
+ run: |
+ if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
+ echo "tag=$INPUT_TAG" >> "$GITHUB_OUTPUT"
+ else
+ echo "tag=${GITHUB_REF#refs/tags/}" >> "$GITHUB_OUTPUT"
+ fi
+
+ - name: Create draft release
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ TAG: ${{ steps.tag.outputs.tag }}
+ REPO: ${{ github.repository }}
+ run: |
+ if gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1; then
+ echo "Release $TAG already exists, reusing it."
+ else
+ gh release create "$TAG" \
+ --repo "$REPO" \
+ --draft \
+ --title "$TAG" \
+ --generate-notes
+ fi
+
+ build-linux:
+ name: "Linux ${{ matrix.arch }} - PHP ${{ matrix.php }} - ${{ matrix.ts }}"
+ needs: create-release
+ runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
+ permissions:
+ contents: write
+ strategy:
+ fail-fast: false
+ matrix:
+ php: ['8.3', '8.4', '8.5']
+ arch: ['x86_64', 'arm64']
+ ts: ['nts', 'zts']
+
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ ref: ${{ needs.create-release.outputs.tag }}
+ persist-credentials: false
+
+ - name: Set build variables
+ id: build-vars
+ env:
+ TS: ${{ matrix.ts }}
+ PHP: ${{ matrix.php }}
+ run: |
+ if [ "$TS" = "zts" ]; then
+ echo "php-image=${PHP}-zts" >> "$GITHUB_OUTPUT"
+ echo "ts-suffix=-zts" >> "$GITHUB_OUTPUT"
+ else
+ echo "php-image=${PHP}-cli" >> "$GITHUB_OUTPUT"
+ echo "ts-suffix=" >> "$GITHUB_OUTPUT"
+ fi
+
+ - name: Create output directory
+ run: mkdir -p dist
+
+ - name: Build in Docker container
+ env:
+ WORKSPACE: ${{ github.workspace }}
+ FLOW_PHP_EXT_VERSION: ${{ needs.create-release.outputs.tag }}
+ PHP_IMAGE: ${{ steps.build-vars.outputs.php-image }}
+ run: |
+ docker run --rm \
+ -v "$WORKSPACE:/workspace" \
+ -w /workspace \
+ -e FLOW_PHP_EXT_VERSION="$FLOW_PHP_EXT_VERSION" \
+ "php:$PHP_IMAGE" \
+ bash -c '
+ set -euo pipefail
+
+ echo "::group::Install system dependencies"
+ apt-get update
+ apt-get install -y build-essential clang libclang-dev curl pkg-config
+ echo "::endgroup::"
+
+ echo "::group::Install Rust"
+ curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ source "$HOME/.cargo/env"
+ echo "::endgroup::"
+
+ echo "::group::Build extension"
+ cargo build --release
+ echo "::endgroup::"
+
+ echo "::group::Verify extension loads"
+ php -n -d extension=/workspace/target/release/libflow_php.so -m | grep -q flow_php || { echo "ERROR: flow_php extension failed to load"; exit 1; }
+ echo "flow_php extension loaded successfully"
+ echo "::endgroup::"
+
+ echo "::group::Package artifact"
+ cp /workspace/target/release/libflow_php.so /workspace/dist/flow_php.so
+ echo "::endgroup::"
+ '
+
+ - name: Create ZIP archive
+ env:
+ TAG: ${{ needs.create-release.outputs.tag }}
+ PHP: ${{ matrix.php }}
+ ARCH: ${{ matrix.arch }}
+ TS_SUFFIX: ${{ steps.build-vars.outputs.ts-suffix }}
+ run: |
+ cd dist
+ ARTIFACT_NAME="php_${EXTENSION_NAME}-${TAG}_php${PHP}-${ARCH}-linux-glibc${TS_SUFFIX}.zip"
+ zip "$ARTIFACT_NAME" flow_php.so
+ echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> "$GITHUB_ENV"
+ echo "ARTIFACT_PATH=dist/$ARTIFACT_NAME" >> "$GITHUB_ENV"
+
+ - name: Upload to release
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ TAG: ${{ needs.create-release.outputs.tag }}
+ REPO: ${{ github.repository }}
+ run: |
+ gh release upload "$TAG" \
+ "$ARTIFACT_PATH" \
+ --repo "$REPO" \
+ --clobber
+
+ build-macos:
+ name: "macOS arm64 - PHP ${{ matrix.php }} - ${{ matrix.ts }}"
+ needs: create-release
+ runs-on: macos-latest
+ permissions:
+ contents: write
+ strategy:
+ fail-fast: false
+ matrix:
+ php: ['8.3', '8.4', '8.5']
+ ts: ['nts', 'zts']
+
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ ref: ${{ needs.create-release.outputs.tag }}
+ persist-credentials: false
+
+ - name: Setup PHP
+ uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # v2.37.2
+ with:
+ php-version: ${{ matrix.php }}
+ env:
+ phpts: ${{ matrix.ts }}
+
+ - name: Set build variables
+ id: build-vars
+ env:
+ TS: ${{ matrix.ts }}
+ run: |
+ if [ "$TS" = "zts" ]; then
+ echo "ts-suffix=-zts" >> "$GITHUB_OUTPUT"
+ else
+ echo "ts-suffix=" >> "$GITHUB_OUTPUT"
+ fi
+
+ - name: Install Rust toolchain
+ uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+
+ - name: Install build dependencies
+ run: |
+ brew install llvm
+ echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> "$GITHUB_ENV"
+ echo "CLANG_PATH=$(brew --prefix llvm)/bin/clang" >> "$GITHUB_ENV"
+
+ - name: Build extension
+ env:
+ FLOW_PHP_EXT_VERSION: ${{ needs.create-release.outputs.tag }}
+ run: cargo build --release
+
+ - name: Verify extension loads
+ run: |
+ php -n -d extension=./target/release/libflow_php.dylib -m | grep -q flow_php || { echo "ERROR: flow_php extension failed to load"; exit 1; }
+ echo "flow_php extension loaded successfully"
+
+ - name: Create ZIP archive
+ env:
+ TAG: ${{ needs.create-release.outputs.tag }}
+ PHP: ${{ matrix.php }}
+ TS_SUFFIX: ${{ steps.build-vars.outputs.ts-suffix }}
+ run: |
+ mkdir -p dist
+ cp target/release/libflow_php.dylib dist/flow_php.so
+ cd dist
+ ARTIFACT_NAME="php_${EXTENSION_NAME}-${TAG}_php${PHP}-arm64-darwin-bsdlibc${TS_SUFFIX}.zip"
+ zip "$ARTIFACT_NAME" flow_php.so
+ echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> "$GITHUB_ENV"
+ echo "ARTIFACT_PATH=dist/$ARTIFACT_NAME" >> "$GITHUB_ENV"
+
+ - name: Upload to release
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ TAG: ${{ needs.create-release.outputs.tag }}
+ REPO: ${{ github.repository }}
+ run: |
+ gh release upload "$TAG" \
+ "$ARTIFACT_PATH" \
+ --repo "$REPO" \
+ --clobber
+
+ publish-release:
+ name: Publish Release
+ needs: [create-release, build-linux, build-macos]
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ steps:
+ - name: Mark release as published
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ TAG: ${{ needs.create-release.outputs.tag }}
+ REPO: ${{ github.repository }}
+ run: |
+ gh release edit "$TAG" \
+ --repo "$REPO" \
+ --draft=false
diff --git a/src/extension/flow-php-ext/.gitignore b/src/extension/flow-php-ext/.gitignore
new file mode 100644
index 0000000000..9461243056
--- /dev/null
+++ b/src/extension/flow-php-ext/.gitignore
@@ -0,0 +1,40 @@
+# Composer
+/vendor/
+composer.lock
+
+# Rust/Cargo build artifacts
+/target/
+
+# Extension output
+/ext/modules/
+/ext/flow_php.c
+
+# Build artifacts from phpize (if config.m4 is used)
+/ext/.libs/
+/ext/acinclude.m4
+/ext/aclocal.m4
+/ext/autom4te.cache/
+/ext/build/
+/ext/config.guess
+/ext/config.h
+/ext/config.h.in
+/ext/config.log
+/ext/config.nice
+/ext/config.status
+/ext/config.sub
+/ext/configure
+/ext/configure.ac
+/ext/install-sh
+/ext/libtool
+/ext/ltmain.sh
+/ext/Makefile
+/ext/Makefile.fragments
+/ext/Makefile.global
+/ext/Makefile.objects
+/ext/missing
+/ext/mkinstalldirs
+/ext/run-tests.php
+/ext/*.la
+/ext/*.lo
+/ext/*.o
+/ext/*.dep
diff --git a/src/extension/flow-php-ext/Cargo.lock b/src/extension/flow-php-ext/Cargo.lock
new file mode 100644
index 0000000000..94c62349d5
--- /dev/null
+++ b/src/extension/flow-php-ext/Cargo.lock
@@ -0,0 +1,1465 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "adler2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+
+[[package]]
+name = "aes"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138"
+dependencies = [
+ "cipher",
+ "cpubits",
+ "cpufeatures",
+]
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
+
+[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
+[[package]]
+name = "base64ct"
+version = "1.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
+
+[[package]]
+name = "bitflags"
+version = "2.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
+
+[[package]]
+name = "block-buffer"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
+dependencies = [
+ "hybrid-array",
+ "zeroize",
+]
+
+[[package]]
+name = "bumpalo"
+version = "3.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
+
+[[package]]
+name = "bytecount"
+version = "0.6.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
+
+[[package]]
+name = "bytes"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593"
+
+[[package]]
+name = "bzip2"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c"
+dependencies = [
+ "libbz2-rs-sys",
+]
+
+[[package]]
+name = "camino"
+version = "1.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f2d30e4173c4026932d51d31d6b0613b1fd3014bf3f9f8943d4ba139c437ba0"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "cargo-platform"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "cargo_metadata"
+version = "0.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa"
+dependencies = [
+ "camino",
+ "cargo-platform",
+ "semver",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "cc"
+version = "1.2.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996"
+dependencies = [
+ "find-msvc-tools",
+ "jobserver",
+ "libc",
+ "shlex 2.0.1",
+]
+
+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+
+[[package]]
+name = "cipher"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c"
+dependencies = [
+ "crypto-common",
+ "inout",
+]
+
+[[package]]
+name = "clang-sys"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
+[[package]]
+name = "cmov"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
+
+[[package]]
+name = "const-oid"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
+
+[[package]]
+name = "constant_time_eq"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
+
+[[package]]
+name = "convert_case"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49"
+dependencies = [
+ "unicode-segmentation",
+]
+
+[[package]]
+name = "core-foundation"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+
+[[package]]
+name = "cpubits"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae"
+
+[[package]]
+name = "cpufeatures"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "crc32fast"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crypto-common"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"
+dependencies = [
+ "hybrid-array",
+]
+
+[[package]]
+name = "ctutils"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e"
+dependencies = [
+ "cmov",
+]
+
+[[package]]
+name = "darling"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
+dependencies = [
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "deflate64"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2"
+
+[[package]]
+name = "der"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71fd89660b2dc699704064e59e9dba0147b903e85319429e131620d022be411b"
+dependencies = [
+ "pem-rfc7468",
+ "zeroize",
+]
+
+[[package]]
+name = "deranged"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
+
+[[package]]
+name = "digest"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
+dependencies = [
+ "block-buffer",
+ "const-oid",
+ "crypto-common",
+ "ctutils",
+ "zeroize",
+]
+
+[[package]]
+name = "either"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
+
+[[package]]
+name = "equivalent"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
+
+[[package]]
+name = "errno"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "error-chain"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc"
+dependencies = [
+ "version_check",
+]
+
+[[package]]
+name = "ext-php-rs"
+version = "0.15.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abe62f25cd053f5f95dc7bbf94e5b41818ea3cf39b36cc25de8ca6b4de68a0eb"
+dependencies = [
+ "anyhow",
+ "bitflags",
+ "cc",
+ "cfg-if",
+ "ext-php-rs-bindgen",
+ "ext-php-rs-build",
+ "ext-php-rs-derive",
+ "inventory",
+ "native-tls",
+ "once_cell",
+ "parking_lot",
+ "skeptic",
+ "ureq",
+ "zip",
+]
+
+[[package]]
+name = "ext-php-rs-bindgen"
+version = "0.72.1-extphprs.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32ceb73da3b8b18a3424765dd6926dbc29464a0a7bc31ab491600bfa02adc4bc"
+dependencies = [
+ "bitflags",
+ "cexpr",
+ "clang-sys",
+ "itertools",
+ "log",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex 1.3.0",
+ "syn",
+]
+
+[[package]]
+name = "ext-php-rs-build"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "561bce8a6312a6182c078cd987d8d9cb6bf9292a35817cfd009ea0bffa794f5f"
+dependencies = [
+ "anyhow",
+]
+
+[[package]]
+name = "ext-php-rs-derive"
+version = "0.11.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f57a075f878ad51bcc56745fb21efe601429a509204f8a385bbe526566597320"
+dependencies = [
+ "convert_case",
+ "darling",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "fastrand"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
+
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
+
+[[package]]
+name = "flate2"
+version = "1.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+ "zlib-rs",
+]
+
+[[package]]
+name = "flow_php"
+version = "0.1.0"
+dependencies = [
+ "ext-php-rs",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
+[[package]]
+name = "futures-core"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
+
+[[package]]
+name = "futures-task"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
+
+[[package]]
+name = "futures-util"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "pin-project-lite",
+ "slab",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi 5.3.0",
+ "wasip2",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "libc",
+ "r-efi 6.0.0",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "glob"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+
+[[package]]
+name = "hashbrown"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+
+[[package]]
+name = "hmac"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f"
+dependencies = [
+ "digest",
+]
+
+[[package]]
+name = "http"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
+dependencies = [
+ "bytes",
+ "itoa",
+]
+
+[[package]]
+name = "httparse"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
+
+[[package]]
+name = "hybrid-array"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c"
+dependencies = [
+ "typenum",
+]
+
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
+[[package]]
+name = "indexmap"
+version = "2.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
+dependencies = [
+ "equivalent",
+ "hashbrown",
+]
+
+[[package]]
+name = "inout"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7"
+dependencies = [
+ "hybrid-array",
+]
+
+[[package]]
+name = "inventory"
+version = "0.3.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "itertools"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "jobserver"
+version = "0.1.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
+dependencies = [
+ "getrandom 0.3.4",
+ "libc",
+]
+
+[[package]]
+name = "js-sys"
+version = "0.3.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102"
+dependencies = [
+ "cfg-if",
+ "futures-util",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "libbz2-rs-sys"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c"
+
+[[package]]
+name = "libc"
+version = "0.2.186"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+
+[[package]]
+name = "libloading"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
+dependencies = [
+ "cfg-if",
+ "windows-link",
+]
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
+
+[[package]]
+name = "lock_api"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
+dependencies = [
+ "scopeguard",
+]
+
+[[package]]
+name = "log"
+version = "0.4.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
+
+[[package]]
+name = "lzma-rust2"
+version = "0.16.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca93e534d1142d1d0dcca6d25fe302508a5dfb40b302802904577725ea0b695b"
+dependencies = [
+ "sha2",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
+
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "miniz_oxide"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
+dependencies = [
+ "adler2",
+ "simd-adler32",
+]
+
+[[package]]
+name = "native-tls"
+version = "0.2.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"
+dependencies = [
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "num-conv"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "openssl"
+version = "0.10.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.117"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.12.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-link",
+]
+
+[[package]]
+name = "pbkdf2"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629"
+dependencies = [
+ "digest",
+ "hmac",
+]
+
+[[package]]
+name = "pem-rfc7468"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9"
+dependencies = [
+ "base64ct",
+]
+
+[[package]]
+name = "percent-encoding"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
+
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
+[[package]]
+name = "ppmd-rust"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24"
+
+[[package]]
+name = "prettyplease"
+version = "0.2.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
+dependencies = [
+ "proc-macro2",
+ "syn",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pulldown-cmark"
+version = "0.9.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b"
+dependencies = [
+ "bitflags",
+ "memchr",
+ "unicase",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
+[[package]]
+name = "r-efi"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
+
+[[package]]
+name = "redox_syscall"
+version = "0.5.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
+name = "regex"
+version = "1.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+
+[[package]]
+name = "rustc-hash"
+version = "2.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
+
+[[package]]
+name = "rustix"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
+dependencies = [
+ "zeroize",
+]
+
+[[package]]
+name = "rustversion"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
+
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "schannel"
+version = "0.1.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
+[[package]]
+name = "security-framework"
+version = "3.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
+dependencies = [
+ "bitflags",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "semver"
+version = "1.0.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
+dependencies = [
+ "serde",
+ "serde_core",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.150"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "sha1"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
+name = "sha2"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
+[[package]]
+name = "simd-adler32"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
+
+[[package]]
+name = "skeptic"
+version = "0.13.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8"
+dependencies = [
+ "bytecount",
+ "cargo_metadata",
+ "error-chain",
+ "glob",
+ "pulldown-cmark",
+ "tempfile",
+ "walkdir",
+]
+
+[[package]]
+name = "slab"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
+
+[[package]]
+name = "smallvec"
+version = "1.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
+
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
+[[package]]
+name = "syn"
+version = "2.0.118"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "tempfile"
+version = "3.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
+dependencies = [
+ "fastrand",
+ "getrandom 0.4.3",
+ "once_cell",
+ "rustix",
+ "windows-sys",
+]
+
+[[package]]
+name = "time"
+version = "0.3.53"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50"
+dependencies = [
+ "deranged",
+ "js-sys",
+ "num-conv",
+ "powerfmt",
+ "serde_core",
+ "time-core",
+]
+
+[[package]]
+name = "time-core"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
+
+[[package]]
+name = "typed-path"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
+
+[[package]]
+name = "typenum"
+version = "1.20.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
+
+[[package]]
+name = "unicase"
+version = "2.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
+
+[[package]]
+name = "ureq"
+version = "3.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0"
+dependencies = [
+ "base64",
+ "der",
+ "flate2",
+ "log",
+ "native-tls",
+ "percent-encoding",
+ "rustls-pki-types",
+ "ureq-proto",
+ "utf8-zero",
+ "webpki-root-certs",
+]
+
+[[package]]
+name = "ureq-proto"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c"
+dependencies = [
+ "base64",
+ "http",
+ "httparse",
+ "log",
+]
+
+[[package]]
+name = "utf8-zero"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
+
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
+[[package]]
+name = "wasip2"
+version = "1.0.4+wasi-0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
+dependencies = [
+ "wit-bindgen",
+]
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "rustversion",
+ "wasm-bindgen-macro",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e"
+dependencies = [
+ "bumpalo",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "webpki-root-certs"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267"
+dependencies = [
+ "rustls-pki-types",
+]
+
+[[package]]
+name = "winapi-util"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
+[[package]]
+name = "windows-sys"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "wit-bindgen"
+version = "0.57.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
+
+[[package]]
+name = "zeroize"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
+
+[[package]]
+name = "zip"
+version = "8.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
+dependencies = [
+ "aes",
+ "bzip2",
+ "constant_time_eq",
+ "crc32fast",
+ "deflate64",
+ "flate2",
+ "getrandom 0.4.3",
+ "hmac",
+ "indexmap",
+ "lzma-rust2",
+ "memchr",
+ "pbkdf2",
+ "ppmd-rust",
+ "sha1",
+ "time",
+ "typed-path",
+ "zeroize",
+ "zopfli",
+ "zstd",
+]
+
+[[package]]
+name = "zlib-rs"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5431d5661c32445236631278f27946e444ddafe4684cac70b185272d4f9c52d5"
+
+[[package]]
+name = "zmij"
+version = "1.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
+
+[[package]]
+name = "zopfli"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
+dependencies = [
+ "bumpalo",
+ "crc32fast",
+ "log",
+ "simd-adler32",
+]
+
+[[package]]
+name = "zstd"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
+dependencies = [
+ "zstd-safe",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "7.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
+dependencies = [
+ "zstd-sys",
+]
+
+[[package]]
+name = "zstd-sys"
+version = "2.0.16+zstd.1.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
diff --git a/src/extension/flow-php-ext/Cargo.toml b/src/extension/flow-php-ext/Cargo.toml
new file mode 100644
index 0000000000..74d620099c
--- /dev/null
+++ b/src/extension/flow-php-ext/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+name = "flow_php"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+ext-php-rs = "0.15"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
diff --git a/src/extension/flow-php-ext/Makefile b/src/extension/flow-php-ext/Makefile
new file mode 100644
index 0000000000..5cdd7fb8af
--- /dev/null
+++ b/src/extension/flow-php-ext/Makefile
@@ -0,0 +1,81 @@
+# Flow PHP Extension Makefile
+EXTENSION_DIR := ./ext
+MODULE_DIR := $(EXTENSION_DIR)/modules
+
+# PHP build tools
+PHP ?= $(shell which php)
+
+# Detect OS for library extension
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Darwin)
+ LIB_EXT := dylib
+else
+ LIB_EXT := so
+endif
+
+CARGO_OUTPUT := target/release/libflow_php.$(LIB_EXT)
+EXTENSION_SO := $(MODULE_DIR)/flow_php.so
+
+.PHONY: all build clean test install rebuild
+
+all: build
+
+# Build extension via cargo
+build:
+ @echo "Building flow_php extension (Rust)..."
+ cargo build --release
+ @mkdir -p $(MODULE_DIR)
+ @cp $(CARGO_OUTPUT) $(EXTENSION_SO)
+ @echo "Extension built: $(EXTENSION_SO)"
+
+# Run PHPT tests
+test: build
+ @echo "Running PHPT tests..."
+ @failed=0; total=0; passed=0; skipped=0; \
+ for f in tests/phpt/*.phpt; do \
+ total=$$((total + 1)); \
+ test_name=$$(basename "$$f"); \
+ tmp="$$(dirname "$$f")/_run_$${test_name%.phpt}.php"; \
+ skipif="$$(dirname "$$f")/_skipif_$${test_name%.phpt}.php"; \
+ sed -n '/^--SKIPIF--$$/,/^--FILE--$$/p' "$$f" | sed '1d;$$d' > "$$skipif"; \
+ skip_out=""; \
+ if [ -s "$$skipif" ]; then \
+ skip_out=$$($(PHP) -d extension=$$(realpath $(EXTENSION_SO)) "$$skipif" 2>&1) || true; \
+ fi; \
+ rm -f "$$skipif"; \
+ case "$$skip_out" in skip*) \
+ echo "SKIP: $$test_name ($$skip_out)"; \
+ skipped=$$((skipped + 1)); \
+ continue;; \
+ esac; \
+ sed -n '/^--FILE--$$/,/^--EXPECT/p' "$$f" | sed '1d;$$d' > "$$tmp"; \
+ expected=$$(sed -n '/^--EXPECT\(F\)\{0,1\}--$$/,$$p' "$$f" | sed '1d' | tr -d '\r'); \
+ actual=$$($(PHP) -d extension=$$(realpath $(EXTENSION_SO)) "$$tmp" 2>&1) || true; \
+ actual=$$(echo "$$actual" | tr -d '\r'); \
+ rm -f "$$tmp"; \
+ if [ "$$actual" = "$$expected" ]; then \
+ echo "PASS: $$test_name"; \
+ passed=$$((passed + 1)); \
+ else \
+ echo "FAIL: $$test_name"; \
+ echo " Expected: $$expected"; \
+ echo " Actual: $$actual"; \
+ failed=1; \
+ fi; \
+ done; \
+ echo "$$passed/$$total tests passed ($$skipped skipped)"; \
+ if [ $$failed -eq 1 ]; then exit 1; fi
+
+# Install to current PHP extension directory
+install: build
+ @EXT_DIR=$$($(PHP) -r 'echo ini_get("extension_dir");') && \
+ cp $(EXTENSION_SO) "$$EXT_DIR/" && \
+ echo "Extension installed to $$EXT_DIR/flow_php.so"
+
+# Clean all build artifacts
+clean:
+ cargo clean
+ @rm -rf $(MODULE_DIR)
+
+# Full rebuild
+rebuild: clean build
diff --git a/src/extension/flow-php-ext/README.md b/src/extension/flow-php-ext/README.md
new file mode 100644
index 0000000000..8795b21169
--- /dev/null
+++ b/src/extension/flow-php-ext/README.md
@@ -0,0 +1,39 @@
+# Flow PHP - Native Extension
+
+A PHP extension written in Rust ([ext-php-rs](https://github.com/extphprs/ext-php-rs)) providing
+native implementations for performance-critical parts of the Flow DataFrame framework. It encodes and
+decodes the frames of the Flow **Floe** binary format (`.floe`; the canonical PHP implementation
+lives in `Flow\Floe` in `flow-php/etl`). `Flow\Floe\FloeReader`/`FloeWriter` keep file header/footer
+assembly in PHP and delegate frame work to the extension:
+
+```php
+$decoder = new Flow\Floe\RowsDecoder();
+$decoder->schema($schemaFrameBody);
+$row = $decoder->row($rowFrameBody); // Flow\ETL\Row
+$rows = $decoder->rows([$rowFrameBody, ...]); // batched: Flow\ETL\Rows
+
+$encoder = new Flow\Floe\RowsEncoder();
+$encoder->schema($schemaFrameBody);
+$rowFrameBody = $encoder->row($row); // byte-identical to Flow\Floe\RowEncoder
+$segments = $encoder->rows($rows); // batched: framed ROW bytes per section,
+ // list of Flow\Floe\FrameSegment
+```
+
+The extension is always optional: `FloeReader`/`FloeWriter` route to it automatically when `flow_php`
+is loaded, and behavior is identical with or without it. Extension errors surface as
+`Flow\Floe\Exception\ExtensionException` — there is no silent fallback. All 17 entry types are
+covered; xml/xml_element/html/html_element values delegate DOM (de)construction to the pure-PHP
+`Flow\Floe\ValueDecoder`/`ValueEncoder` static methods. Only `DateTime`/`DateTimeImmutable` are
+supported — datetime subclasses throw.
+
+## Build
+
+```bash
+nix-shell --arg with-rust true --run "cd src/extension/flow-php-ext && make build"
+```
+
+## Test
+
+```bash
+nix-shell --arg with-rust true --run "cd src/extension/flow-php-ext && make test"
+```
diff --git a/src/extension/flow-php-ext/build.rs b/src/extension/flow-php-ext/build.rs
new file mode 100644
index 0000000000..bcf686cea3
--- /dev/null
+++ b/src/extension/flow-php-ext/build.rs
@@ -0,0 +1,18 @@
+fn main() {
+ let version = std::env::var("FLOW_PHP_EXT_VERSION")
+ .ok()
+ .filter(|s| !s.is_empty())
+ .or_else(|| {
+ std::process::Command::new("git")
+ .args(["describe", "--tags", "--always"])
+ .output()
+ .ok()
+ .filter(|o| o.status.success())
+ .and_then(|o| String::from_utf8(o.stdout).ok())
+ .map(|s| s.trim().to_string())
+ .filter(|s| !s.is_empty())
+ })
+ .unwrap_or_else(|| env!("CARGO_PKG_VERSION").to_string());
+
+ println!("cargo:rustc-env=FLOW_PHP_EXT_VERSION={version}");
+}
diff --git a/src/extension/flow-php-ext/composer.json b/src/extension/flow-php-ext/composer.json
new file mode 100644
index 0000000000..af377f33df
--- /dev/null
+++ b/src/extension/flow-php-ext/composer.json
@@ -0,0 +1,36 @@
+{
+ "name": "flow-php/flow-php-ext",
+ "type": "php-ext",
+ "description": "Flow PHP native extension (Rust) - Floe frame-body encoder/decoder for DataFrame Rows",
+ "keywords": ["serializer", "binary", "extension", "rust", "native", "dataframe"],
+ "license": "MIT",
+ "require": {
+ "php": "~8.3.0 || ~8.4.0 || ~8.5.0"
+ },
+ "autoload": {
+ "psr-4": {
+ "Flow\\Floe\\": "php/Flow/Floe/"
+ }
+ },
+ "scripts": {
+ "build": "make build",
+ "test:ext": "make test"
+ },
+ "config": {
+ "sort-packages": true
+ },
+ "extra": {
+ "branch-alias": {
+ "dev-1.x": "1.x-dev"
+ }
+ },
+ "php-ext": {
+ "extension-name": "flow_php",
+ "build-path": "ext",
+ "configure-options": [],
+ "support-zts": true,
+ "support-nts": true,
+ "download-url-method": ["pre-packaged-binary", "composer-default"],
+ "os-families-exclude": ["windows"]
+ }
+}
diff --git a/src/extension/flow-php-ext/ext/config.m4 b/src/extension/flow-php-ext/ext/config.m4
new file mode 100644
index 0000000000..f5a34d11e3
--- /dev/null
+++ b/src/extension/flow-php-ext/ext/config.m4
@@ -0,0 +1,43 @@
+dnl config.m4 for extension flow_php (Rust-based via ext-php-rs)
+
+PHP_ARG_ENABLE([flow_php],
+ [whether to enable flow_php support],
+ [AS_HELP_STRING([--enable-flow_php],
+ [Enable Flow PHP extension (requires Rust/cargo)])])
+
+if test "$PHP_FLOW_PHP" != "no"; then
+ AC_MSG_CHECKING([for cargo (Rust build tool)])
+ AC_PATH_PROG(CARGO, cargo)
+ if test -z "$CARGO"; then
+ RUSTUP_CARGO=$(rustup which cargo 2>/dev/null)
+ if test -n "$RUSTUP_CARGO"; then
+ CARGO=$RUSTUP_CARGO
+ fi
+ fi
+ if test -z "$CARGO"; then
+ AC_MSG_ERROR([cargo is required to build the flow_php extension. Install Rust from https://rustup.rs/])
+ fi
+ AC_MSG_RESULT([$CARGO])
+
+ AC_MSG_CHECKING([for rustc (Rust compiler)])
+ AC_PATH_PROG(RUSTC, rustc)
+ if test -z "$RUSTC"; then
+ RUSTUP_RUSTC=$(rustup which rustc 2>/dev/null)
+ if test -n "$RUSTUP_RUSTC"; then
+ RUSTC=$RUSTUP_RUSTC
+ fi
+ fi
+ if test -z "$RUSTC"; then
+ AC_MSG_ERROR([rustc is required to build the flow_php extension. Install Rust from https://rustup.rs/])
+ fi
+ AC_MSG_RESULT([$RUSTC])
+
+ dnl Create a dummy C file so phpize/configure infrastructure does not complain
+ FLOW_PHP_EXT_DIR=$(pwd)
+ if test ! -f "$FLOW_PHP_EXT_DIR/flow_php.c"; then
+ echo "/* Dummy - actual extension built by Rust/cargo */" > "$FLOW_PHP_EXT_DIR/flow_php.c"
+ fi
+
+ PHP_NEW_EXTENSION(flow_php, flow_php.c, $ext_shared)
+ PHP_ADD_MAKEFILE_FRAGMENT
+fi
diff --git a/src/extension/flow-php-ext/php/Flow/Floe/Exception/ExtensionException.php b/src/extension/flow-php-ext/php/Flow/Floe/Exception/ExtensionException.php
new file mode 100644
index 0000000000..28972d4f34
--- /dev/null
+++ b/src/extension/flow-php-ext/php/Flow/Floe/Exception/ExtensionException.php
@@ -0,0 +1,15 @@
+ $frameBodies
+ *
+ * @throws ExtensionException
+ */
+ public function rows(array $frameBodies): Rows
+ {
+ throw new RuntimeException('flow_php extension is not loaded');
+ }
+
+ /**
+ * @throws ExtensionException
+ */
+ public function schema(string $frameBody): void
+ {
+ throw new RuntimeException('flow_php extension is not loaded');
+ }
+}
diff --git a/src/extension/flow-php-ext/php/Flow/Floe/RowsEncoder.php b/src/extension/flow-php-ext/php/Flow/Floe/RowsEncoder.php
new file mode 100644
index 0000000000..927ef4c5ee
--- /dev/null
+++ b/src/extension/flow-php-ext/php/Flow/Floe/RowsEncoder.php
@@ -0,0 +1,57 @@
+
+ */
+ public function rows(Rows $rows): array
+ {
+ throw new RuntimeException('flow_php extension is not loaded');
+ }
+
+ /**
+ * @throws ExtensionException
+ */
+ public function schema(string $frameBody): void
+ {
+ throw new RuntimeException('flow_php extension is not loaded');
+ }
+}
diff --git a/src/extension/flow-php-ext/src/ctx.rs b/src/extension/flow-php-ext/src/ctx.rs
new file mode 100644
index 0000000000..b9b88dfb80
--- /dev/null
+++ b/src/extension/flow-php-ext/src/ctx.rs
@@ -0,0 +1,809 @@
+//! PHP-engine plumbing: class-entry/slot lookups, hashtable helpers, calls and
+//! per-instance caches (timezones, enums, callables).
+
+use std::collections::HashMap;
+
+use ext_php_rs::boxed::ZBox;
+use ext_php_rs::convert::IntoZvalDyn;
+use ext_php_rs::exception::PhpException;
+use ext_php_rs::ffi::{
+ _zend_property_info, zend_call_known_function, zend_hash_index_update, zend_hash_str_find,
+ zend_hash_str_update, zend_objects_clone_members, zend_ulong,
+};
+use ext_php_rs::flags::ClassFlags;
+use ext_php_rs::types::{ZendHashTable, ZendObject, ZendStr, Zval};
+use ext_php_rs::zend::{ClassEntry, ExecutorGlobals, Function};
+
+use crate::exception::ext_exception;
+
+// Not in ext-php-rs's allowed bindings; ZEND_API symbols with the same ABI as
+// the bound zend_hash_*_update functions.
+extern "C" {
+ fn zend_hash_str_add(
+ ht: *mut ZendHashTable,
+ key: *const std::ffi::c_char,
+ len: usize,
+ data: *mut Zval,
+ ) -> *mut Zval;
+
+ fn zend_hash_index_add(ht: *mut ZendHashTable, h: zend_ulong, data: *mut Zval) -> *mut Zval;
+}
+
+/// PHP array-key coercion (`_zend_handle_numeric_str`): parse + canonical
+/// re-format equality rejects "+5", "-0", leading zeros and whitespace exactly
+/// like the engine.
+pub fn array_key_index(key: &[u8]) -> Option {
+ let string = std::str::from_utf8(key).ok()?;
+ let value: i64 = string.parse().ok()?;
+
+ (value.to_string().as_bytes() == key).then_some(value)
+}
+
+/// Inserts with PHP array-assignment semantics; ownership of `value` moves
+/// into the table.
+pub fn ht_insert(ht: &mut ZendHashTable, key: &[u8], mut value: Zval) {
+ unsafe {
+ match array_key_index(key) {
+ Some(index) => {
+ zend_hash_index_update(ht, index as u64, std::ptr::from_mut(&mut value));
+ }
+ None => {
+ zend_hash_str_update(
+ ht,
+ key.as_ptr().cast(),
+ key.len(),
+ std::ptr::from_mut(&mut value),
+ );
+ }
+ }
+ }
+
+ std::mem::forget(value);
+}
+
+pub fn ht_insert_index(ht: &mut ZendHashTable, index: i64, mut value: Zval) {
+ unsafe {
+ zend_hash_index_update(ht, index as u64, std::ptr::from_mut(&mut value));
+ }
+
+ std::mem::forget(value);
+}
+
+/// Inserts only when the key is absent (PHP array-key coercion applies) and
+/// reports whether it did; on `false` the value is dropped, not inserted.
+pub fn ht_add(ht: &mut ZendHashTable, key: &[u8], mut value: Zval) -> bool {
+ let added = unsafe {
+ match array_key_index(key) {
+ Some(index) => {
+ !zend_hash_index_add(ht, index as u64, std::ptr::from_mut(&mut value)).is_null()
+ }
+ None => !zend_hash_str_add(
+ ht,
+ key.as_ptr().cast(),
+ key.len(),
+ std::ptr::from_mut(&mut value),
+ )
+ .is_null(),
+ }
+ };
+
+ if added {
+ std::mem::forget(value);
+ }
+
+ added
+}
+
+pub fn zval_str(bytes: &[u8]) -> Zval {
+ let mut zv = Zval::new();
+ zv.set_zend_string(ZendStr::new(bytes, false));
+ zv
+}
+
+pub fn zval_long(value: i64) -> Zval {
+ let mut zv = Zval::new();
+ zv.set_long(value);
+ zv
+}
+
+pub fn zval_null() -> Zval {
+ let mut zv = Zval::new();
+ zv.set_null();
+ zv
+}
+
+pub fn ht_contains(ht: &ZendHashTable, key: &[u8]) -> bool {
+ match array_key_index(key) {
+ Some(index) => ht.get_index(index).is_some(),
+ None => unsafe {
+ !zend_hash_str_find(
+ std::ptr::from_ref(ht).cast_mut(),
+ key.as_ptr().cast(),
+ key.len(),
+ )
+ .is_null()
+ },
+ }
+}
+
+pub fn find_class(name: &str) -> Result<&'static ClassEntry, PhpException> {
+ ClassEntry::try_find(name).ok_or_else(|| {
+ ext_exception(format!(
+ "flow_php requires class \"{name}\" to be autoloadable"
+ ))
+ })
+}
+
+/// Fails when the last engine call left an exception pending, so PHP-level
+/// failures (constructor throws, decode() throws) are never silently ignored.
+pub fn ensure_no_pending_exception(context: &str) -> Result<(), PhpException> {
+ if ExecutorGlobals::get().exception.is_null() {
+ return Ok(());
+ }
+
+ Err(ext_exception(format!("flow_php failed to {context}")))
+}
+
+/// Writes one property through the object's write handler, which copies the
+/// value with assign semantics - the caller keeps ownership of `value`.
+pub fn write_property_raw(
+ obj: &mut ZendObject,
+ name: &mut ZendStr,
+ mut value: Zval,
+) -> Result<(), PhpException> {
+ let handler = unsafe { obj.handlers.as_ref() }
+ .and_then(|handlers| handlers.write_property)
+ .ok_or_else(|| ext_exception("flow_php failed to resolve a property write handler"))?;
+
+ unsafe {
+ handler(
+ std::ptr::from_mut(obj),
+ std::ptr::from_mut(name),
+ std::ptr::from_mut(&mut value),
+ std::ptr::null_mut(),
+ );
+ }
+
+ ensure_no_pending_exception("write an object property")
+}
+
+/// Resolves a declared property's property-table slot offset; keyed by the
+/// plain (unmangled) name for private properties too.
+pub fn property_offset(ce: &ClassEntry, name: &str) -> Result {
+ let properties_info: &ZendHashTable = &ce.properties_info;
+
+ let info = properties_info
+ .get(name)
+ .and_then(|zv| unsafe { zv.ptr::<_zend_property_info>() })
+ .ok_or_else(|| {
+ ext_exception(format!(
+ "flow_php failed to resolve property \"{name}\" on class \"{}\"",
+ ce.name().unwrap_or_default()
+ ))
+ })?;
+
+ Ok(unsafe { (*info).offset })
+}
+
+/// Moves an owned zval directly into a declared-property slot (what native
+/// unserialize() effectively does). Only valid on a freshly created object of
+/// a plain userland class when the zval matches the declared type exactly.
+pub fn write_slot(obj: &mut ZendObject, offset: u32, value: Zval) {
+ unsafe {
+ let slot = std::ptr::from_mut(obj)
+ .cast::()
+ .add(offset as usize)
+ .cast::();
+ std::ptr::write(slot, value);
+ }
+}
+
+/// Reads one property with `BP_VAR_R` semantics; `ZendObject::get_property`
+/// passes `BP_VAR_W`, which trips the readonly-property guard on `HydratorColumn`.
+pub fn read_property(obj: &ZendObject, name: &str) -> Result {
+ let handler = unsafe { obj.handlers.as_ref() }
+ .and_then(|handlers| handlers.read_property)
+ .ok_or_else(|| ext_exception("flow_php failed to resolve a property read handler"))?;
+
+ let mut name_zstr = ZendStr::new(name, false);
+ let mut rv = Zval::new();
+
+ let value = unsafe {
+ handler(
+ std::ptr::from_ref(obj).cast_mut(),
+ std::ptr::from_mut(&mut *name_zstr),
+ 0,
+ std::ptr::null_mut(),
+ std::ptr::from_mut(&mut rv),
+ )
+ .as_ref()
+ }
+ .ok_or_else(|| ext_exception(format!("flow_php failed to read property \"{name}\"")))?;
+ ensure_no_pending_exception(&format!("read property \"{name}\""))?;
+
+ Ok(value.shallow_clone())
+}
+
+/// Clone matching PHP `clone` (`zend_objects_clone_members`, including
+/// `__clone`). Only valid for plain userland classes (no `create_object` state).
+pub fn clone_object(src: &Zval, context: &str) -> Result, PhpException> {
+ let src_obj = src
+ .object()
+ .ok_or_else(|| ext_exception(format!("flow_php expected {context} to be an object")))?;
+
+ let ce = unsafe { src_obj.ce.as_ref() }
+ .ok_or_else(|| ext_exception(format!("flow_php failed to resolve {context} class")))?;
+
+ let mut cloned = ZendObject::new(ce);
+
+ unsafe {
+ zend_objects_clone_members(
+ std::ptr::from_mut(&mut *cloned),
+ std::ptr::from_ref(src_obj).cast_mut(),
+ );
+ }
+
+ Ok(cloned)
+}
+
+pub fn construct_object(
+ ce: &'static ClassEntry,
+ args: Vec<&dyn IntoZvalDyn>,
+ context: &str,
+) -> Result, PhpException> {
+ let obj = ZendObject::new(ce);
+
+ obj.try_call_method("__construct", args)
+ .map_err(|e| ext_exception(format!("flow_php failed to construct {context}: {e:?}")))?;
+ ensure_no_pending_exception(&format!("construct {context}"))?;
+
+ Ok(obj)
+}
+
+fn method_handle(class: &str, method: &str) -> Result {
+ Function::try_from_method(class, method).ok_or_else(|| {
+ ext_exception(format!(
+ "flow_php failed to resolve method {class}::{method}"
+ ))
+ })
+}
+
+fn function_handle(name: &str) -> Result {
+ Function::try_from_function(name)
+ .ok_or_else(|| ext_exception(format!("flow_php failed to resolve function {name}")))
+}
+
+/// Resolves a method to a reference into the class function table WITHOUT
+/// copying the zend_function: the engine writes into userland op_arrays
+/// (run-time cache), and a copy discards those writes on every call.
+fn method_handle_ref(class: &str, method: &str) -> Result<&'static Function, PhpException> {
+ ce_method_ref(find_class(class)?, method)
+}
+
+pub fn ce_method_ref(ce: &ClassEntry, method: &str) -> Result<&'static Function, PhpException> {
+ let function = unsafe {
+ ext_php_rs::ffi::zend_hash_str_find_ptr_lc(
+ &raw const ce.function_table,
+ method.as_ptr().cast(),
+ method.len(),
+ )
+ .cast::()
+ .as_ref()
+ };
+
+ function.ok_or_else(|| {
+ ext_exception(format!(
+ "flow_php failed to resolve method {}::{method}",
+ ce.name().unwrap_or_default()
+ ))
+ })
+}
+
+/// [`call_handle`] for receivers behind a shared reference - the engine does
+/// not require exclusive access to call a method.
+pub fn call_handle_on(
+ func: &Function,
+ object: &ZendObject,
+ args: &mut [Zval],
+ context: &str,
+) -> Result {
+ let mut retval = Zval::new();
+
+ unsafe {
+ zend_call_known_function(
+ std::ptr::from_ref(func).cast_mut(),
+ std::ptr::from_ref(object).cast_mut(),
+ object.ce,
+ std::ptr::from_mut(&mut retval),
+ args.len() as u32,
+ args.as_mut_ptr(),
+ std::ptr::null_mut(),
+ );
+ }
+
+ ensure_no_pending_exception(context)?;
+
+ Ok(retval)
+}
+
+/// Calls a pre-resolved function handle. Argument zvals stay caller-owned
+/// (the engine copies what it keeps), so cached zvals pass as shallow clones.
+pub fn call_handle(
+ func: &Function,
+ object: Option<&mut ZendObject>,
+ args: &mut [Zval],
+ context: &str,
+) -> Result {
+ let mut retval = Zval::new();
+
+ let (object_ptr, called_scope) = match object {
+ Some(obj) => {
+ let scope = obj.ce;
+ (std::ptr::from_mut(obj), scope)
+ }
+ None => (std::ptr::null_mut(), unsafe { func.common.scope }),
+ };
+
+ unsafe {
+ zend_call_known_function(
+ std::ptr::from_ref(func).cast_mut(),
+ object_ptr,
+ called_scope,
+ std::ptr::from_mut(&mut retval),
+ args.len() as u32,
+ args.as_mut_ptr(),
+ std::ptr::null_mut(),
+ );
+ }
+
+ ensure_no_pending_exception(context)?;
+
+ Ok(retval)
+}
+
+pub struct DateTimeFns {
+ pub ce: &'static ClassEntry,
+ pub set_timezone: Function,
+}
+
+pub struct IntervalFns {
+ pub ce: &'static ClassEntry,
+ pub construct: Function,
+ pub spec: Zval,
+ pub names: [ZBox; 8],
+}
+
+pub struct Ctx {
+ pub entries_ce: &'static ClassEntry,
+ pub row_ce: &'static ClassEntry,
+ pub rows_ce: &'static ClassEntry,
+ pub partitions_ce: &'static ClassEntry,
+ pub entries_entries_slot: u32,
+ pub row_entries_slot: u32,
+ pub rows_rows_slot: u32,
+ pub rows_partitions_slot: u32,
+ uuid_ce: Option<(&'static ClassEntry, u32)>,
+ json_ce: Option<(&'static ClassEntry, u32, u32)>,
+ interval: Option,
+ timezone_ce: Option<&'static ClassEntry>,
+ datetime_immutable: Option,
+ datetime_mutable: Option,
+ datetime_interface_ce: Option<&'static ClassEntry>,
+ timezones: HashMap, Zval>,
+ enums: HashMap, Zval>,
+ fn_defined: Option,
+ fn_constant: Option,
+ value_decoder_statics: HashMap<&'static str, &'static Function>,
+ value_encoder_statics: HashMap<&'static str, &'static Function>,
+ schema_decoder: Option>,
+ fn_json_encode: Option,
+ encoder_plan_slots: Option,
+ encoder_column_slots: Option,
+ metadata_map_slot: Option,
+ timezone_get_name: Option<&'static Function>,
+ datetime_encode: HashMap,
+ save_html: HashMap,
+ format_u: Option,
+ frame_segment: Option<(&'static ClassEntry, &'static Function)>,
+}
+
+pub struct DateTimeEncFns {
+ pub get_timestamp: &'static Function,
+ pub format: &'static Function,
+ pub get_timezone: &'static Function,
+}
+
+/// Property-table slot offsets on `Flow\Floe\EncoderPlan`.
+pub struct EncoderPlanSlots {
+ pub schema_body: u32,
+ pub columns: u32,
+}
+
+/// Property-table slot offsets on `Flow\Floe\EncoderColumn`.
+pub struct EncoderColumnSlots {
+ pub name: u32,
+ pub type_fingerprint: u32,
+}
+
+impl Ctx {
+ pub fn new() -> Result {
+ let entries_ce = find_class("Flow\\ETL\\Row\\Entries")?;
+ let row_ce = find_class("Flow\\ETL\\Row")?;
+ let rows_ce = find_class("Flow\\ETL\\Rows")?;
+
+ Ok(Self {
+ entries_ce,
+ row_ce,
+ rows_ce,
+ partitions_ce: find_class("Flow\\Filesystem\\Partitions")?,
+ entries_entries_slot: property_offset(entries_ce, "entries")?,
+ row_entries_slot: property_offset(row_ce, "entries")?,
+ rows_rows_slot: property_offset(rows_ce, "rows")?,
+ rows_partitions_slot: property_offset(rows_ce, "partitions")?,
+ uuid_ce: None,
+ json_ce: None,
+ interval: None,
+ timezone_ce: None,
+ datetime_immutable: None,
+ datetime_mutable: None,
+ datetime_interface_ce: None,
+ timezones: HashMap::new(),
+ enums: HashMap::new(),
+ fn_defined: None,
+ fn_constant: None,
+ value_decoder_statics: HashMap::new(),
+ value_encoder_statics: HashMap::new(),
+ schema_decoder: None,
+ fn_json_encode: None,
+ encoder_plan_slots: None,
+ encoder_column_slots: None,
+ metadata_map_slot: None,
+ timezone_get_name: None,
+ datetime_encode: HashMap::new(),
+ save_html: HashMap::new(),
+ format_u: None,
+ frame_segment: None,
+ })
+ }
+
+ /// Cached `Flow\Floe\FrameSegment` class entry + constructor handle.
+ pub fn frame_segment(
+ &mut self,
+ ) -> Result<(&'static ClassEntry, &'static Function), PhpException> {
+ if self.frame_segment.is_none() {
+ let ce = find_class("Flow\\Floe\\FrameSegment")?;
+ self.frame_segment = Some((ce, ce_method_ref(ce, "__construct")?));
+ }
+
+ Ok(self.frame_segment.expect("just initialized"))
+ }
+
+ pub fn metadata_map_slot(&mut self) -> Result {
+ if self.metadata_map_slot.is_none() {
+ self.metadata_map_slot = Some(property_offset(
+ find_class("Flow\\ETL\\Schema\\Metadata")?,
+ "map",
+ )?);
+ }
+
+ Ok(self.metadata_map_slot.expect("just initialized"))
+ }
+
+ pub fn timezone_get_name(&mut self) -> Result<&'static Function, PhpException> {
+ if self.timezone_get_name.is_none() {
+ self.timezone_get_name = Some(method_handle_ref("DateTimeZone", "getName")?);
+ }
+
+ Ok(self.timezone_get_name.expect("just initialized"))
+ }
+
+ pub fn datetime_interface(&mut self) -> Result<&'static ClassEntry, PhpException> {
+ if self.datetime_interface_ce.is_none() {
+ self.datetime_interface_ce = Some(find_class("DateTimeInterface")?);
+ }
+
+ Ok(self.datetime_interface_ce.expect("just initialized"))
+ }
+
+ /// Per-datetime-class encode handles (getTimestamp/format/getTimezone).
+ pub fn datetime_encode_fns(
+ &mut self,
+ ce: *const ClassEntry,
+ ) -> Result<&DateTimeEncFns, PhpException> {
+ let key = ce as usize;
+
+ if let std::collections::hash_map::Entry::Vacant(entry) = self.datetime_encode.entry(key) {
+ let ce_ref = unsafe { ce.as_ref() }
+ .ok_or_else(|| ext_exception("flow_php failed to resolve a datetime class"))?;
+
+ entry.insert(DateTimeEncFns {
+ get_timestamp: ce_method_ref(ce_ref, "getTimestamp")?,
+ format: ce_method_ref(ce_ref, "format")?,
+ get_timezone: ce_method_ref(ce_ref, "getTimezone")?,
+ });
+ }
+
+ Ok(self.datetime_encode.get(&key).expect("just inserted"))
+ }
+
+ pub fn save_html(&mut self, ce: *const ClassEntry) -> Result<&'static Function, PhpException> {
+ let key = ce as usize;
+
+ if let std::collections::hash_map::Entry::Vacant(entry) = self.save_html.entry(key) {
+ let ce_ref = unsafe { ce.as_ref() }
+ .ok_or_else(|| ext_exception("flow_php failed to resolve an HTML class"))?;
+ entry.insert(ce_method_ref(ce_ref, "saveHtml")?);
+ }
+
+ Ok(self.save_html.get(&key).expect("just inserted"))
+ }
+
+ /// Cached `"u"` format-string zval for microsecond reads.
+ pub fn format_u(&mut self) -> Result<&Zval, PhpException> {
+ if self.format_u.is_none() {
+ self.format_u = Some(zval_str(b"u"));
+ }
+
+ Ok(self.format_u.as_ref().expect("just initialized"))
+ }
+
+ /// Cached handles to `ValueEncoder`'s public static markup serializers.
+ pub fn value_encoder_static(
+ &mut self,
+ method: &'static str,
+ ) -> Result<&'static Function, PhpException> {
+ if !self.value_encoder_statics.contains_key(method) {
+ let handle = method_handle_ref("Flow\\Floe\\ValueEncoder", method)?;
+ self.value_encoder_statics.insert(method, handle);
+ }
+
+ Ok(self
+ .value_encoder_statics
+ .get(method)
+ .expect("just inserted"))
+ }
+
+ pub fn json(&mut self) -> Result<(&'static ClassEntry, u32, u32), PhpException> {
+ if self.json_ce.is_none() {
+ let ce = find_class("Flow\\Types\\Value\\Json")?;
+ self.json_ce = Some((
+ ce,
+ property_offset(ce, "value")?,
+ property_offset(ce, "isObject")?,
+ ));
+ }
+
+ Ok(self.json_ce.expect("just initialized"))
+ }
+
+ /// Enum cases cached per `Class::Case`, resolved with the same validation
+ /// and errors as `ValueDecoder` (enum_exists + defined + constant).
+ pub fn enum_case(&mut self, class: &[u8], case: &[u8]) -> Result {
+ let mut cache_key = Vec::with_capacity(class.len() + case.len() + 2);
+ cache_key.extend_from_slice(class);
+ cache_key.extend_from_slice(b"::");
+ cache_key.extend_from_slice(case);
+
+ if let Some(cached) = self.enums.get(&cache_key) {
+ return Ok(cached.shallow_clone());
+ }
+
+ let constant_name = std::str::from_utf8(&cache_key)
+ .map_err(|_| ext_exception("flow_php expected an enum name to be UTF-8"))?
+ .to_string();
+ let class_name = std::str::from_utf8(class)
+ .expect("validated above")
+ .to_string();
+ let case_name = std::str::from_utf8(case)
+ .expect("validated above")
+ .to_string();
+
+ let is_enum = ClassEntry::try_find(&class_name)
+ .is_some_and(|ce| ce.flags().contains(ClassFlags::Enum));
+
+ if !is_enum {
+ return Err(ext_exception(format!(
+ "flow_php cannot restore enum of class \"{class_name}\", enum not found"
+ )));
+ }
+
+ if self.fn_defined.is_none() {
+ self.fn_defined = Some(function_handle("defined")?);
+ self.fn_constant = Some(function_handle("constant")?);
+ }
+
+ let constant_name_zv = zval_str(constant_name.as_bytes());
+
+ let defined = call_handle(
+ &self.fn_defined.expect("just initialized"),
+ None,
+ &mut [constant_name_zv.shallow_clone()],
+ "check an enum case",
+ )?;
+
+ if !defined.bool().unwrap_or(false) {
+ return Err(ext_exception(format!(
+ "flow_php cannot restore enum case \"{class_name}::{case_name}\""
+ )));
+ }
+
+ let case_zv = call_handle(
+ &self.fn_constant.expect("just initialized"),
+ None,
+ &mut [constant_name_zv],
+ "restore an enum case",
+ )?;
+
+ if !case_zv.is_object() {
+ return Err(ext_exception(format!(
+ "flow_php cannot restore enum case \"{class_name}::{case_name}\""
+ )));
+ }
+
+ let result = case_zv.shallow_clone();
+ self.enums.insert(cache_key, case_zv);
+
+ Ok(result)
+ }
+
+ /// Cached handles to `ValueDecoder`'s public static markup restorers.
+ pub fn value_decoder_static(
+ &mut self,
+ method: &'static str,
+ ) -> Result<&'static Function, PhpException> {
+ if !self.value_decoder_statics.contains_key(method) {
+ let handle = method_handle_ref("Flow\\Floe\\ValueDecoder", method)?;
+ self.value_decoder_statics.insert(method, handle);
+ }
+
+ Ok(self
+ .value_decoder_statics
+ .get(method)
+ .expect("just inserted"))
+ }
+
+ pub fn uuid(&mut self) -> Result<(&'static ClassEntry, u32), PhpException> {
+ if self.uuid_ce.is_none() {
+ let ce = find_class("Flow\\Types\\Value\\Uuid")?;
+ self.uuid_ce = Some((ce, property_offset(ce, "value")?));
+ }
+
+ Ok(self.uuid_ce.expect("just initialized"))
+ }
+
+ pub fn interval(&mut self) -> Result<&mut IntervalFns, PhpException> {
+ if self.interval.is_none() {
+ self.interval = Some(IntervalFns {
+ ce: find_class("DateInterval")?,
+ construct: method_handle("DateInterval", "__construct")?,
+ spec: zval_str(b"PT0S"),
+ names: [
+ ZendStr::new("y", false),
+ ZendStr::new("m", false),
+ ZendStr::new("d", false),
+ ZendStr::new("h", false),
+ ZendStr::new("i", false),
+ ZendStr::new("s", false),
+ ZendStr::new("invert", false),
+ ZendStr::new("f", false),
+ ],
+ });
+ }
+
+ Ok(self.interval.as_mut().expect("just initialized"))
+ }
+
+ /// Datetime class entry + pre-resolved setTimezone handle per class.
+ pub fn datetime_fns(&mut self, mutable: bool) -> Result<&DateTimeFns, PhpException> {
+ let slot = if mutable {
+ &mut self.datetime_mutable
+ } else {
+ &mut self.datetime_immutable
+ };
+
+ if slot.is_none() {
+ let class = if mutable {
+ "DateTime"
+ } else {
+ "DateTimeImmutable"
+ };
+ *slot = Some(DateTimeFns {
+ ce: find_class(class)?,
+ set_timezone: method_handle(class, "setTimezone")?,
+ });
+ }
+
+ Ok(slot.as_ref().expect("just initialized"))
+ }
+
+ /// `DateTimeZone` instances cached per timezone name, mirroring
+ /// `ValueDecoder::$timezones`.
+ pub fn timezone(&mut self, name: &[u8]) -> Result<&Zval, PhpException> {
+ if !self.timezones.contains_key(name) {
+ if self.timezone_ce.is_none() {
+ self.timezone_ce = Some(find_class("DateTimeZone")?);
+ }
+
+ let name_string = std::str::from_utf8(name)
+ .map_err(|_| ext_exception("flow_php found a non UTF-8 timezone name"))?
+ .to_string();
+ let mut timezone = construct_object(
+ self.timezone_ce.expect("just initialized"),
+ vec![&name_string as &dyn IntoZvalDyn],
+ "DateTimeZone",
+ )?;
+
+ let mut zv = Zval::new();
+ zv.set_object(&mut timezone);
+ self.timezones.insert(name.to_vec(), zv);
+ }
+
+ Ok(self.timezones.get(name).expect("just inserted"))
+ }
+
+ /// The canonical PHP schema decoder, constructed lazily - schema frames are
+ /// the cold path and reusing `SchemaDecoder` prevents semantic drift.
+ pub fn schema_decoder(&mut self) -> Result<&ZBox, PhpException> {
+ if self.schema_decoder.is_none() {
+ let mut value_decoder = construct_object(
+ find_class("Flow\\Floe\\ValueDecoder")?,
+ vec![],
+ "Flow\\Floe\\ValueDecoder",
+ )?;
+ // EntryInstantiator declares no constructor; defaults apply on init.
+ let mut entry_instantiator =
+ ZendObject::new(find_class("Flow\\Floe\\EntryInstantiator")?);
+
+ let mut value_decoder_zv = Zval::new();
+ value_decoder_zv.set_object(&mut value_decoder);
+ let mut entry_instantiator_zv = Zval::new();
+ entry_instantiator_zv.set_object(&mut entry_instantiator);
+
+ self.schema_decoder = Some(construct_object(
+ find_class("Flow\\Floe\\SchemaDecoder")?,
+ vec![
+ &value_decoder_zv as &dyn IntoZvalDyn,
+ &entry_instantiator_zv as &dyn IntoZvalDyn,
+ ],
+ "Flow\\Floe\\SchemaDecoder",
+ )?);
+ }
+
+ Ok(self.schema_decoder.as_ref().expect("just initialized"))
+ }
+
+ pub fn json_encode(&mut self) -> Result<&Function, PhpException> {
+ if self.fn_json_encode.is_none() {
+ self.fn_json_encode = Some(function_handle("json_encode")?);
+ }
+
+ Ok(self.fn_json_encode.as_ref().expect("just initialized"))
+ }
+
+ pub fn encoder_plan_slots(&mut self) -> Result<&EncoderPlanSlots, PhpException> {
+ if self.encoder_plan_slots.is_none() {
+ let ce = find_class("Flow\\Floe\\EncoderPlan")?;
+ self.encoder_plan_slots = Some(EncoderPlanSlots {
+ schema_body: property_offset(ce, "schemaBody")?,
+ columns: property_offset(ce, "columns")?,
+ });
+ }
+
+ Ok(self.encoder_plan_slots.as_ref().expect("just initialized"))
+ }
+
+ pub fn encoder_column_slots(&mut self) -> Result<&EncoderColumnSlots, PhpException> {
+ if self.encoder_column_slots.is_none() {
+ let ce = find_class("Flow\\Floe\\EncoderColumn")?;
+ self.encoder_column_slots = Some(EncoderColumnSlots {
+ name: property_offset(ce, "name")?,
+ type_fingerprint: property_offset(ce, "typeFingerprint")?,
+ });
+ }
+
+ Ok(self
+ .encoder_column_slots
+ .as_ref()
+ .expect("just initialized"))
+ }
+}
diff --git a/src/extension/flow-php-ext/src/encode.rs b/src/extension/flow-php-ext/src/encode.rs
new file mode 100644
index 0000000000..95675fc4e5
--- /dev/null
+++ b/src/extension/flow-php-ext/src/encode.rs
@@ -0,0 +1,768 @@
+//! Floe ROW frame-body encoder mirroring `Flow\Floe\RowEncoder::encode`
+//! byte-for-byte; entry properties are read through cached slot offsets.
+
+use ext_php_rs::exception::PhpException;
+use ext_php_rs::ffi::zend_ulong;
+use ext_php_rs::types::{ZendHashTable, ZendObject, ZendStr, Zval};
+use ext_php_rs::zend::ClassEntry;
+
+use crate::ctx::{call_handle, call_handle_on, property_offset, read_property, Ctx};
+use crate::exception::ext_exception;
+use crate::format::{
+ write_u32, DATETIME_IMMUTABLE, DATETIME_MUTABLE, KEY_INTEGER, KEY_STRING, TAG_ARRAY,
+ TAG_BOOLEAN, TAG_DATETIME, TAG_FLOAT, TAG_INTEGER, TAG_JSON, TAG_NULL, TAG_STRING, TAG_UUID,
+ VALUE_ABSENT, VALUE_NULL, VALUE_NULL_FROM_NULL, VALUE_PRESENT,
+};
+use crate::plan::{entry_class_for_type, parse_schema_json, TypeJson};
+
+enum EncodeMapKey {
+ Integer,
+ String,
+ Dynamic,
+}
+
+/// Declared structure element key after PHP's array-key coercion, mirroring
+/// `ValueEncoder::structureEncoder`.
+enum DeclaredKey {
+ Index(i64),
+ Str(Vec),
+}
+
+/// Recursive value encoder mirroring `ValueEncoder::encoderFor`.
+enum Encoder {
+ Integer,
+ Float,
+ Boolean,
+ String,
+ Null,
+ Dynamic,
+ DateTime,
+ Interval,
+ Uuid,
+ Json,
+ TimeZone,
+ Enum,
+ Xml,
+ XmlElement,
+ Html,
+ HtmlElement,
+ List(Box),
+ Map(EncodeMapKey, Box),
+ Structure(Vec<(DeclaredKey, Vec, Encoder)>, bool),
+ Optional(Box),
+}
+
+fn build_encoder(type_json: &TypeJson) -> Result {
+ let missing = |part: &str| {
+ ext_exception(format!(
+ "flow_php schema JSON for type \"{}\" is missing its {part}",
+ type_json.type_
+ ))
+ };
+
+ Ok(match type_json.type_.as_str() {
+ "integer" | "positive_integer" => Encoder::Integer,
+ "float" => Encoder::Float,
+ "boolean" => Encoder::Boolean,
+ "string" | "non_empty_string" | "numeric-string" | "class_string" => Encoder::String,
+ "null" => Encoder::Null,
+ "mixed" | "union" | "scalar" | "literal" | "array" => Encoder::Dynamic,
+ "datetime" | "date" => Encoder::DateTime,
+ "time" => Encoder::Interval,
+ "uuid" => Encoder::Uuid,
+ "json" => Encoder::Json,
+ "timezone" => Encoder::TimeZone,
+ "enum" => Encoder::Enum,
+ "xml" => Encoder::Xml,
+ "xml_element" => Encoder::XmlElement,
+ "html" => Encoder::Html,
+ "html_element" => Encoder::HtmlElement,
+ "list" => Encoder::List(Box::new(build_encoder(
+ type_json.element().ok_or_else(|| missing("element"))?,
+ )?)),
+ "map" => {
+ let key = match type_json
+ .key()
+ .ok_or_else(|| missing("key"))?
+ .type_
+ .as_str()
+ {
+ "integer" => EncodeMapKey::Integer,
+ "string" => EncodeMapKey::String,
+ _ => EncodeMapKey::Dynamic,
+ };
+
+ Encoder::Map(
+ key,
+ Box::new(build_encoder(
+ type_json.value().ok_or_else(|| missing("value"))?,
+ )?),
+ )
+ }
+ "structure" => {
+ let mut elements = Vec::new();
+
+ for (name, element) in type_json.all_elements() {
+ let bytes = name.clone().into_bytes();
+ let declared = match crate::ctx::array_key_index(&bytes) {
+ Some(index) => DeclaredKey::Index(index),
+ None => DeclaredKey::Str(bytes.clone()),
+ };
+ elements.push((declared, bytes, build_encoder(element)?));
+ }
+
+ Encoder::Structure(elements, type_json.allow_extra())
+ }
+ "optional" => Encoder::Optional(Box::new(build_encoder(
+ type_json.base().ok_or_else(|| missing("base"))?,
+ )?)),
+ other => {
+ return Err(ext_exception(format!(
+ "flow_php does not support values of type \"{other}\" in this build"
+ )));
+ }
+ })
+}
+
+struct EncodeColumn {
+ name: Vec,
+ value_slot: u32,
+ definition_slot: u32,
+ encoder: Encoder,
+ /// (definition ce, `metadata` slot, `type` slot) - resolved lazily from the
+ /// first definition instance seen for this column.
+ definition_slots: Option<(*const ClassEntry, u32, u32)>,
+}
+
+pub struct EncodePlan {
+ columns: Vec,
+}
+
+/// `HASH_FLAG_PACKED` from zend_types.h.
+const HASH_FLAG_PACKED: u8 = 1 << 2;
+
+/// Raw order-preserving iteration over packed and bucket layouts, binary-safe
+/// for arbitrary keys (ext-php-rs's `iter()` panics on non-UTF-8 string keys).
+pub fn ht_for_each(
+ ht: &ZendHashTable,
+ mut f: impl FnMut(Option<&ZendStr>, zend_ulong, &Zval) -> Result<(), PhpException>,
+) -> Result<(), PhpException> {
+ let packed = unsafe { ht.u.v.flags } & HASH_FLAG_PACKED != 0;
+
+ for index in 0..ht.nNumUsed {
+ if packed {
+ let value = unsafe { &*ht.__bindgen_anon_1.arPacked.add(index as usize) };
+
+ if value.get_type() == ext_php_rs::flags::DataType::Undef {
+ continue;
+ }
+
+ f(None, zend_ulong::from(index), value)?;
+ } else {
+ let bucket = unsafe { &*ht.__bindgen_anon_1.arData.add(index as usize) };
+
+ if bucket.val.get_type() == ext_php_rs::flags::DataType::Undef {
+ continue;
+ }
+
+ let key = unsafe { bucket.key.as_ref() };
+ f(key, bucket.h, &bucket.val)?;
+ }
+ }
+
+ Ok(())
+}
+
+pub(crate) fn read_slot(obj: &ZendObject, offset: u32) -> &Zval {
+ unsafe {
+ &*std::ptr::from_ref(obj)
+ .cast::()
+ .add(offset as usize)
+ .cast::()
+ }
+}
+
+pub(crate) fn expect_object<'a>(
+ zv: &'a Zval,
+ context: &str,
+) -> Result<&'a ZendObject, PhpException> {
+ zv.object()
+ .ok_or_else(|| ext_exception(format!("flow_php expected {context} to be an object")))
+}
+
+fn write_len_prefixed(out: &mut Vec, bytes: &[u8]) {
+ write_u32(out, bytes.len() as u32);
+ out.extend_from_slice(bytes);
+}
+
+/// Builds the encode plan from a SCHEMA frame body (the JSON the PHP
+/// `SchemaEncoder` emits) - once built, no PHP call is needed to encode.
+pub fn build_encode_plan(schema_json: &[u8]) -> Result {
+ let definitions = parse_schema_json(schema_json)?;
+
+ let mut columns = Vec::with_capacity(definitions.len());
+
+ for definition in &definitions {
+ let entry_ce = entry_class_for_type(&definition.type_)?;
+
+ columns.push(EncodeColumn {
+ name: definition.name.clone().into_bytes(),
+ value_slot: property_offset(entry_ce, "value")?,
+ definition_slot: property_offset(entry_ce, "definition")?,
+ encoder: build_encoder(&definition.type_)?,
+ definition_slots: None,
+ });
+ }
+
+ Ok(EncodePlan { columns })
+}
+
+/// Encodes one Row into a bare ROW frame body (no length prefix, no frame type).
+pub fn encode_row_body(
+ plan: &mut EncodePlan,
+ row: &Zval,
+ ctx: &mut Ctx,
+) -> Result, PhpException> {
+ let row_obj = expect_object(row, "a Row")?;
+ let entries_obj = expect_object(read_slot(row_obj, ctx.row_entries_slot), "Row entries")?;
+ let entries_ht = read_slot(entries_obj, ctx.entries_entries_slot)
+ .array()
+ .ok_or_else(|| ext_exception("flow_php expected Entries to hold an array"))?;
+
+ let mut out = Vec::with_capacity(1024);
+ encode_row(plan, entries_ht, &mut out, ctx)?;
+
+ Ok(out)
+}
+
+fn definition_slots(
+ column: &mut EncodeColumn,
+ definition: &ZendObject,
+) -> Result<(u32, u32), PhpException> {
+ let ce = definition.ce.cast_const();
+
+ if let Some((cached_ce, metadata_slot, type_slot)) = column.definition_slots {
+ if std::ptr::eq(cached_ce, ce) {
+ return Ok((metadata_slot, type_slot));
+ }
+ }
+
+ let ce_ref = unsafe { ce.as_ref() }
+ .ok_or_else(|| ext_exception("flow_php failed to resolve a definition class"))?;
+ let metadata_slot = property_offset(ce_ref, "metadata")?;
+ let type_slot = property_offset(ce_ref, "type")?;
+ column.definition_slots = Some((ce, metadata_slot, type_slot));
+
+ Ok((metadata_slot, type_slot))
+}
+
+fn encode_row(
+ plan: &mut EncodePlan,
+ entries_ht: &ZendHashTable,
+ out: &mut Vec,
+ ctx: &mut Ctx,
+) -> Result<(), PhpException> {
+ for column in &mut plan.columns {
+ let Some(entry_zv) = ht_find(entries_ht, &column.name) else {
+ out.push(VALUE_ABSENT);
+
+ continue;
+ };
+
+ let entry = expect_object(entry_zv, "a row entry")?;
+ let value = read_slot(entry, column.value_slot);
+
+ if value.is_null() {
+ let definition =
+ expect_object(read_slot(entry, column.definition_slot), "a definition")?;
+ let (metadata_slot, _) = definition_slots(column, definition)?;
+ let metadata =
+ expect_object(read_slot(definition, metadata_slot), "definition metadata")?;
+ let map = read_slot(metadata, ctx.metadata_map_slot()?)
+ .array()
+ .ok_or_else(|| ext_exception("flow_php expected Metadata to hold an array"))?;
+
+ out.push(if crate::ctx::ht_contains(map, b"from_null") {
+ VALUE_NULL_FROM_NULL
+ } else {
+ VALUE_NULL
+ });
+ } else {
+ out.push(VALUE_PRESENT);
+ encode_value(&column.encoder, value, out, ctx)?;
+ }
+ }
+
+ Ok(())
+}
+
+fn encode_value(
+ encoder: &Encoder,
+ value: &Zval,
+ out: &mut Vec,
+ ctx: &mut Ctx,
+) -> Result<(), PhpException> {
+ match encoder {
+ Encoder::Integer => out.extend_from_slice(
+ &value
+ .long()
+ .ok_or_else(|| ext_exception("flow_php expected an integer value"))?
+ .to_le_bytes(),
+ ),
+ Encoder::Float => out.extend_from_slice(
+ &php_float(value)
+ .ok_or_else(|| ext_exception("flow_php expected a float value"))?
+ .to_le_bytes(),
+ ),
+ Encoder::Boolean => {
+ out.push(u8::from(value.bool().ok_or_else(|| {
+ ext_exception("flow_php expected a boolean value")
+ })?))
+ }
+ Encoder::String => write_len_prefixed(
+ out,
+ value
+ .zend_str()
+ .ok_or_else(|| ext_exception("flow_php expected a string value"))?
+ .as_bytes(),
+ ),
+ Encoder::Null => {}
+ Encoder::Dynamic => encode_dynamic(value, out, ctx)?,
+ Encoder::DateTime => encode_datetime(expect_object(value, "a datetime value")?, out, ctx)?,
+ Encoder::Interval => encode_interval(expect_object(value, "a time value")?, out, ctx)?,
+ Encoder::Uuid => {
+ let uuid = expect_object(value, "a uuid value")?;
+ let (_, value_slot) = ctx.uuid()?;
+ let bytes = read_slot(uuid, value_slot)
+ .zend_str()
+ .ok_or_else(|| ext_exception("flow_php expected Uuid to hold a string"))?;
+ out.extend_from_slice(bytes.as_bytes());
+ }
+ Encoder::Json => encode_json(expect_object(value, "a json value")?, out, ctx)?,
+ Encoder::TimeZone => {
+ let timezone = expect_object(value, "a timezone value")?;
+ let name = call_handle_on(
+ ctx.timezone_get_name()?,
+ timezone,
+ &mut [],
+ "read a timezone name",
+ )?;
+ write_len_prefixed(
+ out,
+ name.zend_str()
+ .ok_or_else(|| ext_exception("flow_php expected a timezone name string"))?
+ .as_bytes(),
+ );
+ }
+ Encoder::Enum => {
+ let case = expect_object(value, "an enum value")?;
+ let class = unsafe { case.ce.as_ref() }
+ .and_then(ClassEntry::name)
+ .ok_or_else(|| ext_exception("flow_php failed to resolve an enum class"))?;
+ write_len_prefixed(out, class.as_bytes());
+
+ let name = read_property(case, "name")?;
+ write_len_prefixed(
+ out,
+ name.zend_str()
+ .ok_or_else(|| ext_exception("flow_php expected an enum case name"))?
+ .as_bytes(),
+ );
+ }
+ Encoder::Xml => encode_via_static(value, out, ctx, "xmlDocumentToString")?,
+ Encoder::XmlElement => encode_via_static(value, out, ctx, "xmlElementToString")?,
+ Encoder::HtmlElement => encode_via_static(value, out, ctx, "htmlElementToString")?,
+ Encoder::Html => {
+ let document = expect_object(value, "an html value")?;
+ let html = call_handle_on(
+ ctx.save_html(document.ce.cast_const())?,
+ document,
+ &mut [],
+ "convert an HTML document",
+ )?;
+ write_len_prefixed(
+ out,
+ html.zend_str()
+ .ok_or_else(|| ext_exception("flow_php expected saveHtml to return a string"))?
+ .as_bytes(),
+ );
+ }
+ Encoder::List(element) => {
+ let values = value
+ .array()
+ .ok_or_else(|| ext_exception("flow_php expected a list value"))?;
+ write_u32(out, values.len() as u32);
+ ht_for_each(values, |_, _, item| encode_value(element, item, out, ctx))?;
+ }
+ Encoder::Map(key, element) => {
+ let values = value
+ .array()
+ .ok_or_else(|| ext_exception("flow_php expected a map value"))?;
+ write_u32(out, values.len() as u32);
+ ht_for_each(values, |map_key, index, item| {
+ match key {
+ EncodeMapKey::Integer => {
+ let int_key = match map_key {
+ None => index as i64,
+ Some(_) => {
+ return Err(ext_exception("flow_php expected an integer map key"));
+ }
+ };
+ out.extend_from_slice(&int_key.to_le_bytes());
+ }
+ EncodeMapKey::String => match map_key {
+ // numeric-string keys coerced to int by the PHP array
+ // are cast back, like ValueEncoder
+ None => write_len_prefixed(out, (index as i64).to_string().as_bytes()),
+ Some(key) => write_len_prefixed(out, key.as_bytes()),
+ },
+ EncodeMapKey::Dynamic => match map_key {
+ None => {
+ out.push(TAG_INTEGER);
+ out.extend_from_slice(&(index as i64).to_le_bytes());
+ }
+ Some(key) => {
+ out.push(TAG_STRING);
+ write_len_prefixed(out, key.as_bytes());
+ }
+ },
+ }
+
+ encode_value(element, item, out, ctx)
+ })?;
+ }
+ Encoder::Structure(elements, allow_extra) => {
+ let values = value
+ .array()
+ .ok_or_else(|| ext_exception("flow_php expected a structure value"))?;
+
+ for (declared, _, element) in elements {
+ let item = match declared {
+ DeclaredKey::Index(index) => values.get_index(*index),
+ DeclaredKey::Str(key) => ht_find(values, key),
+ };
+
+ match item {
+ None => out.push(VALUE_ABSENT),
+ Some(item) if item.is_null() => out.push(VALUE_NULL),
+ Some(item) => {
+ out.push(VALUE_PRESENT);
+ encode_value(element, item, out, ctx)?;
+ }
+ }
+ }
+
+ if *allow_extra {
+ let is_declared = |key: Option<&ZendStr>, index: zend_ulong| {
+ elements
+ .iter()
+ .any(|(declared, _, _)| match (declared, key) {
+ (DeclaredKey::Index(declared_index), None) => {
+ *declared_index == index as i64
+ }
+ (DeclaredKey::Str(declared_key), Some(key)) => {
+ declared_key.as_slice() == key.as_bytes()
+ }
+ _ => false,
+ })
+ };
+
+ let mut extra_count = 0u32;
+ ht_for_each(values, |key, index, _| {
+ if !is_declared(key, index) {
+ extra_count += 1;
+ }
+
+ Ok(())
+ })?;
+ write_u32(out, extra_count);
+
+ ht_for_each(values, |key, index, item| {
+ if is_declared(key, index) {
+ return Ok(());
+ }
+
+ match key {
+ None => write_len_prefixed(out, (index as i64).to_string().as_bytes()),
+ Some(key) => write_len_prefixed(out, key.as_bytes()),
+ }
+
+ encode_dynamic(item, out, ctx)
+ })?;
+ }
+ }
+ Encoder::Optional(base) => {
+ if value.is_null() {
+ out.push(VALUE_NULL);
+ } else {
+ out.push(VALUE_PRESENT);
+ encode_value(base, value, out, ctx)?;
+ }
+ }
+ }
+
+ Ok(())
+}
+
+/// Accepts ints where FloatType is declared, permissive like `pack('e')`.
+fn php_float(value: &Zval) -> Option {
+ value.double().or_else(|| value.long().map(|l| l as f64))
+}
+
+fn ht_find<'a>(ht: &'a ZendHashTable, key: &[u8]) -> Option<&'a Zval> {
+ match crate::ctx::array_key_index(key) {
+ Some(index) => ht.get_index(index),
+ None => unsafe {
+ ext_php_rs::ffi::zend_hash_str_find(
+ std::ptr::from_ref(ht).cast_mut(),
+ key.as_ptr().cast(),
+ key.len(),
+ )
+ .as_ref()
+ },
+ }
+}
+
+/// Mirrors `ValueEncoder::encodeDynamic`.
+fn encode_dynamic(value: &Zval, out: &mut Vec, ctx: &mut Ctx) -> Result<(), PhpException> {
+ if value.is_null() {
+ out.push(TAG_NULL);
+
+ return Ok(());
+ }
+
+ if let Some(long) = value.long() {
+ out.push(TAG_INTEGER);
+ out.extend_from_slice(&long.to_le_bytes());
+
+ return Ok(());
+ }
+
+ if let Some(double) = value.double() {
+ out.push(TAG_FLOAT);
+ out.extend_from_slice(&double.to_le_bytes());
+
+ return Ok(());
+ }
+
+ if let Some(boolean) = value.bool() {
+ out.push(TAG_BOOLEAN);
+ out.push(u8::from(boolean));
+
+ return Ok(());
+ }
+
+ if let Some(string) = value.zend_str() {
+ out.push(TAG_STRING);
+ write_len_prefixed(out, string.as_bytes());
+
+ return Ok(());
+ }
+
+ if let Some(array) = value.array() {
+ out.push(TAG_ARRAY);
+ write_u32(out, array.len() as u32);
+
+ return ht_for_each(array, |key, index, item| {
+ match key {
+ None => {
+ out.push(KEY_INTEGER);
+ out.extend_from_slice(&(index as i64).to_le_bytes());
+ }
+ Some(key) => {
+ out.push(KEY_STRING);
+ write_len_prefixed(out, key.as_bytes());
+ }
+ }
+
+ encode_dynamic(item, out, ctx)
+ });
+ }
+
+ if let Some(object) = value.object() {
+ if object.instance_of(ctx.datetime_interface()?) {
+ out.push(TAG_DATETIME);
+
+ return encode_datetime(object, out, ctx);
+ }
+
+ let (uuid_ce, uuid_value_slot) = ctx.uuid()?;
+
+ if std::ptr::eq(object.ce.cast_const(), std::ptr::from_ref(uuid_ce)) {
+ out.push(TAG_UUID);
+ out.extend_from_slice(
+ read_slot(object, uuid_value_slot)
+ .zend_str()
+ .ok_or_else(|| ext_exception("flow_php expected Uuid to hold a string"))?
+ .as_bytes(),
+ );
+
+ return Ok(());
+ }
+
+ let (json_ce, ..) = ctx.json()?;
+
+ if std::ptr::eq(object.ce.cast_const(), std::ptr::from_ref(json_ce)) {
+ out.push(TAG_JSON);
+
+ return encode_json(object, out, ctx);
+ }
+ }
+
+ Err(ext_exception(format!(
+ "flow_php does not support values of type \"{}\" in mixed/union context",
+ debug_type(value)
+ )))
+}
+
+fn debug_type(value: &Zval) -> String {
+ if let Some(object) = value.object() {
+ return unsafe { object.ce.as_ref() }
+ .and_then(ClassEntry::name)
+ .unwrap_or("object")
+ .to_string();
+ }
+
+ format!("{:?}", value.get_type())
+}
+
+/// Mirrors `ValueEncoder::encodeDateTime`.
+fn encode_datetime(
+ value: &ZendObject,
+ out: &mut Vec,
+ ctx: &mut Ctx,
+) -> Result<(), PhpException> {
+ let ce = value.ce.cast_const();
+ let immutable_ce = std::ptr::from_ref(ctx.datetime_fns(false)?.ce);
+ let mutable_ce = std::ptr::from_ref(ctx.datetime_fns(true)?.ce);
+
+ if std::ptr::eq(ce, immutable_ce) {
+ out.push(DATETIME_IMMUTABLE);
+ } else if std::ptr::eq(ce, mutable_ce) {
+ out.push(DATETIME_MUTABLE);
+ } else {
+ let class = unsafe { ce.as_ref() }
+ .and_then(ClassEntry::name)
+ .ok_or_else(|| ext_exception("flow_php failed to resolve a datetime class"))?;
+
+ return Err(ext_exception(format!(
+ "Floe supports only DateTime and DateTimeImmutable, got {class} - convert custom datetime instances before writing"
+ )));
+ }
+
+ let enc = ctx.datetime_encode_fns(ce)?;
+ let get_timestamp = enc.get_timestamp;
+ let format = enc.format;
+ let get_timezone = enc.get_timezone;
+
+ let timestamp = call_handle_on(get_timestamp, value, &mut [], "read a timestamp")?
+ .long()
+ .ok_or_else(|| ext_exception("flow_php expected getTimestamp to return an int"))?;
+
+ let format_u = ctx.format_u()?.shallow_clone();
+ let microseconds_zv =
+ call_handle_on(format, value, &mut [format_u], "read datetime microseconds")?;
+ let microseconds: u32 = microseconds_zv
+ .zend_str()
+ .and_then(|s| std::str::from_utf8(s.as_bytes()).ok())
+ .and_then(|s| s.parse().ok())
+ .ok_or_else(|| ext_exception("flow_php expected format('u') to return digits"))?;
+
+ let timezone_zv = call_handle_on(get_timezone, value, &mut [], "read a timezone")?;
+ let timezone = expect_object(&timezone_zv, "a timezone")?;
+ let name = call_handle_on(
+ ctx.timezone_get_name()?,
+ timezone,
+ &mut [],
+ "read a timezone name",
+ )?;
+
+ out.extend_from_slice(×tamp.to_le_bytes());
+ write_u32(out, microseconds);
+ write_len_prefixed(
+ out,
+ name.zend_str()
+ .ok_or_else(|| ext_exception("flow_php expected a timezone name string"))?
+ .as_bytes(),
+ );
+
+ Ok(())
+}
+
+/// Mirrors `ValueEncoder::encodeInterval`.
+fn encode_interval(
+ value: &ZendObject,
+ out: &mut Vec,
+ _ctx: &mut Ctx,
+) -> Result<(), PhpException> {
+ let read = |name: &str| -> Result { read_property(value, name) };
+
+ let invert = read("invert")?
+ .long()
+ .ok_or_else(|| ext_exception("flow_php expected DateInterval::invert to be an int"))?;
+ out.push(invert as u8);
+
+ for field in ["y", "m", "d", "h", "i", "s"] {
+ let field_value = read(field)?
+ .long()
+ .ok_or_else(|| ext_exception("flow_php expected DateInterval fields to be ints"))?;
+ write_u32(out, field_value as u32);
+ }
+
+ let fraction = read("f")?;
+ let fraction = php_float(&fraction)
+ .ok_or_else(|| ext_exception("flow_php expected DateInterval::f to be a float"))?;
+ out.extend_from_slice(&fraction.to_le_bytes());
+
+ Ok(())
+}
+
+fn encode_json(value: &ZendObject, out: &mut Vec, ctx: &mut Ctx) -> Result<(), PhpException> {
+ let (_, value_slot, is_object_slot) = ctx.json()?;
+
+ write_len_prefixed(
+ out,
+ read_slot(value, value_slot)
+ .zend_str()
+ .ok_or_else(|| ext_exception("flow_php expected Json to hold a string"))?
+ .as_bytes(),
+ );
+ out.push(u8::from(
+ read_slot(value, is_object_slot)
+ .bool()
+ .ok_or_else(|| ext_exception("flow_php expected Json::isObject to be a bool"))?,
+ ));
+
+ Ok(())
+}
+
+/// Markup serialization delegated to the pure-PHP `ValueEncoder` public
+/// statics - the exact code its closures run.
+fn encode_via_static(
+ value: &Zval,
+ out: &mut Vec,
+ ctx: &mut Ctx,
+ method: &'static str,
+) -> Result<(), PhpException> {
+ let handle = ctx.value_encoder_static(method)?;
+ let string = call_handle(
+ handle,
+ None,
+ &mut [value.shallow_clone()],
+ "serialize a markup value",
+ )?;
+
+ write_len_prefixed(
+ out,
+ string
+ .zend_str()
+ .ok_or_else(|| {
+ ext_exception(format!(
+ "flow_php expected ValueEncoder::{method} to return a string"
+ ))
+ })?
+ .as_bytes(),
+ );
+
+ Ok(())
+}
diff --git a/src/extension/flow-php-ext/src/exception.rs b/src/extension/flow-php-ext/src/exception.rs
new file mode 100644
index 0000000000..29bbb12c3c
--- /dev/null
+++ b/src/extension/flow-php-ext/src/exception.rs
@@ -0,0 +1,27 @@
+use ext_php_rs::builders::ClassBuilder;
+use ext_php_rs::error::Result;
+use ext_php_rs::exception::PhpException;
+use ext_php_rs::flags::ClassFlags;
+use ext_php_rs::zend::{ce, ClassEntry};
+
+const EXCEPTION_CLASS: &str = "Flow\\Floe\\Exception\\ExtensionException";
+
+fn noop(_ce: &'static mut ClassEntry) {}
+
+pub fn register() -> Result<()> {
+ ClassBuilder::new(EXCEPTION_CLASS)
+ .extends((ce::exception, "Exception"))
+ .flags(ClassFlags::Final)
+ .registration(noop)
+ .register()?;
+
+ Ok(())
+}
+
+fn exception_ce() -> &'static ClassEntry {
+ ClassEntry::try_find(EXCEPTION_CLASS).unwrap_or_else(|| ce::exception())
+}
+
+pub fn ext_exception(message: impl Into) -> PhpException {
+ PhpException::new(message.into(), 0, exception_ce())
+}
diff --git a/src/extension/flow-php-ext/src/format.rs b/src/extension/flow-php-ext/src/format.rs
new file mode 100644
index 0000000000..2ea9a57c7d
--- /dev/null
+++ b/src/extension/flow-php-ext/src/format.rs
@@ -0,0 +1,103 @@
+//! Floe frame-body wire constants and a bounds-checked byte reader. The canonical
+//! definitions live in the pure-PHP `Flow\Floe\Format`; mirror any change in the same PR.
+
+use ext_php_rs::exception::PhpException;
+
+use crate::exception::ext_exception;
+
+// Floe is little-endian by construction; a big-endian host could never match its bytes.
+#[cfg(target_endian = "big")]
+compile_error!("flow_php only supports little-endian targets");
+
+pub const FRAME_ROW: u8 = 0x02;
+
+pub const VALUE_NULL: u8 = 0x00;
+pub const VALUE_PRESENT: u8 = 0x01;
+pub const VALUE_NULL_FROM_NULL: u8 = 0x02;
+pub const VALUE_ABSENT: u8 = 0x03;
+
+pub const DATETIME_IMMUTABLE: u8 = 0x00;
+pub const DATETIME_MUTABLE: u8 = 0x01;
+
+pub const TAG_NULL: u8 = 0x00;
+pub const TAG_INTEGER: u8 = 0x01;
+pub const TAG_FLOAT: u8 = 0x02;
+pub const TAG_BOOLEAN: u8 = 0x03;
+pub const TAG_STRING: u8 = 0x04;
+pub const TAG_ARRAY: u8 = 0x05;
+pub const TAG_DATETIME: u8 = 0x06;
+pub const TAG_UUID: u8 = 0x07;
+pub const TAG_JSON: u8 = 0x08;
+
+pub const KEY_INTEGER: u8 = 0x00;
+pub const KEY_STRING: u8 = 0x01;
+
+pub struct Reader<'a> {
+ data: &'a [u8],
+ pos: usize,
+}
+
+impl<'a> Reader<'a> {
+ pub fn new(data: &'a [u8]) -> Self {
+ Self { data, pos: 0 }
+ }
+
+ pub fn is_eof(&self) -> bool {
+ self.pos >= self.data.len()
+ }
+
+ fn take(&mut self, len: usize, context: &str) -> Result<&'a [u8], PhpException> {
+ let end = self
+ .pos
+ .checked_add(len)
+ .filter(|end| *end <= self.data.len());
+
+ match end {
+ Some(end) => {
+ let slice = &self.data[self.pos..end];
+ self.pos = end;
+ Ok(slice)
+ }
+ None => Err(ext_exception(format!(
+ "flow_php frame body is truncated, {context} is incomplete"
+ ))),
+ }
+ }
+
+ pub fn u8(&mut self, context: &str) -> Result {
+ Ok(self.take(1, context)?[0])
+ }
+
+ pub fn u32(&mut self, context: &str) -> Result {
+ let bytes = self.take(4, context)?;
+
+ Ok(u32::from_le_bytes(bytes.try_into().expect("4-byte slice")))
+ }
+
+ pub fn i64(&mut self, context: &str) -> Result {
+ let bytes = self.take(8, context)?;
+
+ Ok(i64::from_le_bytes(bytes.try_into().expect("8-byte slice")))
+ }
+
+ pub fn f64(&mut self, context: &str) -> Result {
+ let bytes = self.take(8, context)?;
+
+ Ok(f64::from_le_bytes(bytes.try_into().expect("8-byte slice")))
+ }
+
+ pub fn bytes(&mut self, len: usize, context: &str) -> Result<&'a [u8], PhpException> {
+ self.take(len, context)
+ }
+}
+
+pub fn write_u32(out: &mut Vec, value: u32) {
+ out.extend_from_slice(&value.to_le_bytes());
+}
+
+/// Writes a whole `type:u8 len:u32 body` Floe frame.
+pub fn write_frame(out: &mut Vec, frame_type: u8, body: &[u8]) {
+ out.push(frame_type);
+ write_u32(out, body.len() as u32);
+ out.extend_from_slice(body);
+}
diff --git a/src/extension/flow-php-ext/src/hydrate.rs b/src/extension/flow-php-ext/src/hydrate.rs
new file mode 100644
index 0000000000..20e5104355
--- /dev/null
+++ b/src/extension/flow-php-ext/src/hydrate.rs
@@ -0,0 +1,109 @@
+//! Object-graph assembly mirroring `Flow\Floe\RowHydrator` and `EntryInstantiator`:
+//! constructor-less instantiation with direct property-slot writes.
+
+use ext_php_rs::boxed::ZBox;
+use ext_php_rs::convert::IntoZval;
+use ext_php_rs::exception::PhpException;
+use ext_php_rs::types::{ZendHashTable, ZendObject, Zval};
+
+use crate::ctx::{clone_object, construct_object, ht_add, write_slot, Ctx};
+use crate::exception::ext_exception;
+use crate::format::{Reader, VALUE_ABSENT, VALUE_NULL, VALUE_NULL_FROM_NULL, VALUE_PRESENT};
+use crate::plan::Plan;
+use crate::values::decode_value;
+
+/// Decodes one ROW frame body into a `Flow\ETL\Row` object.
+pub fn hydrate_row(
+ plan: &Plan,
+ reader: &mut Reader,
+ ctx: &mut Ctx,
+) -> Result, PhpException> {
+ let mut entries_ht = ZendHashTable::with_capacity(plan.columns.len() as u32);
+
+ for column in &plan.columns {
+ let flag = reader.u8("row value flag")?;
+
+ if flag == VALUE_ABSENT {
+ continue;
+ }
+
+ let (value, definition_source) = match flag {
+ VALUE_PRESENT => (
+ decode_value(&column.decoder, reader, ctx)?,
+ &column.definition,
+ ),
+ VALUE_NULL => (Zval::new(), &column.nullable_definition),
+ VALUE_NULL_FROM_NULL => (Zval::new(), &column.from_null_definition),
+ other => {
+ return Err(ext_exception(format!(
+ "flow_php found unknown value flag 0x{other:02X}"
+ )));
+ }
+ };
+
+ let definition_zv = clone_object(definition_source, "a column definition")?
+ .into_zval(false)
+ .map_err(|e| {
+ ext_exception(format!("flow_php failed to hydrate a definition: {e:?}"))
+ })?;
+
+ let mut entry = ZendObject::new(column.entry_ce);
+ write_slot(&mut entry, column.name_slot, column.name_zv.shallow_clone());
+ write_slot(&mut entry, column.value_slot, value);
+ write_slot(&mut entry, column.definition_slot, definition_zv);
+
+ let entry_zv = entry
+ .into_zval(false)
+ .map_err(|e| ext_exception(format!("flow_php failed to collect row entries: {e:?}")))?;
+
+ // `new Entries(...)` rejects duplicated names; a name-keyed hash would
+ // silently collapse them instead, so duplicates must fail loudly here.
+ if !ht_add(&mut entries_ht, column.name.as_bytes(), entry_zv) {
+ return Err(ext_exception(format!(
+ "flow_php found duplicated entry name \"{}\" in a row frame",
+ column.name
+ )));
+ }
+ }
+
+ let mut entries = ZendObject::new(ctx.entries_ce);
+ let mut entries_ht_zv = Zval::new();
+ entries_ht_zv.set_hashtable(entries_ht);
+ write_slot(&mut entries, ctx.entries_entries_slot, entries_ht_zv);
+
+ let mut row = ZendObject::new(ctx.row_ce);
+ let entries_zv = entries
+ .into_zval(false)
+ .map_err(|e| ext_exception(format!("flow_php failed to hydrate a row: {e:?}")))?;
+ write_slot(&mut row, ctx.row_entries_slot, entries_zv);
+
+ Ok(row)
+}
+
+/// Builds a `Flow\ETL\Rows` with empty `Partitions`, matching `new Rows(...$rows)`.
+pub fn build_rows(rows: Vec>, ctx: &mut Ctx) -> Result {
+ let mut rows_ht = ZendHashTable::with_capacity(rows.len() as u32);
+
+ for row in rows {
+ rows_ht
+ .push(row)
+ .map_err(|e| ext_exception(format!("flow_php failed to collect rows: {e:?}")))?;
+ }
+
+ let partitions_zv =
+ construct_object(ctx.partitions_ce, vec![], "Flow\\Filesystem\\Partitions")?
+ .into_zval(false)
+ .map_err(|e| ext_exception(format!("flow_php failed to hydrate Rows: {e:?}")))?;
+
+ let mut rows_ht_zv = Zval::new();
+ rows_ht_zv.set_hashtable(rows_ht);
+
+ let mut rows_obj = ZendObject::new(ctx.rows_ce);
+ write_slot(&mut rows_obj, ctx.rows_rows_slot, rows_ht_zv);
+ write_slot(&mut rows_obj, ctx.rows_partitions_slot, partitions_zv);
+
+ let mut zv = Zval::new();
+ zv.set_object(&mut rows_obj);
+
+ Ok(zv)
+}
diff --git a/src/extension/flow-php-ext/src/lib.rs b/src/extension/flow-php-ext/src/lib.rs
new file mode 100644
index 0000000000..886a451d9d
--- /dev/null
+++ b/src/extension/flow-php-ext/src/lib.rs
@@ -0,0 +1,276 @@
+mod ctx;
+mod encode;
+mod exception;
+mod format;
+mod hydrate;
+mod plan;
+mod section;
+mod values;
+
+use std::alloc::System;
+
+use ext_php_rs::binary_slice::BinarySlice;
+use ext_php_rs::convert::IntoZval;
+use ext_php_rs::prelude::*;
+use ext_php_rs::types::{ZendObject, Zval};
+use ext_php_rs::zend::ModuleEntry;
+use ext_php_rs::{info_table_end, info_table_row, info_table_start};
+
+use crate::ctx::{call_handle_on, zval_long, zval_null, zval_str, Ctx};
+use crate::encode::{encode_row_body, expect_object, ht_for_each, read_slot, EncodePlan};
+use crate::exception::ext_exception;
+use crate::format::{write_frame, Reader, FRAME_ROW};
+use crate::plan::Plan;
+use crate::section::SectionTracker;
+
+#[global_allocator]
+static GLOBAL: System = System;
+
+pub extern "C" fn php_module_info(_module: *mut ModuleEntry) {
+ info_table_start!();
+ info_table_row!("flow_php.enabled", "true");
+ info_table_row!("flow_php.extension_version", env!("FLOW_PHP_EXT_VERSION"));
+ info_table_end!();
+}
+
+/// # Safety
+///
+/// Only called by the PHP engine during module startup (MINIT).
+pub unsafe extern "C" fn module_startup(_type: i32, _module_number: i32) -> i32 {
+ if let Err(e) = exception::register() {
+ eprintln!("flow_php: failed to register Flow\\Floe\\Exception\\ExtensionException: {e}");
+ return -1;
+ }
+ 0
+}
+
+/// Stateful frame decoder for streaming Floe reads (`FloeReader` fast path):
+/// the PHP side keeps buffering/framing and hands over bare frame bodies.
+#[php_class]
+#[php(name = "Flow\\Floe\\RowsDecoder")]
+pub struct RowsDecoder {
+ ctx: Ctx,
+ plan: Option,
+}
+
+#[php_impl]
+impl RowsDecoder {
+ pub fn __construct() -> PhpResult {
+ Ok(Self {
+ ctx: Ctx::new()?,
+ plan: None,
+ })
+ }
+
+ /// Replaces the current column plan with the given SCHEMA frame body.
+ pub fn schema(&mut self, frame_body: BinarySlice) -> PhpResult<()> {
+ self.plan = Some(plan::build_plan(&frame_body, &mut self.ctx)?);
+
+ Ok(())
+ }
+
+ /// Decodes one ROW frame body against the current plan into a `Flow\ETL\Row`.
+ pub fn row(&mut self, frame_body: BinarySlice) -> PhpResult {
+ let Some(plan) = self.plan.as_ref() else {
+ return Err(ext_exception(
+ "flow_php found a row frame before any schema frame",
+ ));
+ };
+
+ let mut reader = Reader::new(&frame_body);
+ let row = hydrate::hydrate_row(plan, &mut reader, &mut self.ctx)?;
+
+ if !reader.is_eof() {
+ return Err(ext_exception(
+ "flow_php row frame length does not match its content",
+ ));
+ }
+
+ row.into_zval(false)
+ .map_err(|e| ext_exception(format!("flow_php failed to return a Row: {e:?}")))
+ }
+
+ /// Decodes a list of ROW frame bodies against the current plan into one
+ /// `Flow\ETL\Rows`, graph-identical to calling `row()` per body.
+ pub fn rows(&mut self, frame_bodies: &Zval) -> PhpResult {
+ let bodies_ht = frame_bodies
+ .array()
+ .ok_or_else(|| ext_exception("flow_php expected a list of row frame bodies"))?;
+
+ let Some(plan) = self.plan.as_ref() else {
+ return Err(ext_exception(
+ "flow_php found a row frame before any schema frame",
+ ));
+ };
+ let ctx = &mut self.ctx;
+
+ let mut rows = Vec::with_capacity(bodies_ht.len());
+
+ ht_for_each(bodies_ht, |_, _, body_zv| {
+ let bytes = body_zv
+ .zend_str()
+ .ok_or_else(|| ext_exception("flow_php expected a row frame body to be a string"))?
+ .as_bytes();
+
+ let mut reader = Reader::new(bytes);
+ let row = hydrate::hydrate_row(plan, &mut reader, ctx)?;
+
+ if !reader.is_eof() {
+ return Err(ext_exception(
+ "flow_php row frame length does not match its content",
+ ));
+ }
+
+ rows.push(row);
+
+ Ok(())
+ })?;
+
+ hydrate::build_rows(rows, ctx)
+ }
+}
+
+/// Stateful frame encoder for streaming Floe writes (`FloeWriter` fast path).
+/// The `rows()` segment state is independent from the `schema()`/`row()`
+/// per-frame state; the two modes must not be mixed on one instance.
+#[php_class]
+#[php(name = "Flow\\Floe\\RowsEncoder")]
+pub struct RowsEncoder {
+ ctx: Ctx,
+ plan: Option,
+ stream: SectionTracker,
+}
+
+#[php_impl]
+impl RowsEncoder {
+ pub fn __construct() -> PhpResult {
+ Ok(Self {
+ ctx: Ctx::new()?,
+ plan: None,
+ stream: SectionTracker::new(),
+ })
+ }
+
+ /// Primes the encode plan from a SCHEMA frame body.
+ pub fn schema(&mut self, frame_body: BinarySlice) -> PhpResult<()> {
+ self.plan = Some(encode::build_encode_plan(&frame_body)?);
+
+ Ok(())
+ }
+
+ /// Encodes one `Flow\ETL\Row` into a bare ROW frame body against the plan.
+ pub fn row(&mut self, row: &Zval) -> PhpResult {
+ let Some(plan) = self.plan.as_mut() else {
+ return Err(ext_exception(
+ "flow_php found a row before any schema frame",
+ ));
+ };
+
+ Ok(zval_str(&encode::encode_row_body(
+ plan,
+ row,
+ &mut self.ctx,
+ )?))
+ }
+
+ /// Encodes a whole `Flow\ETL\Rows` into a list of `Flow\Floe\FrameSegment`
+ /// in one call, mirroring `Flow\Floe\PhpRowFrameEncoder`.
+ pub fn rows(&mut self, rows: &Zval) -> PhpResult {
+ struct Segment {
+ schema_body: Option>,
+ frames: Vec,
+ row_count: i64,
+ }
+
+ let rows_obj = expect_object(rows, "Rows")?;
+ let rows_ht = read_slot(rows_obj, self.ctx.rows_rows_slot)
+ .array()
+ .ok_or_else(|| ext_exception("flow_php expected Rows to hold an array"))?;
+
+ let ctx = &mut self.ctx;
+ let stream = &mut self.stream;
+
+ let mut segments: Vec = Vec::new();
+ let mut current: Option = None;
+
+ ht_for_each(rows_ht, |_, _, row_zv| {
+ let entries_ht = SectionTracker::row_entries(row_zv, ctx)?;
+
+ if let Some(schema_body) = stream.ensure_section(row_zv, entries_ht, ctx)? {
+ if let Some(segment) = current.take() {
+ segments.push(segment);
+ }
+
+ current = Some(Segment {
+ schema_body: Some(schema_body),
+ frames: Vec::new(),
+ row_count: 0,
+ });
+ } else if current.is_none() {
+ current = Some(Segment {
+ schema_body: None,
+ frames: Vec::new(),
+ row_count: 0,
+ });
+ }
+
+ let body = encode_row_body(stream.plan_mut()?, row_zv, ctx)?;
+
+ let segment = current
+ .as_mut()
+ .ok_or_else(|| ext_exception("flow_php has no active segment"))?;
+ write_frame(&mut segment.frames, FRAME_ROW, &body);
+ segment.row_count += 1;
+
+ Ok(())
+ })?;
+
+ if let Some(segment) = current.take() {
+ segments.push(segment);
+ }
+
+ let (segment_ce, segment_ctor) = ctx.frame_segment()?;
+ let mut out = ext_php_rs::types::ZendHashTable::with_capacity(segments.len() as u32);
+
+ for segment in segments {
+ let schema_body_zv = match &segment.schema_body {
+ Some(body) => zval_str(body),
+ None => zval_null(),
+ };
+
+ let obj = ZendObject::new(segment_ce);
+ call_handle_on(
+ segment_ctor,
+ &obj,
+ &mut [
+ schema_body_zv,
+ zval_str(&segment.frames),
+ zval_long(segment.row_count),
+ ],
+ "construct a FrameSegment",
+ )?;
+
+ let obj_zv = obj.into_zval(false).map_err(|e| {
+ ext_exception(format!("flow_php failed to return a FrameSegment: {e:?}"))
+ })?;
+
+ out.push(obj_zv).map_err(|e| {
+ ext_exception(format!("flow_php failed to build a segment list: {e:?}"))
+ })?;
+ }
+
+ let mut zv = Zval::new();
+ zv.set_hashtable(out);
+
+ Ok(zv)
+ }
+}
+
+#[php_module]
+#[php(startup = "module_startup")]
+pub fn get_module(module: ModuleBuilder) -> ModuleBuilder {
+ module
+ .info_function(php_module_info)
+ .class::()
+ .class::()
+}
diff --git a/src/extension/flow-php-ext/src/plan.rs b/src/extension/flow-php-ext/src/plan.rs
new file mode 100644
index 0000000000..ce180034b5
--- /dev/null
+++ b/src/extension/flow-php-ext/src/plan.rs
@@ -0,0 +1,446 @@
+//! Decode plan built from a SCHEMA frame body via the canonical PHP `SchemaDecoder`.
+
+use std::fmt;
+
+use ext_php_rs::convert::IntoZvalDyn;
+use ext_php_rs::exception::PhpException;
+use ext_php_rs::types::Zval;
+use ext_php_rs::zend::ClassEntry;
+use serde::de::{MapAccess, Visitor};
+use serde::{Deserialize, Deserializer};
+
+use crate::ctx::{ensure_no_pending_exception, find_class, property_offset, read_property, Ctx};
+use crate::exception::ext_exception;
+
+/// Mirror of `SchemaDecoder::ENTRY_CLASSES`; dual-engine tests break loudly
+/// when either side changes.
+const DEFINITION_TO_ENTRY: [(&str, &str); 17] = [
+ (
+ "Flow\\ETL\\Schema\\Definition\\BooleanDefinition",
+ "Flow\\ETL\\Row\\Entry\\BooleanEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\DateDefinition",
+ "Flow\\ETL\\Row\\Entry\\DateEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\DateTimeDefinition",
+ "Flow\\ETL\\Row\\Entry\\DateTimeEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\EnumDefinition",
+ "Flow\\ETL\\Row\\Entry\\EnumEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\FloatDefinition",
+ "Flow\\ETL\\Row\\Entry\\FloatEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\HTMLDefinition",
+ "Flow\\ETL\\Row\\Entry\\HTMLEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\HTMLElementDefinition",
+ "Flow\\ETL\\Row\\Entry\\HTMLElementEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\IntegerDefinition",
+ "Flow\\ETL\\Row\\Entry\\IntegerEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\JsonDefinition",
+ "Flow\\ETL\\Row\\Entry\\JsonEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\ListDefinition",
+ "Flow\\ETL\\Row\\Entry\\ListEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\MapDefinition",
+ "Flow\\ETL\\Row\\Entry\\MapEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\StringDefinition",
+ "Flow\\ETL\\Row\\Entry\\StringEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\StructureDefinition",
+ "Flow\\ETL\\Row\\Entry\\StructureEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\TimeDefinition",
+ "Flow\\ETL\\Row\\Entry\\TimeEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\UuidDefinition",
+ "Flow\\ETL\\Row\\Entry\\UuidEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\XMLDefinition",
+ "Flow\\ETL\\Row\\Entry\\XMLEntry",
+ ),
+ (
+ "Flow\\ETL\\Schema\\Definition\\XMLElementDefinition",
+ "Flow\\ETL\\Row\\Entry\\XMLElementEntry",
+ ),
+];
+
+pub enum MapKey {
+ Integer,
+ String,
+ Dynamic,
+}
+
+/// Recursive value decoder mirroring `ValueDecoder::decoderFor` dispatch.
+pub enum Decoder {
+ Integer,
+ Float,
+ Boolean,
+ String,
+ Null,
+ Dynamic,
+ DateTime,
+ Interval,
+ Uuid,
+ Json,
+ TimeZone,
+ Enum,
+ Xml,
+ XmlElement,
+ Html,
+ HtmlElement,
+ List(Box),
+ Map(MapKey, Box),
+ Structure(Vec<(Vec, Decoder)>, bool),
+ Optional(Box),
+}
+
+pub struct Column {
+ pub name: String,
+ pub name_zv: Zval,
+ pub entry_ce: &'static ClassEntry,
+ pub name_slot: u32,
+ pub value_slot: u32,
+ pub definition_slot: u32,
+ pub definition: Zval,
+ pub nullable_definition: Zval,
+ pub from_null_definition: Zval,
+ pub decoder: Decoder,
+}
+
+pub struct Plan {
+ pub columns: Vec,
+}
+
+#[derive(Deserialize, Default)]
+pub struct TypeJson {
+ #[serde(rename = "type")]
+ pub type_: String,
+ element: Option>,
+ key: Option>,
+ value: Option>,
+ base: Option>,
+ #[serde(default)]
+ elements: OrderedTypes,
+ #[serde(default)]
+ optional_elements: OrderedTypes,
+ #[serde(default)]
+ allow_extra: bool,
+}
+
+impl TypeJson {
+ pub fn element(&self) -> Option<&TypeJson> {
+ self.element.as_deref()
+ }
+
+ pub fn key(&self) -> Option<&TypeJson> {
+ self.key.as_deref()
+ }
+
+ pub fn value(&self) -> Option<&TypeJson> {
+ self.value.as_deref()
+ }
+
+ pub fn base(&self) -> Option<&TypeJson> {
+ self.base.as_deref()
+ }
+
+ pub fn all_elements(&self) -> impl Iterator- {
+ self.elements
+ .0
+ .iter()
+ .chain(&self.optional_elements.0)
+ .map(|(name, element)| (name, element))
+ }
+
+ pub fn allow_extra(&self) -> bool {
+ self.allow_extra
+ }
+}
+
+#[derive(Deserialize)]
+pub struct NormalizedDefinition {
+ #[serde(rename = "ref")]
+ pub name: String,
+ #[serde(rename = "type")]
+ pub type_: TypeJson,
+}
+
+pub fn parse_schema_json(schema_json: &[u8]) -> Result, PhpException> {
+ let json = std::str::from_utf8(schema_json)
+ .map_err(|_| ext_exception("flow_php failed to decode schema JSON: invalid UTF-8"))?;
+
+ serde_json::from_str(json)
+ .map_err(|e| ext_exception(format!("flow_php failed to decode schema JSON: {e}")))
+}
+
+#[derive(Default)]
+struct OrderedTypes(Vec<(String, TypeJson)>);
+
+impl<'de> Deserialize<'de> for OrderedTypes {
+ fn deserialize>(deserializer: D) -> Result {
+ struct OrderedTypesVisitor;
+
+ impl<'de> Visitor<'de> for OrderedTypesVisitor {
+ type Value = OrderedTypes;
+
+ fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+ formatter.write_str("a map of element names to types")
+ }
+
+ fn visit_map>(self, mut access: A) -> Result {
+ let mut entries = Vec::with_capacity(access.size_hint().unwrap_or(0));
+
+ while let Some((key, value)) = access.next_entry::()? {
+ entries.push((key, value));
+ }
+
+ Ok(OrderedTypes(entries))
+ }
+
+ fn visit_seq>(
+ self,
+ _: A,
+ ) -> Result {
+ // an empty PHP array json_encodes as [] instead of {}
+ Ok(OrderedTypes(Vec::new()))
+ }
+ }
+
+ deserializer.deserialize_any(OrderedTypesVisitor)
+ }
+}
+
+fn build_decoder(type_json: &TypeJson) -> Result {
+ let missing = |part: &str| {
+ ext_exception(format!(
+ "flow_php schema JSON for type \"{}\" is missing its {part}",
+ type_json.type_
+ ))
+ };
+
+ Ok(match type_json.type_.as_str() {
+ "integer" | "positive_integer" => Decoder::Integer,
+ "float" => Decoder::Float,
+ "boolean" => Decoder::Boolean,
+ "string" | "non_empty_string" | "numeric-string" | "class_string" => Decoder::String,
+ "null" => Decoder::Null,
+ "mixed" | "union" | "scalar" | "literal" | "array" => Decoder::Dynamic,
+ "datetime" | "date" => Decoder::DateTime,
+ "time" => Decoder::Interval,
+ "uuid" => Decoder::Uuid,
+ "json" => Decoder::Json,
+ "timezone" => Decoder::TimeZone,
+ "enum" => Decoder::Enum,
+ "xml" => Decoder::Xml,
+ "xml_element" => Decoder::XmlElement,
+ "html" => Decoder::Html,
+ "html_element" => Decoder::HtmlElement,
+ "list" => Decoder::List(Box::new(build_decoder(
+ type_json
+ .element
+ .as_ref()
+ .ok_or_else(|| missing("element"))?,
+ )?)),
+ "map" => {
+ let key = match type_json
+ .key
+ .as_ref()
+ .ok_or_else(|| missing("key"))?
+ .type_
+ .as_str()
+ {
+ "integer" => MapKey::Integer,
+ "string" => MapKey::String,
+ _ => MapKey::Dynamic,
+ };
+
+ Decoder::Map(
+ key,
+ Box::new(build_decoder(
+ type_json.value.as_ref().ok_or_else(|| missing("value"))?,
+ )?),
+ )
+ }
+ "structure" => {
+ let mut elements = Vec::with_capacity(
+ type_json.elements.0.len() + type_json.optional_elements.0.len(),
+ );
+
+ for (name, element) in type_json
+ .elements
+ .0
+ .iter()
+ .chain(&type_json.optional_elements.0)
+ {
+ elements.push((name.clone().into_bytes(), build_decoder(element)?));
+ }
+
+ Decoder::Structure(elements, type_json.allow_extra)
+ }
+ "optional" => Decoder::Optional(Box::new(build_decoder(
+ type_json.base.as_ref().ok_or_else(|| missing("base"))?,
+ )?)),
+ other => {
+ return Err(ext_exception(format!(
+ "flow_php does not support values of type \"{other}\" in this build"
+ )));
+ }
+ })
+}
+
+fn definition_class_for_type(type_str: &str) -> Option<&'static str> {
+ Some(match type_str {
+ "integer" | "positive_integer" => "Flow\\ETL\\Schema\\Definition\\IntegerDefinition",
+ "float" => "Flow\\ETL\\Schema\\Definition\\FloatDefinition",
+ "boolean" => "Flow\\ETL\\Schema\\Definition\\BooleanDefinition",
+ "string" | "non_empty_string" | "numeric-string" | "class_string" => {
+ "Flow\\ETL\\Schema\\Definition\\StringDefinition"
+ }
+ "date" => "Flow\\ETL\\Schema\\Definition\\DateDefinition",
+ "datetime" => "Flow\\ETL\\Schema\\Definition\\DateTimeDefinition",
+ "time" => "Flow\\ETL\\Schema\\Definition\\TimeDefinition",
+ "json" | "array" => "Flow\\ETL\\Schema\\Definition\\JsonDefinition",
+ "uuid" => "Flow\\ETL\\Schema\\Definition\\UuidDefinition",
+ "list" => "Flow\\ETL\\Schema\\Definition\\ListDefinition",
+ "map" => "Flow\\ETL\\Schema\\Definition\\MapDefinition",
+ "structure" => "Flow\\ETL\\Schema\\Definition\\StructureDefinition",
+ "enum" => "Flow\\ETL\\Schema\\Definition\\EnumDefinition",
+ "html" => "Flow\\ETL\\Schema\\Definition\\HTMLDefinition",
+ "html_element" => "Flow\\ETL\\Schema\\Definition\\HTMLElementDefinition",
+ "xml" => "Flow\\ETL\\Schema\\Definition\\XMLDefinition",
+ "xml_element" => "Flow\\ETL\\Schema\\Definition\\XMLElementDefinition",
+ _ => return None,
+ })
+}
+
+pub fn entry_class_for_type(type_json: &TypeJson) -> Result<&'static ClassEntry, PhpException> {
+ let definition_class =
+ definition_class_for_type(type_json.type_.as_str()).ok_or_else(|| {
+ ext_exception(format!(
+ "flow_php cannot encode a column of type \"{}\"",
+ type_json.type_
+ ))
+ })?;
+
+ let entry_class = DEFINITION_TO_ENTRY
+ .iter()
+ .find(|(definition_name, _)| *definition_name == definition_class)
+ .map(|(_, entry_class)| *entry_class)
+ .ok_or_else(|| {
+ ext_exception(format!(
+ "flow_php cannot encode entries for definition \"{definition_class}\""
+ ))
+ })?;
+
+ find_class(entry_class)
+}
+
+fn entry_class_for(definition: &Zval) -> Result<&'static ClassEntry, PhpException> {
+ let definition_class = definition
+ .object()
+ .and_then(|obj| unsafe { obj.ce.as_ref() })
+ .and_then(ClassEntry::name)
+ .ok_or_else(|| ext_exception("flow_php failed to resolve a definition class"))?;
+
+ let entry_class = DEFINITION_TO_ENTRY
+ .iter()
+ .find(|(definition_name, _)| *definition_name == definition_class)
+ .map(|(_, entry_class)| *entry_class)
+ .ok_or_else(|| {
+ ext_exception(format!(
+ "flow_php cannot hydrate entries for definition \"{definition_class}\""
+ ))
+ })?;
+
+ find_class(entry_class)
+}
+
+pub fn build_plan(schema_json: &[u8], ctx: &mut Ctx) -> Result {
+ let definitions = parse_schema_json(schema_json)?;
+
+ let json_string = std::str::from_utf8(schema_json)
+ .expect("validated by parse_schema_json")
+ .to_string();
+ let hydrator_columns = ctx
+ .schema_decoder()?
+ .try_call_method("decode", vec![&json_string as &dyn IntoZvalDyn])
+ .map_err(|e| ext_exception(format!("flow_php failed to decode a schema frame: {e:?}")))?;
+ ensure_no_pending_exception("decode a schema frame")?;
+
+ let hydrator_columns = hydrator_columns.array().ok_or_else(|| {
+ ext_exception("flow_php expected SchemaDecoder::decode to return an array")
+ })?;
+
+ if hydrator_columns.len() != definitions.len() {
+ return Err(ext_exception(
+ "flow_php schema JSON and SchemaDecoder disagree on the column count",
+ ));
+ }
+
+ let mut columns = Vec::with_capacity(definitions.len());
+
+ for (index, definition) in definitions.iter().enumerate() {
+ let hydrator_column = hydrator_columns
+ .get_index(index as i64)
+ .and_then(Zval::object)
+ .ok_or_else(|| ext_exception("flow_php expected a HydratorColumn object"))?;
+
+ let name_zv = read_property(hydrator_column, "name")?;
+ let name = name_zv
+ .zend_str()
+ .and_then(|s| std::str::from_utf8(s.as_bytes()).ok())
+ .ok_or_else(|| ext_exception("flow_php expected column names to be UTF-8 strings"))?
+ .to_string();
+
+ let read_definition = |property: &str| -> Result {
+ let zv = read_property(hydrator_column, property)?;
+
+ if !zv.is_object() {
+ return Err(ext_exception(
+ "flow_php expected HydratorColumn definitions to be objects",
+ ));
+ }
+
+ Ok(zv)
+ };
+
+ let definition_zv = read_definition("definition")?;
+ let entry_ce = entry_class_for(&definition_zv)?;
+
+ columns.push(Column {
+ name,
+ name_zv,
+ entry_ce,
+ name_slot: property_offset(entry_ce, "name")?,
+ value_slot: property_offset(entry_ce, "value")?,
+ definition_slot: property_offset(entry_ce, "definition")?,
+ definition: definition_zv,
+ nullable_definition: read_definition("nullableDefinition")?,
+ from_null_definition: read_definition("fromNullDefinition")?,
+ decoder: build_decoder(&definition.type_)?,
+ });
+ }
+
+ Ok(Plan { columns })
+}
diff --git a/src/extension/flow-php-ext/src/section.rs b/src/extension/flow-php-ext/src/section.rs
new file mode 100644
index 0000000000..1bcbace12d
--- /dev/null
+++ b/src/extension/flow-php-ext/src/section.rs
@@ -0,0 +1,284 @@
+use std::collections::HashMap;
+
+use ext_php_rs::exception::PhpException;
+use ext_php_rs::types::{ZendHashTable, ZendObject, Zval};
+
+use crate::ctx::{
+ call_handle, call_handle_on, ce_method_ref, find_class, property_offset, zval_null, zval_str,
+ Ctx,
+};
+use crate::encode::{build_encode_plan, expect_object, ht_for_each, read_slot, EncodePlan};
+use crate::exception::ext_exception;
+
+struct Tracking {
+ columns: HashMap, Vec>,
+}
+
+const CONSTANT_NORMALIZE_TYPES: [&str; 18] = [
+ "Flow\\Types\\Type\\Native\\IntegerType",
+ "Flow\\Types\\Type\\Logical\\PositiveIntegerType",
+ "Flow\\Types\\Type\\Native\\FloatType",
+ "Flow\\Types\\Type\\Native\\BooleanType",
+ "Flow\\Types\\Type\\Native\\StringType",
+ "Flow\\Types\\Type\\Logical\\NonEmptyStringType",
+ "Flow\\Types\\Type\\Logical\\NumericStringType",
+ "Flow\\Types\\Type\\Logical\\ScalarType",
+ "Flow\\Types\\Type\\Logical\\DateTimeType",
+ "Flow\\Types\\Type\\Logical\\DateType",
+ "Flow\\Types\\Type\\Logical\\TimeType",
+ "Flow\\Types\\Type\\Logical\\HTMLType",
+ "Flow\\Types\\Type\\Logical\\HTMLElementType",
+ "Flow\\Types\\Type\\Logical\\TimeZoneType",
+ "Flow\\Types\\Type\\Logical\\UuidType",
+ "Flow\\Types\\Type\\Logical\\JsonType",
+ "Flow\\Types\\Type\\Logical\\XMLType",
+ "Flow\\Types\\Type\\Logical\\XMLElementType",
+];
+
+#[derive(Default)]
+struct FingerprintCache {
+ entry_definition_slots: HashMap,
+ definition_type_slots: HashMap,
+ fingerprints: HashMap>>,
+}
+
+/// `json_encode($type->normalize())`.
+fn compute_fingerprint(type_obj: &ZendObject, ctx: &mut Ctx) -> Result, PhpException> {
+ let type_ce = unsafe { type_obj.ce.as_ref() }
+ .ok_or_else(|| ext_exception("flow_php failed to resolve a type class"))?;
+
+ let normalized = call_handle_on(
+ ce_method_ref(type_ce, "normalize")?,
+ type_obj,
+ &mut [],
+ "normalize a type",
+ )?;
+
+ let json = call_handle(
+ ctx.json_encode()?,
+ None,
+ &mut [normalized],
+ "encode a type fingerprint",
+ )?;
+
+ Ok(json
+ .zend_str()
+ .ok_or_else(|| ext_exception("flow_php expected json_encode to return a string"))?
+ .as_bytes()
+ .to_vec())
+}
+
+fn fingerprint_matches(
+ entry: &ZendObject,
+ expected: &[u8],
+ ctx: &mut Ctx,
+ cache: &mut FingerprintCache,
+) -> Result {
+ let entry_ce_ptr = entry.ce as usize;
+ let definition_slot = match cache.entry_definition_slots.get(&entry_ce_ptr) {
+ Some(slot) => *slot,
+ None => {
+ let entry_ce = unsafe { entry.ce.as_ref() }
+ .ok_or_else(|| ext_exception("flow_php failed to resolve an entry class"))?;
+ let slot = property_offset(entry_ce, "definition")?;
+ cache.entry_definition_slots.insert(entry_ce_ptr, slot);
+ slot
+ }
+ };
+
+ let def_obj = expect_object(read_slot(entry, definition_slot), "an entry definition")?;
+ let def_ce_ptr = def_obj.ce as usize;
+ let type_slot = match cache.definition_type_slots.get(&def_ce_ptr) {
+ Some(slot) => *slot,
+ None => {
+ let def_ce = unsafe { def_obj.ce.as_ref() }
+ .ok_or_else(|| ext_exception("flow_php failed to resolve a definition class"))?;
+ let slot = property_offset(def_ce, "type")?;
+ cache.definition_type_slots.insert(def_ce_ptr, slot);
+ slot
+ }
+ };
+
+ let type_obj = expect_object(read_slot(def_obj, type_slot), "a definition type")?;
+ let type_ce_ptr = type_obj.ce as usize;
+
+ if let Some(cached) = cache.fingerprints.get(&type_ce_ptr) {
+ return match cached {
+ Some(fingerprint) => Ok(fingerprint.as_slice() == expected),
+ None => Ok(compute_fingerprint(type_obj, ctx)?.as_slice() == expected),
+ };
+ }
+
+ let type_ce = unsafe { type_obj.ce.as_ref() }
+ .ok_or_else(|| ext_exception("flow_php failed to resolve a type class"))?;
+ let constant = type_ce
+ .name()
+ .is_some_and(|name| CONSTANT_NORMALIZE_TYPES.contains(&name));
+ let fingerprint = compute_fingerprint(type_obj, ctx)?;
+ let matches = fingerprint.as_slice() == expected;
+
+ cache
+ .fingerprints
+ .insert(type_ce_ptr, constant.then_some(fingerprint));
+
+ Ok(matches)
+}
+
+/// Mirrors `SchemaTracker::fits`: a narrower row still fits; a new column or an
+/// incompatible type forces a new section.
+fn fits(
+ tracking: &Tracking,
+ entries_ht: &ZendHashTable,
+ ctx: &mut Ctx,
+ cache: &mut FingerprintCache,
+) -> Result {
+ let mut fits = true;
+
+ ht_for_each(entries_ht, |key, index, entry_zv| {
+ if !fits {
+ return Ok(());
+ }
+
+ let index_name;
+ let name: &[u8] = match key {
+ Some(name) => name.as_bytes(),
+ None => {
+ index_name = (index as i64).to_string();
+ index_name.as_bytes()
+ }
+ };
+
+ match tracking.columns.get(name) {
+ None => fits = false,
+ Some(expected) => {
+ let entry = expect_object(entry_zv, "a row entry")?;
+
+ if !fingerprint_matches(entry, expected, ctx, cache)? {
+ fits = false;
+ }
+ }
+ }
+
+ Ok(())
+ })?;
+
+ Ok(fits)
+}
+
+/// Fingerprints come straight from `EncoderColumn::typeFingerprint`, the same
+/// value `fits` compares against, so parity with `SchemaTracker::fits` holds.
+fn build_tracking(plan_obj: &ZendObject, ctx: &mut Ctx) -> Result {
+ let columns_slot = ctx.encoder_plan_slots()?.columns;
+ let (name_slot, fingerprint_slot) = {
+ let slots = ctx.encoder_column_slots()?;
+ (slots.name, slots.type_fingerprint)
+ };
+
+ let columns_ht = read_slot(plan_obj, columns_slot)
+ .array()
+ .ok_or_else(|| ext_exception("flow_php expected EncoderPlan::columns to be an array"))?;
+
+ let mut columns = HashMap::with_capacity(columns_ht.len());
+ ht_for_each(columns_ht, |_, _, column_zv| {
+ let column = expect_object(column_zv, "an EncoderColumn")?;
+
+ let name = read_slot(column, name_slot)
+ .zend_str()
+ .ok_or_else(|| ext_exception("flow_php expected EncoderColumn::name to be a string"))?
+ .as_bytes()
+ .to_vec();
+ let fingerprint = read_slot(column, fingerprint_slot)
+ .zend_str()
+ .ok_or_else(|| {
+ ext_exception("flow_php expected EncoderColumn::typeFingerprint to be a string")
+ })?
+ .as_bytes()
+ .to_vec();
+
+ columns.insert(name, fingerprint);
+
+ Ok(())
+ })?;
+
+ Ok(Tracking { columns })
+}
+
+pub struct SectionTracker {
+ tracking: Option,
+ current_schema_body: Option>,
+ plan: Option,
+ fingerprints: FingerprintCache,
+}
+
+impl SectionTracker {
+ pub fn new() -> Self {
+ Self {
+ tracking: None,
+ current_schema_body: None,
+ plan: None,
+ fingerprints: FingerprintCache::default(),
+ }
+ }
+
+ /// Reads a Row's entries hashtable through the cached slot offsets.
+ pub fn row_entries<'a>(row_zv: &'a Zval, ctx: &Ctx) -> Result<&'a ZendHashTable, PhpException> {
+ let row_obj = expect_object(row_zv, "a Row")?;
+ let entries_obj = expect_object(read_slot(row_obj, ctx.row_entries_slot), "Row entries")?;
+
+ read_slot(entries_obj, ctx.entries_entries_slot)
+ .array()
+ .ok_or_else(|| ext_exception("flow_php expected Entries to hold an array"))
+ }
+
+ /// Returns the new section's SCHEMA frame body when the row does not fit,
+ /// grown via `FloeWriter::growSectionPlan`; None when the row rides the section.
+ pub fn ensure_section(
+ &mut self,
+ row_zv: &Zval,
+ entries_ht: &ZendHashTable,
+ ctx: &mut Ctx,
+ ) -> Result