feat(flow-php/types): remove structure allow_extra

documentation/components/bridges/mago-types-bridge.md

@@ -52,6 +52,5 @@ Requires `carthage-software/mago` 1.47.4 or newer (the first release with extens
 
 - `type_structure(['a' => type_integer()])` - `array{a: int}`, every plain `Type` value required.
 - A `structure_element()` value with a literal `optional: true` - `a?:` (possibly undefined key).
-- A literal `allow_extra: true` - the shape stays open (`array-key => mixed` parameters).
-- Anything not statically known (unsealed arrays, non-literal flags, non-`Type` values) - the
+- Anything not statically known (unsealed arrays, non-`Type` values) - the
   derivation backs off to the native docblock instead of guessing.

documentation/components/core/floe.md

@@ -254,10 +254,9 @@ These throw `FloeException` when the write session opens:
 ```
 Floe does not support values of type "mixed"
 Floe does not support map keys of type "uuid"
-Floe does not support structures that allow extra values
 ```
 
-That covers `mixed` and `type_structure(..., allow_extra: true)`.
+That covers `mixed`.
 Use a declared element type, or a `json_schema()` column when the shape is genuinely dynamic.
 
 ## On-Disk Layout

documentation/upgrading.md

@@ -467,6 +467,23 @@ The same applies to map values and structure elements.
 |---------------------------------------------|-------------------------------------|
 | `'d' => [1641600000]` - seconds since epoch | `'d' => [19000]` - days since epoch |
 
+### 48) `flow-php/types` - a structure declares every element it carries
+
+| Before                                                                                                       | After                                                                            |
+|--------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------|
+| `type_structure($elements, allow_extra: true)`                                                               | `Error: Unknown named parameter $allow_extra`                                    |
+| `type_structure($elements, true)`, `StructureType::fromElements($elements, true)`                            | the extra argument is ignored by PHP - the structure is sealed                   |
+| `new StructureType($elements, true)`                                                                         | the extra argument is ignored by PHP - the structure is sealed                   |
+| `StructureType::allowsExtra()`                                                                               | removed                                                                          |
+| `normalize()` / `schema_to_json()` write `"allow_extra": false` on every structure                           | the key is not written                                                           |
+| a stored schema with `"allow_extra": false`                                                                  | read, the key is ignored                                                         |
+| a stored schema with `"allow_extra": true`                                                                   | `InvalidArgumentException: Structure "allow_extra": true is no longer supported` |
+| `Rows` / `Schema` with a structure column natively serialized by 0.44.x (`ApcuCache`, `NativePHPSerializer`) | `Error: Cannot create dynamic property StructureType::$allowExtra`               |
+
+A value with a key its structure does not declare is invalid. Declare every element, or use `type_map()` /
+`type_json()` when the keys are dynamic. flow-php 0.44.x cannot read a Floe file, cached schema or `schema_to_json()`
+document written by 0.45.x that has a structure column. Clear APCu and natively serialized caches after upgrading.
+
 ---
 
 ## Upgrading from 0.43.x to 0.44.x

src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLEncoderTest.php

@@ -213,7 +213,7 @@ public function test_encoding_structure_with_more_values_than_declared_elements_
             'city' => 'Krakow',
             'zip' => '31-021',
             'extra' => 'lost',
-        ]], ['address' => type_structure(['city' => type_string(), 'zip' => type_string()], true)])]);
+        ]], ['address' => type_structure(['city' => type_string(), 'zip' => type_string()])])]);
     }
 
     public function test_encodes_null_scalar_as_an_empty_node(): void

src/bridge/mago/types/src/Flow/Bridge/Mago/Types/TypeStructureShape.php

@@ -9,8 +9,6 @@
 use Mago\Sdk\Analyzer\Type\ArrayItem;
 use Mago\Sdk\Analyzer\Type\KeyedArrayType;
 use Mago\Sdk\Analyzer\Type\NamedObjectType;
-use Mago\Sdk\Analyzer\Type\ScalarType;
-use Mago\Sdk\Analyzer\Type\ScalarTypeKind;
 
 /**
  * Derives the array shape a `Flow\Types\DSL\type_structure()` call represents. Beyond Mago's
@@ -34,16 +32,6 @@ public function derive(Invocation $invocation): ?Type
             return null;
         }
 
-        $allowExtra = false;
-
-        if (($allowExtraArgument = $invocation->getArgument(1, 'allow_extra')) !== null) {
-            $allowExtra = $allowExtraArgument->type?->getLiteralBool();
-
-            if ($allowExtra === null) {
-                return null;
-            }
-        }
-
         $items = [];
 
         foreach ($elementsArray->knownItems as $item) {
@@ -58,8 +46,8 @@ public function derive(Invocation $invocation): ?Type
 
         $shape = Type::fromAtomic(new KeyedArrayType(
             knownItems: array_values($items),
-            keyType: $allowExtra ? Type::fromAtomic(new ScalarType(ScalarTypeKind::ArrayKey)) : null,
-            valueType: $allowExtra ? Type::mixed() : null,
+            keyType: null,
+            valueType: null,
             nonEmpty: $items !== [],
         ));
 

src/bridge/mago/types/tests/Flow/Bridge/Mago/Types/Tests/Unit/TypeStructureShapeTest.php

@@ -51,25 +51,15 @@ public function test_marker_without_flag_or_with_false_stays_required(): void
         static::assertSame([false, false, false], [$shape['a'][0], $shape['b'][0], $shape['c'][0]]);
     }
 
-    public function test_literal_allow_extra_unseals_the_shape(): void
+    public function test_derived_shape_is_sealed(): void
     {
-        $derived = (new TypeStructureShape())->derive(InvocationMother::typeStructure(
-            InvocationMother::sealedMap(InvocationMother::item('id', InvocationMother::flowType(Type::int()))),
-            Type::true(),
-        ));
-
-        $keyedArray = DerivedShapeContext::keyedArray($derived);
-
-        static::assertNotNull($keyedArray->keyType);
-        static::assertNotNull($keyedArray->valueType);
-    }
+        $keyedArray = DerivedShapeContext::keyedArray((new TypeStructureShape())->derive(InvocationMother::typeStructure(InvocationMother::sealedMap(InvocationMother::item(
+            'id',
+            InvocationMother::flowType(Type::int()),
+        )))));
 
-    public function test_non_literal_allow_extra_refuses_to_derive(): void
-    {
-        static::assertNull((new TypeStructureShape())->derive(InvocationMother::typeStructure(
-            InvocationMother::sealedMap(InvocationMother::item('id', InvocationMother::flowType(Type::int()))),
-            Type::bool(),
-        )));
+        static::assertNull($keyedArray->keyType);
+        static::assertNull($keyedArray->valueType);
     }
 
     public function test_unsealed_elements_refuse_to_derive(): void

src/cli/tests/Flow/CLI/Tests/Integration/SchemaFormatCommandTest.php

@@ -107,8 +107,7 @@ public function test_run_schema_format(): void
                                 },
                                 "optional": false
                             }
-                        ],
-                        "allow_extra": false
+                        ]
                     },
                     "nullable": false,
                     "metadata": []
@@ -152,8 +151,7 @@ public function test_run_schema_format(): void
                                     },
                                     "optional": false
                                 }
-                            ],
-                            "allow_extra": false
+                            ]
                         }
                     },
                     "nullable": false,

src/core/etl/src/Flow/ETL/Function/ArrayKeysStyleConvert.php

@@ -82,7 +82,7 @@ public function returns(): Type
             $elements[] = structure_element($converted, $element->type, $element->optional);
         }
 
-        return new StructureType($elements, $array->allowsExtra());
+        return new StructureType($elements);
     }
 
     public function eval(Row $row, FlowContext $context): mixed

src/core/etl/src/Flow/ETL/Schema/Definition/StructureDefinition.php

@@ -108,10 +108,6 @@ public function isCompatible(Definition $definition): bool
             }
         }
 
-        if ($this->type->allowsExtra()) {
-            return true;
-        }
-
         foreach ($givenByName as $given) {
             if (!array_key_exists($given->name, $declaredByName)) {
                 return false;

src/core/etl/src/Flow/ETL/Schema/Definition/TypeProjection.php

@@ -113,6 +113,6 @@ public function structure(StructureType $type): StructureType
             $elements[] = structure_element($element->name, $projected, $element->optional);
         }
 
-        return $changed ? new StructureType($elements, $type->allowsExtra()) : $type;
+        return $changed ? new StructureType($elements) : $type;
     }
 }

src/core/etl/src/Flow/ETL/Schema/Formatter/PHPFormatter/TypeFormatter.php

@@ -158,16 +158,10 @@ private function formatStructureType(StructureType $type, bool $nullable): strin
                 : sprintf('%s => %s', $name, $this->format($element->type));
         }
 
-        $arguments = sprintf('elements: [%s]', implode(', ', $fields));
-
-        if ($type->allowsExtra()) {
-            $arguments .= ', allow_extra: true';
-        }
-
         return sprintf(
             $nullable ? '\\Flow\\Types\\DSL\\type_optional(\%s(%s))' : '\%s(%s)',
             $reflection->getName(),
-            $arguments,
+            sprintf('elements: [%s]', implode(', ', $fields)),
         );
     }
 }

src/core/etl/src/Flow/ETL/Schema/Inference/TypeFloor.php

@@ -85,6 +85,6 @@ public function structure(StructureType $type): StructureType
             );
         }
 
-        return new StructureType($elements, $type->allowsExtra());
+        return new StructureType($elements);
     }
 }

src/core/etl/src/Flow/Floe/Footer.php

@@ -11,6 +11,7 @@
 use Flow\Types\Exception\InvalidTypeException;
 use JsonException;
 
+use function array_intersect_key;
 use function Flow\Types\DSL\type_array;
 use function Flow\Types\DSL\type_integer;
 use function Flow\Types\DSL\type_list;
@@ -71,7 +72,14 @@ public static function fromArray(array $data): self
                 'sections' => type_list(type_array()),
                 'statistics' => type_array(),
                 'metadata' => type_array(),
-            ], allow_extra: true)->assert($data);
+            ])->assert(array_intersect_key($data, [
+                'version' => true,
+                'writer' => true,
+                'schema' => true,
+                'sections' => true,
+                'statistics' => true,
+                'metadata' => true,
+            ]));
         } catch (InvalidTypeException $e) {
             throw new FloeException('Floe footer is malformed: ' . $e->getMessage(), 0, $e);
         }

src/core/etl/src/Flow/Floe/Section.php

@@ -7,6 +7,7 @@
 use Flow\Floe\Exception\FloeException;
 use Flow\Types\Exception\InvalidTypeException;
 
+use function array_intersect_key;
 use function Flow\Types\DSL\type_integer;
 use function Flow\Types\DSL\type_structure;
 
@@ -28,7 +29,7 @@ public static function fromArray(array $data): self
             $data = type_structure([
                 'offset' => type_integer(),
                 'rowCount' => type_integer(),
-            ], allow_extra: true)->assert($data);
+            ])->assert(array_intersect_key($data, ['offset' => true, 'rowCount' => true]));
         } catch (InvalidTypeException $e) {
             throw new FloeException('Floe footer section is malformed: ' . $e->getMessage(), 0, $e);
         }

src/core/etl/src/Flow/Floe/Statistics.php

@@ -7,6 +7,7 @@
 use Flow\Floe\Exception\FloeException;
 use Flow\Types\Exception\InvalidTypeException;
 
+use function array_intersect_key;
 use function Flow\Types\DSL\type_integer;
 use function Flow\Types\DSL\type_structure;
 
@@ -42,7 +43,7 @@ public static function fromArray(array $data): self
             $data = type_structure([
                 'rows' => type_integer(),
                 'byteSize' => type_integer(),
-            ], allow_extra: true)->assert($data);
+            ])->assert(array_intersect_key($data, ['rows' => true, 'byteSize' => true]));
         } catch (InvalidTypeException $e) {
             throw new FloeException('Floe footer statistics are malformed: ' . $e->getMessage(), 0, $e);
         }

src/core/etl/src/Flow/Floe/ValueDecoder.php

@@ -276,10 +276,6 @@ private function optionalDecoder(Type $type): Decoding\ValueDecoder
     private function structureDecoder(Type $type): Decoding\ValueDecoder
     {
         /** @var StructureType<array<array-key, mixed>> $type */
-        if ($type->allowsExtra()) {
-            throw new FloeException('Floe does not support structures that allow extra values');
-        }
-
         $elements = [];
 
         foreach ($type->elements() as $element) {

src/core/etl/src/Flow/Floe/ValueEncoder.php

@@ -254,10 +254,6 @@ private function mapEncoder(Type $type): Encoding\ValueEncoder
     private function structureEncoder(Type $type): Encoding\ValueEncoder
     {
         /** @var StructureType<array<array-key, mixed>> $type */
-        if ($type->allowsExtra()) {
-            throw new FloeException('Floe does not support structures that allow extra values');
-        }
-
         $elements = [];
 
         foreach ($type->elements() as $element) {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/StructureSchemaMatchTest.php

@@ -62,20 +62,8 @@ public static function provideMatrixCases(): Generator
             true,
         ];
 
-        yield 'allow extra, undeclared key present' => [
-            type_structure(['id' => type_integer(), 'email' => type_string()], true),
-            $withNickname,
-            true,
-        ];
-
-        yield 'allow extra, undeclared key absent' => [
-            type_structure(['id' => type_integer(), 'email' => type_string()], true),
-            $withoutNickname,
-            true,
-        ];
-
-        yield 'without allow extra, undeclared key present' => [
-            type_structure(['id' => type_integer(), 'email' => type_string()], false),
+        yield 'undeclared key present' => [
+            type_structure(['id' => type_integer(), 'email' => type_string()]),
             $withNickname,
             false,
         ];

src/core/etl/tests/Flow/ETL/Tests/Unit/Row/NativeRowHydratorTest.php

@@ -19,7 +19,6 @@
 use Flow\ETL\Tests\Double\ForeignTypeDefinition;
 use Flow\ETL\Tests\Double\ThrowingType;
 use Flow\ETL\Tests\FlowTestCase;
-use Flow\Types\Type\Logical\StructureType;
 use Flow\Types\Value\Json;
 use Flow\Types\Value\Uuid;
 use Generator;
@@ -249,12 +248,6 @@ public static function hydratable_datasets(): Generator
             ],
         ];
 
-        /** @var StructureType<array<array-key, mixed>> $allowExtraStructure */
-        $allowExtraStructure = type_structure([
-            'a' => type_integer(),
-            'b' => structure_element('b', type_string(), optional: true),
-        ], true);
-
         yield 'containers from raw values' => [
             schema(
                 list_schema('l', type_list(type_integer())),
@@ -266,7 +259,6 @@ public static function hydratable_datasets(): Generator
                     'a' => type_integer(),
                     'b' => structure_element('b', type_string(), optional: true),
                 ])),
-                structure_schema('se', $allowExtraStructure),
             ),
             [
                 new RawRowValues([
@@ -276,7 +268,6 @@ public static function hydratable_datasets(): Generator
                     'mi' => [0 => 'x', 5 => 7],
                     'lo' => ['1', null, 3],
                     'st' => ['a' => '5', 'extra' => 'dropped'],
-                    'se' => ['a' => 1, 'b' => 'kept', 'other' => 'dropped'],
                 ]),
                 new RawRowValues([
                     'l' => [],
@@ -285,7 +276,6 @@ public static function hydratable_datasets(): Generator
                     'mi' => [],
                     'lo' => [],
                     'st' => ['a' => 1, 'b' => 'present'],
-                    'se' => ['a' => 2],
                 ]),
             ],
         ];