diff --git a/src/FastExpressionCompiler.LightExpression/FlatExpression.cs b/src/FastExpressionCompiler.LightExpression/FlatExpression.cs index 52654ef8..01338164 100644 --- a/src/FastExpressionCompiler.LightExpression/FlatExpression.cs +++ b/src/FastExpressionCompiler.LightExpression/FlatExpression.cs @@ -46,21 +46,27 @@ public enum ExprNodeKind : byte } /// Stores one flat expression node plus its intrusive child-link metadata in 24 bytes on 64-bit runtimes. +/// +/// Layout (64-bit): Type(8) | Obj(8) | _meta(4) | _data(4) = 24 bytes. +/// _meta bits: NodeType(8)|Tag(8)|NextIdx(16). +/// _data bits: ChildCount(16)|ChildIdx(16) for regular nodes, +/// or the raw 32-bit value for inline primitive constants (when == ). +/// [StructLayout(LayoutKind.Explicit, Size = 24)] public struct ExprNode { - private const int NodeTypeShift = 56; - private const int TagShift = 48; - private const int NextShift = 32; - private const int CountShift = 16; - private const ulong IndexMask = 0xFFFF; - private const ulong KindMask = 0x0F; - private const ulong NextMask = IndexMask << NextShift; - private const ulong ChildCountMask = IndexMask << CountShift; - private const ulong ChildInfoMask = ChildCountMask | IndexMask; - private const ulong KeepWithoutNextMask = ~NextMask; - private const ulong KeepWithoutChildInfoMask = ~ChildInfoMask; + // _meta layout: bits [31:24]=NodeType | [23:20]=Flags | [19:16]=Kind | [15:0]=NextIdx + private const int MetaNodeTypeShift = 24; + private const int MetaTagShift = 16; + private const uint MetaKeepWithoutNext = 0xFFFF0000u; + // _data layout: bits [31:16]=ChildCount | [15:0]=ChildIdx (or full uint for inline constants) + private const int DataCountShift = 16; + private const uint DataIdxMask = 0xFFFFu; private const int FlagsShift = 4; + private const uint KindMask = 0x0Fu; + + /// Sentinel placed in to indicate the node holds a small primitive constant in . + internal static readonly object InlineValueMarker = new(); /// Gets or sets the runtime type of the represented node. [FieldOffset(0)] @@ -69,47 +75,60 @@ public struct ExprNode /// Gets or sets the runtime payload associated with the node. [FieldOffset(8)] public object Obj; + + /// NodeType(8b) | Tag=(Flags:4b|Kind:4b)(8b) | NextIdx(16b) [FieldOffset(16)] - private ulong _data; + private uint _meta; + + /// ChildCount(16b) | ChildIdx(16b) —OR— raw 32-bit inline constant value. + [FieldOffset(20)] + private uint _data; /// Gets the expression kind encoded for this node. - public ExpressionType NodeType => (ExpressionType)((_data >> NodeTypeShift) & 0xFF); + public ExpressionType NodeType => (ExpressionType)(_meta >> MetaNodeTypeShift); /// Gets the payload classification for this node. - public ExprNodeKind Kind => (ExprNodeKind)((_data >> TagShift) & KindMask); + public ExprNodeKind Kind => (ExprNodeKind)((_meta >> MetaTagShift) & KindMask); - internal byte Flags => (byte)(((byte)(_data >> TagShift)) >> FlagsShift); + internal byte Flags => (byte)((_meta >> (MetaTagShift + FlagsShift)) & 0xFu); /// Gets the next sibling node index in the intrusive child chain. - public int NextIdx => (int)((_data >> NextShift) & IndexMask); + public int NextIdx => (int)(_meta & 0xFFFFu); /// Gets the number of direct children linked from this node. - public int ChildCount => (int)((_data >> CountShift) & IndexMask); + public int ChildCount => (int)(_data >> DataCountShift); /// Gets the first child index or an auxiliary payload index. - public int ChildIdx => (int)(_data & IndexMask); + public int ChildIdx => (int)(_data & DataIdxMask); + + /// Gets the raw 32-bit value for inline primitive constants. Only valid when == . + internal uint InlineValue => _data; internal ExprNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind kind, byte flags = 0, int childIdx = 0, int childCount = 0, int nextIdx = 0) { Type = type; Obj = obj; var tag = (byte)((flags << FlagsShift) | (byte)kind); - _data = ((ulong)(byte)nodeType << NodeTypeShift) - | ((ulong)tag << TagShift) - | ((ulong)(ushort)nextIdx << NextShift) - | ((ulong)(ushort)childCount << CountShift) - | (ushort)childIdx; + _meta = ((uint)(byte)nodeType << MetaNodeTypeShift) | ((uint)tag << MetaTagShift) | (ushort)nextIdx; + _data = ((uint)(ushort)childCount << DataCountShift) | (ushort)childIdx; + } + + /// Constructs an inline primitive constant node; is set to . + internal ExprNode(Type type, uint inlineValue) + { + Type = type; + Obj = InlineValueMarker; + _meta = (uint)(byte)ExpressionType.Constant << MetaNodeTypeShift; + _data = inlineValue; } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void SetNextIdx(int nextIdx) => - _data = (_data & KeepWithoutNextMask) | ((ulong)(ushort)nextIdx << NextShift); + _meta = (_meta & MetaKeepWithoutNext) | (ushort)nextIdx; [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void SetChildInfo(int childIdx, int childCount) => - _data = (_data & KeepWithoutChildInfoMask) - | ((ulong)(ushort)childCount << CountShift) - | (ushort)childIdx; + _data = ((uint)(ushort)childCount << DataCountShift) | (ushort)childIdx; [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool Is(ExprNodeKind kind) => Kind == kind; @@ -122,7 +141,9 @@ internal void SetChildInfo(int childIdx, int childCount) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool ShouldCloneWhenLinked() => - Kind == ExprNodeKind.LabelTarget || NodeType == ExpressionType.Parameter || Kind == ExprNodeKind.ObjectReference || ChildCount == 0; + ReferenceEquals(Obj, InlineValueMarker) || + Kind == ExprNodeKind.LabelTarget || NodeType == ExpressionType.Parameter || + Kind == ExprNodeKind.ObjectReference || ChildCount == 0; } /// Stores an expression tree as a flat node array plus out-of-line closure constants. @@ -154,35 +175,61 @@ public int Parameter(Type type, string name = null) } /// Adds a typed parameter node and returns its index. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int ParameterOf(string name = null) => Parameter(typeof(T), name); /// Adds a variable node and returns its index. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Variable(Type type, string name = null) => Parameter(type, name); /// Adds a default-value node and returns its index. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Default(Type type) => AddRawExpressionNode(type, null, ExpressionType.Default); /// Adds a constant node using the runtime type of the supplied value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Constant(object value) => Constant(value, value?.GetType() ?? typeof(object)); /// Adds a constant node with an explicit constant type. public int Constant(object value, Type type) { - if (ShouldInlineConstant(value, type)) + if (value == null || value is string || value is Type || value is decimal) return AddRawExpressionNode(type, value, ExpressionType.Constant); + if (type.IsEnum) + { + var underlyingTc = Type.GetTypeCode(Enum.GetUnderlyingType(type)); + if (IsSmallPrimitive(underlyingTc)) + return AddInlineConstantNode(type, (uint)System.Convert.ToInt64(value)); + // long/ulong-backed enum (extremely rare): store boxed in Obj + return AddRawExpressionNode(type, value, ExpressionType.Constant); + } + + if (type.IsPrimitive) + { + var tc = Type.GetTypeCode(type); + if (IsSmallPrimitive(tc)) + return AddInlineConstantNode(type, ToInlineValue(value, tc)); + // long, ulong, double: primitive but too wide for _data, store boxed in Obj + return AddRawExpressionNode(type, value, ExpressionType.Constant); + } + + // Delegate, array types, and user-defined reference/value types go to ClosureConstants var constantIndex = ClosureConstants.Add(value); - return AddRawExpressionNodeWithChildIndex(type, ClosureConstantMarker, ExpressionType.Constant, constantIndex); + return AddRawLeafExpressionNode(type, ClosureConstantMarker, ExpressionType.Constant, childIdx: constantIndex); } /// Adds a null constant node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int ConstantNull(Type type = null) => AddRawExpressionNode(type ?? typeof(object), null, ExpressionType.Constant); /// Adds an constant node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int ConstantInt(int value) => AddRawExpressionNode(typeof(int), value, ExpressionType.Constant); /// Adds a typed constant node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int ConstantOf(T value) => Constant(value, typeof(T)); /// Adds a parameterless new node for the specified type. @@ -234,12 +281,15 @@ public int MakeMemberAccess(int? instance, System.Reflection.MemberInfo member) : AddRawExpressionNode(GetMemberType(member), member, ExpressionType.MemberAccess); /// Adds a field-access node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Field(int instance, System.Reflection.FieldInfo field) => MakeMemberAccess(instance, field); /// Adds a property-access node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Property(int instance, System.Reflection.PropertyInfo property) => MakeMemberAccess(instance, property); /// Adds a static property-access node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Property(System.Reflection.PropertyInfo property) => MakeMemberAccess(null, property); /// Adds an indexed property-access node. @@ -249,6 +299,7 @@ public int Property(int instance, System.Reflection.PropertyInfo property, param : AddFactoryExpressionNode(property.PropertyType, property, ExpressionType.Index, PrependToChildList(instance, arguments)); /// Adds a one-dimensional array index node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int ArrayIndex(int array, int index) => MakeBinary(ExpressionType.ArrayIndex, array, index); /// Adds an array access node. @@ -258,18 +309,22 @@ public int ArrayAccess(int array, params int[] indexes) => : AddFactoryExpressionNode(GetArrayElementType(Nodes[array].Type, indexes?.Length ?? 0), null, ExpressionType.Index, PrependToChildList(array, indexes)); /// Adds a conversion node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Convert(int operand, Type type, System.Reflection.MethodInfo method = null) => AddFactoryExpressionNode(type, method, ExpressionType.Convert, operand); /// Adds a type-as node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int TypeAs(int operand, Type type) => AddFactoryExpressionNode(type, null, ExpressionType.TypeAs, operand); /// Adds a numeric negation node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Negate(int operand, System.Reflection.MethodInfo method = null) => MakeUnary(ExpressionType.Negate, operand, method: method); /// Adds a logical or bitwise not node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Not(int operand, System.Reflection.MethodInfo method = null) => MakeUnary(ExpressionType.Not, operand, method: method); @@ -278,12 +333,15 @@ public int MakeUnary(ExpressionType nodeType, int operand, Type type = null, Sys AddFactoryExpressionNode(type ?? GetUnaryResultType(nodeType, Nodes[operand].Type, method), method, nodeType, operand); /// Adds an assignment node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Assign(int left, int right) => MakeBinary(ExpressionType.Assign, left, right); /// Adds an addition node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Add(int left, int right, System.Reflection.MethodInfo method = null) => MakeBinary(ExpressionType.Add, left, right, method: method); /// Adds an equality node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Equal(int left, int right, System.Reflection.MethodInfo method = null) => MakeBinary(ExpressionType.Equal, left, right, method: method); /// Adds a binary node of the specified kind. @@ -300,10 +358,23 @@ public int Condition(int test, int ifTrue, int ifFalse, Type type = null) => AddFactoryExpressionNode(type ?? Nodes[ifTrue].Type, null, ExpressionType.Conditional, 0, test, ifTrue, ifFalse); /// Adds a block node without explicit variables. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Block(params int[] expressions) => Block(null, null, expressions); /// Adds a block node with optional explicit result type and variables. + /// + /// Child layout of the Block node depends on whether there are explicit variables: + /// + /// With variables: children[0] = ChildList(variable₀, variable₁, …) + /// children[1] = ChildList(expr₀, expr₁, …) + /// Without variables: children[0] = ChildList(expr₀, expr₁, …) + /// + /// A children.Count == 2 check is therefore the canonical way to detect variables. + /// Variable parameter nodes share the same id-slot as the refs used inside the body + /// (out-of-order: the variable decl nodes appear in children[0] before the body expressions + /// that reference them in children[1]). + /// public int Block(Type type, IEnumerable variables, params int[] expressions) { if (expressions == null || expressions.Length == 0) @@ -326,10 +397,24 @@ public int Block(Type type, IEnumerable variables, params int[] expressions } /// Adds a typed lambda node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Lambda(int body, params int[] parameters) where TDelegate : Delegate => Lambda(typeof(TDelegate), body, parameters); /// Adds a lambda node. + /// + /// Child layout of the Lambda node: + /// + /// children[0] = body expression + /// children[1…n] = parameter decl nodes (parameter₀, parameter₁, …) + /// + /// The body is stored first; parameter decl nodes follow. This means that when the + /// body contains refs to those parameters, the ref nodes are encountered by the + /// before the corresponding decl node — an intentional + /// out-of-order decl pattern. The Reader resolves identity through a shared id map + /// so that all refs and the single decl resolve to the same + /// object. + /// public int Lambda(Type delegateType, int body, params int[] parameters) => parameters == null || parameters.Length == 0 ? AddFactoryExpressionNode(delegateType, null, ExpressionType.Lambda, 0, body) @@ -386,9 +471,11 @@ public int MakeGoto(GotoExpressionKind kind, int target, int? value = null, Type } /// Adds a goto node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Goto(int target, int? value = null, Type type = null) => MakeGoto(GotoExpressionKind.Goto, target, value, type); /// Adds a return node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Return(int target, int value) => MakeGoto(GotoExpressionKind.Return, target, value, Nodes[value].Type); /// Adds a loop node. @@ -498,10 +585,12 @@ public int TryCatchFinally(int body, int? @finally, params int[] handlers) } /// Adds a type-test node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int TypeIs(int expression, Type type) => AddFactoryExpressionNode(typeof(bool), type, ExpressionType.TypeIs, expression); /// Adds a type-equality test node. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int TypeEqual(int expression, Type type) => AddFactoryExpressionNode(typeof(bool), type, ExpressionType.TypeEqual, expression); @@ -545,27 +634,35 @@ public SysExpr ToExpression() => [RequiresUnreferencedCode(FastExpressionCompiler.LightExpression.Trimming.Message)] public LightExpression ToLightExpression() => FastExpressionCompiler.LightExpression.FromSysExpressionConverter.ToLightExpression(ToExpression()); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryExpressionNode(Type type, object obj, ExpressionType nodeType, int child) => AddNode(type, obj, nodeType, ExprNodeKind.Expression, 0, CloneChild(child)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryExpressionNode(Type type, object obj, ExpressionType nodeType, byte flags, int child) => AddNode(type, obj, nodeType, ExprNodeKind.Expression, flags, CloneChild(child)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryExpressionNode(Type type, object obj, ExpressionType nodeType, byte flags, int c0, int c1) => AddNode(type, obj, nodeType, ExprNodeKind.Expression, flags, CloneChild(c0), CloneChild(c1)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryExpressionNode(Type type, object obj, ExpressionType nodeType, byte flags, int c0, int c1, int c2) => AddNode(type, obj, nodeType, ExprNodeKind.Expression, flags, CloneChild(c0), CloneChild(c1), CloneChild(c2)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryExpressionNode(Type type, object obj, ExpressionType nodeType, byte flags, int c0, int c1, int c2, int c3) => AddNode(type, obj, nodeType, ExprNodeKind.Expression, flags, CloneChild(c0), CloneChild(c1), CloneChild(c2), CloneChild(c3)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryExpressionNode(Type type, object obj, ExpressionType nodeType, byte flags, int c0, int c1, int c2, int c3, int c4) => AddNode(type, obj, nodeType, ExprNodeKind.Expression, flags, CloneChild(c0), CloneChild(c1), CloneChild(c2), CloneChild(c3), CloneChild(c4)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryExpressionNode(Type type, object obj, ExpressionType nodeType, byte flags, int c0, int c1, int c2, int c3, int c4, int c5) => AddNode(type, obj, nodeType, ExprNodeKind.Expression, flags, CloneChild(c0), CloneChild(c1), CloneChild(c2), CloneChild(c3), CloneChild(c4), CloneChild(c5)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryExpressionNode(Type type, object obj, ExpressionType nodeType, byte flags, int c0, int c1, int c2, int c3, int c4, int c5, int c6) => AddNode(type, obj, nodeType, ExprNodeKind.Expression, flags, CloneChild(c0), CloneChild(c1), CloneChild(c2), CloneChild(c3), CloneChild(c4), CloneChild(c5), CloneChild(c6)); @@ -587,42 +684,49 @@ private int AddFactoryExpressionNode(Type type, object obj, ExpressionType nodeT return AddNode(type, obj, nodeType, ExprNodeKind.Expression, 0, in cloned); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryExpressionNode(Type type, object obj, ExpressionType nodeType, in ChildList children) { var cloned = CloneChildren(children); return AddNode(type, obj, nodeType, ExprNodeKind.Expression, 0, in cloned); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryExpressionNode(Type type, object obj, ExpressionType nodeType, byte flags, in ChildList children) { var cloned = CloneChildren(children); return AddNode(type, obj, nodeType, ExprNodeKind.Expression, flags, in cloned); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddRawExpressionNode(Type type, object obj, ExpressionType nodeType) => AddLeafNode(type, obj, nodeType, ExprNodeKind.Expression, 0, 0, 0); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddRawExpressionNode(Type type, object obj, ExpressionType nodeType, in ChildList children) => AddNode(type, obj, nodeType, ExprNodeKind.Expression, 0, in children); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddRawExpressionNode(Type type, object obj, ExpressionType nodeType, int[] children) => AddNode(type, obj, nodeType, ExprNodeKind.Expression, 0, children); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddRawExpressionNode(Type type, object obj, ExpressionType nodeType, int child0, int child1, int child2) => AddNode(type, obj, nodeType, ExprNodeKind.Expression, 0, child0, child1, child2); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddRawLeafExpressionNode(Type type, object obj, ExpressionType nodeType, byte flags = 0, int childIdx = 0, int childCount = 0) => AddLeafNode(type, obj, nodeType, ExprNodeKind.Expression, flags, childIdx, childCount); - private int AddRawExpressionNodeWithChildIndex(Type type, object obj, ExpressionType nodeType, int childIdx) => - AddRawLeafExpressionNode(type, obj, nodeType, childIdx: childIdx); - + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryAuxNode(Type type, object obj, ExprNodeKind kind, byte flags, int child) => AddNode(type, obj, ExpressionType.Extension, kind, flags, CloneChild(child)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryAuxNode(Type type, object obj, ExprNodeKind kind, int child) => AddFactoryAuxNode(type, obj, kind, 0, child); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryAuxNode(Type type, object obj, ExprNodeKind kind, byte flags, int child0, int child1) => AddNode(type, obj, ExpressionType.Extension, kind, flags, CloneChild(child0), CloneChild(child1)); @@ -632,27 +736,34 @@ private int AddFactoryAuxNode(Type type, object obj, ExprNodeKind kind, int[] ch return AddNode(type, obj, ExpressionType.Extension, kind, 0, in cloned); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryAuxNode(Type type, object obj, ExprNodeKind kind, byte flags, in ChildList children) { var cloned = CloneChildren(children); return AddNode(type, obj, ExpressionType.Extension, kind, flags, in cloned); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddFactoryAuxNode(Type type, object obj, ExprNodeKind kind, in ChildList children) => AddFactoryAuxNode(type, obj, kind, 0, in children); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddRawAuxNode(Type type, object obj, ExprNodeKind kind, in ChildList children) => AddNode(type, obj, ExpressionType.Extension, kind, 0, in children); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddRawLeafAuxNode(Type type, object obj, ExprNodeKind kind, byte flags = 0, int childIdx = 0, int childCount = 0) => AddLeafNode(type, obj, ExpressionType.Extension, kind, flags, childIdx, childCount); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddObjectReferenceNode(Type type, object obj) => AddRawLeafAuxNode(type, obj, ExprNodeKind.ObjectReference); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddChildListNode(in ChildList children) => AddRawAuxNode(null, null, ExprNodeKind.ChildList, in children); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddUInt16PairNode(int first, int second) => AddRawLeafAuxNode(null, null, ExprNodeKind.UInt16Pair, childIdx: checked((ushort)first), childCount: checked((ushort)second)); @@ -703,6 +814,10 @@ private int AddExpression(SysExpr expression) } case ExpressionType.Lambda: { + // Layout: children[0] = body, children[1..n] = parameter decl nodes. + // Body is stored before parameters so that the Reader encounters parameter + // refs in the body before their decl nodes (out-of-order decl); identity + // is preserved via the shared _parametersById id-map. var lambda = (System.Linq.Expressions.LambdaExpression)expression; ChildList children = default; children.Add(AddExpression(lambda.Body)); @@ -712,6 +827,10 @@ private int AddExpression(SysExpr expression) } case ExpressionType.Block: { + // Layout (with variables): children[0] = ChildList(var₀, var₁, …) + // children[1] = ChildList(expr₀, expr₁, …) + // Layout (without variables): children[0] = ChildList(expr₀, expr₁, …) + // children.Count == 2 is the canonical test for the presence of variables. var block = (System.Linq.Expressions.BlockExpression)expression; ChildList children = default; if (block.Variables.Count != 0) @@ -934,14 +1053,8 @@ private int AddExpression(SysExpr expression) } } - private int AddConstant(System.Linq.Expressions.ConstantExpression constant) - { - if (ShouldInlineConstant(constant.Value, constant.Type)) - return _tree.AddRawExpressionNode(constant.Type, constant.Value, constant.NodeType); - - var constantIndex = _tree.ClosureConstants.Add(constant.Value); - return _tree.AddRawExpressionNodeWithChildIndex(constant.Type, ClosureConstantMarker, constant.NodeType, constantIndex); - } + private int AddConstant(System.Linq.Expressions.ConstantExpression constant) => + _tree.Constant(constant.Value, constant.Type); private int AddSwitchCase(SysSwitchCase switchCase) { @@ -1021,6 +1134,7 @@ private static int GetId(ref SmallMap16> ids, object }; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int AddLeafNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind kind, byte flags, int childIdx, int childCount) { var nodeIndex = Nodes.Count; @@ -1029,6 +1143,15 @@ private int AddLeafNode(Type type, object obj, ExpressionType nodeType, ExprNode return nodeIndex; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int AddInlineConstantNode(Type type, uint inlineValue) + { + var nodeIndex = Nodes.Count; + ref var newNode = ref Nodes.AddDefaultAndGetRef(); + newNode = new ExprNode(type, inlineValue); + return nodeIndex; + } + private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind kind, byte flags) { var nodeIndex = Nodes.Count; @@ -1050,7 +1173,7 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind var nodeIndex = Nodes.Count; ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, c0, 2); - Nodes[c0].SetNextIdx(c1); + Nodes.GetSurePresentRef(c0).SetNextIdx(c1); return nodeIndex; } @@ -1059,8 +1182,8 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind var nodeIndex = Nodes.Count; ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, c0, 3); - Nodes[c0].SetNextIdx(c1); - Nodes[c1].SetNextIdx(c2); + Nodes.GetSurePresentRef(c0).SetNextIdx(c1); + Nodes.GetSurePresentRef(c1).SetNextIdx(c2); return nodeIndex; } @@ -1069,9 +1192,9 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind var nodeIndex = Nodes.Count; ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, c0, 4); - Nodes[c0].SetNextIdx(c1); - Nodes[c1].SetNextIdx(c2); - Nodes[c2].SetNextIdx(c3); + Nodes.GetSurePresentRef(c0).SetNextIdx(c1); + Nodes.GetSurePresentRef(c1).SetNextIdx(c2); + Nodes.GetSurePresentRef(c2).SetNextIdx(c3); return nodeIndex; } @@ -1080,10 +1203,10 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind var nodeIndex = Nodes.Count; ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, c0, 5); - Nodes[c0].SetNextIdx(c1); - Nodes[c1].SetNextIdx(c2); - Nodes[c2].SetNextIdx(c3); - Nodes[c3].SetNextIdx(c4); + Nodes.GetSurePresentRef(c0).SetNextIdx(c1); + Nodes.GetSurePresentRef(c1).SetNextIdx(c2); + Nodes.GetSurePresentRef(c2).SetNextIdx(c3); + Nodes.GetSurePresentRef(c3).SetNextIdx(c4); return nodeIndex; } @@ -1092,11 +1215,11 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind var nodeIndex = Nodes.Count; ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, c0, 6); - Nodes[c0].SetNextIdx(c1); - Nodes[c1].SetNextIdx(c2); - Nodes[c2].SetNextIdx(c3); - Nodes[c3].SetNextIdx(c4); - Nodes[c4].SetNextIdx(c5); + Nodes.GetSurePresentRef(c0).SetNextIdx(c1); + Nodes.GetSurePresentRef(c1).SetNextIdx(c2); + Nodes.GetSurePresentRef(c2).SetNextIdx(c3); + Nodes.GetSurePresentRef(c3).SetNextIdx(c4); + Nodes.GetSurePresentRef(c4).SetNextIdx(c5); return nodeIndex; } @@ -1105,12 +1228,12 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind var nodeIndex = Nodes.Count; ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, c0, 7); - Nodes[c0].SetNextIdx(c1); - Nodes[c1].SetNextIdx(c2); - Nodes[c2].SetNextIdx(c3); - Nodes[c3].SetNextIdx(c4); - Nodes[c4].SetNextIdx(c5); - Nodes[c5].SetNextIdx(c6); + Nodes.GetSurePresentRef(c0).SetNextIdx(c1); + Nodes.GetSurePresentRef(c1).SetNextIdx(c2); + Nodes.GetSurePresentRef(c2).SetNextIdx(c3); + Nodes.GetSurePresentRef(c3).SetNextIdx(c4); + Nodes.GetSurePresentRef(c4).SetNextIdx(c5); + Nodes.GetSurePresentRef(c5).SetNextIdx(c6); return nodeIndex; } @@ -1123,7 +1246,7 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, children[0], children.Length); for (var i = 1; i < children.Length; ++i) - Nodes[children[i - 1]].SetNextIdx(children[i]); + Nodes.GetSurePresentRef(children[i - 1]).SetNextIdx(children[i]); return nodeIndex; } @@ -1136,13 +1259,30 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, children[0], children.Count); for (var i = 1; i < children.Count; ++i) - Nodes[children[i - 1]].SetNextIdx(children[i]); + Nodes.GetSurePresentRef(children[i - 1]).SetNextIdx(children[i]); return nodeIndex; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool ShouldInlineConstant(object value, Type type) => - value == null || value is string || value is Type || type.IsEnum || Type.GetTypeCode(type) != TypeCode.Object; + private static bool IsSmallPrimitive(TypeCode tc) => + tc == TypeCode.Boolean || tc == TypeCode.Byte || tc == TypeCode.SByte || + tc == TypeCode.Char || tc == TypeCode.Int16 || tc == TypeCode.UInt16 || + tc == TypeCode.Int32 || tc == TypeCode.UInt32 || tc == TypeCode.Single; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint ToInlineValue(object value, TypeCode tc) => tc switch + { + TypeCode.Boolean => (bool)value ? 1u : 0u, + TypeCode.Byte => (byte)value, + TypeCode.SByte => (uint)(byte)(sbyte)value, + TypeCode.Char => (char)value, + TypeCode.Int16 => (uint)(ushort)(short)value, + TypeCode.UInt16 => (ushort)value, + TypeCode.Int32 => (uint)(int)value, + TypeCode.UInt32 => (uint)value, + TypeCode.Single => FloatBits.ToUInt((float)value), + _ => FlatExpressionThrow.UnsupportedInlineConstantType(value, tc) + }; private static Type GetMemberType(System.Reflection.MemberInfo member) => member switch { @@ -1181,12 +1321,14 @@ private static Type GetArrayElementType(Type arrayType, int depth) return elementType ?? typeof(object); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int CloneChild(int index) { ref var node = ref Nodes[index]; - return node.ShouldCloneWhenLinked() - ? AddLeafNode(node.Type, node.Obj, node.NodeType, node.Kind, node.Flags, node.ChildIdx, node.ChildCount) - : index; + if (!node.ShouldCloneWhenLinked()) return index; + if (ReferenceEquals(node.Obj, ExprNode.InlineValueMarker)) + return AddInlineConstantNode(node.Type, node.InlineValue); + return AddLeafNode(node.Type, node.Obj, node.NodeType, node.Kind, node.Flags, node.ChildIdx, node.ChildCount); } private ChildList CloneChildren(int[] children) @@ -1232,9 +1374,11 @@ public SysExpr ReadExpression(int index) switch (node.NodeType) { case ExpressionType.Constant: - return SysExpr.Constant(ReferenceEquals(node.Obj, ClosureConstantMarker) - ? _tree.ClosureConstants[node.ChildIdx] - : node.Obj, node.Type); + if (ReferenceEquals(node.Obj, ClosureConstantMarker)) + return SysExpr.Constant(_tree.ClosureConstants[node.ChildIdx], node.Type); + if (ReferenceEquals(node.Obj, ExprNode.InlineValueMarker)) + return SysExpr.Constant(ReadInlineValue(node.Type, node.InlineValue), node.Type); + return SysExpr.Constant(node.Obj, node.Type); case ExpressionType.Default: return SysExpr.Default(node.Type); case ExpressionType.Parameter: @@ -1248,6 +1392,9 @@ public SysExpr ReadExpression(int index) } case ExpressionType.Lambda: { + // Layout: children[0] = body, children[1..n] = parameter decl nodes. + // Body is read first; parameter refs inside it are resolved via _parametersById + // even before the decl nodes at children[1..n] are visited (out-of-order decl). var children = GetChildren(index); var body = ReadExpression(children[0]); var parameters = new SysParameterExpression[children.Count - 1]; @@ -1257,6 +1404,13 @@ public SysExpr ReadExpression(int index) } case ExpressionType.Block: { + // Layout (with variables): children[0] = ChildList(var₀, var₁, …) + // children[1] = ChildList(expr₀, expr₁, …) + // Layout (without variables): children[0] = ChildList(expr₀, expr₁, …) + // children.Count == 2 is the canonical test for the presence of variables. + // Variable decl nodes in children[0] are registered in _parametersById before + // the body expressions in children[1] are read, so refs in the body resolve + // to the same SysParameterExpression object as the decl (normal order here). var children = GetChildren(index); var hasVariables = children.Count == 2; var variableIndexes = hasVariables ? GetChildren(children[0]) : default; @@ -1542,18 +1696,48 @@ private SysElementInit ReadElementInit(int index) private ChildList GetChildren(int index) { - ref var node = ref _tree.Nodes[index]; + ref var node = ref _tree.Nodes.GetSurePresentRef(index); var count = node.ChildCount; ChildList children = default; var childIndex = node.ChildIdx; for (var i = 0; i < count; ++i) { children.Add(childIndex); - childIndex = _tree.Nodes[childIndex].NextIdx; + childIndex = _tree.Nodes.GetSurePresentRef(childIndex).NextIdx; } return children; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static object ReadInlineValue(Type type, uint data) + { + if (type.IsEnum) + return Enum.ToObject(type, Type.GetTypeCode(Enum.GetUnderlyingType(type)) switch + { + TypeCode.Byte => (object)(byte)data, + TypeCode.SByte => (object)(sbyte)(byte)data, + TypeCode.Char => (object)(char)(ushort)data, + TypeCode.Int16 => (object)(short)(ushort)data, + TypeCode.UInt16 => (object)(ushort)data, + TypeCode.Int32 => (object)(int)data, + TypeCode.UInt32 => (object)data, + var tc => FlatExpressionThrow.UnsupportedInlineConstantType(type, tc) + }); + return Type.GetTypeCode(type) switch + { + TypeCode.Boolean => (object)(data != 0), + TypeCode.Byte => (object)(byte)data, + TypeCode.SByte => (object)(sbyte)(byte)data, + TypeCode.Char => (object)(char)(ushort)data, + TypeCode.Int16 => (object)(short)(ushort)data, + TypeCode.UInt16 => (object)(ushort)data, + TypeCode.Int32 => (object)(int)data, + TypeCode.UInt32 => (object)data, + TypeCode.Single => (object)FloatBits.ToFloat(data), + _ => FlatExpressionThrow.UnsupportedInlineConstantType(type) + }; + } + [RequiresUnreferencedCode(FastExpressionCompiler.LightExpression.Trimming.Message)] private SysExpr[] ReadExpressions(in ChildList childIndexes) { @@ -1571,6 +1755,36 @@ private SysExpr[] ReadExpressions(in ChildList childIndexes) } +/// Union struct for reinterpreting float bits as uint without unsafe code. +[StructLayout(LayoutKind.Explicit)] +internal struct FloatBits +{ + [FieldOffset(0)] private float _floatValue; + [FieldOffset(0)] private uint _uintValue; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static uint ToUInt(float value) => new FloatBits { _floatValue = value }._uintValue; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static float ToFloat(uint value) => new FloatBits { _uintValue = value }._floatValue; +} + +/// Throw helpers that prevent bare throw from blocking inlining of hot-path callers. +internal static class FlatExpressionThrow +{ + [MethodImpl(MethodImplOptions.NoInlining)] + internal static T UnsupportedInlineConstantType(Type type) => + throw new NotSupportedException($"Cannot reconstruct inline constant of type {type}"); + + [MethodImpl(MethodImplOptions.NoInlining)] + internal static T UnsupportedInlineConstantType(Type type, TypeCode tc) => + throw new NotSupportedException($"Cannot reconstruct inline constant of type {type} with TypeCode {tc}"); + + [MethodImpl(MethodImplOptions.NoInlining)] + internal static T UnsupportedInlineConstantType(object value, TypeCode tc) => + throw new NotSupportedException($"Cannot convert value '{value}' of TypeCode {tc} to an inline constant"); +} + /// Provides conversions from System and LightExpression trees to . public static class FlatExpressionExtensions { diff --git a/test/FastExpressionCompiler.LightExpression.UnitTests/LightExpressionTests.cs b/test/FastExpressionCompiler.LightExpression.UnitTests/LightExpressionTests.cs index b91c23c0..eea42063 100644 --- a/test/FastExpressionCompiler.LightExpression.UnitTests/LightExpressionTests.cs +++ b/test/FastExpressionCompiler.LightExpression.UnitTests/LightExpressionTests.cs @@ -34,7 +34,13 @@ public int Run() Can_build_flat_expression_directly_with_light_expression_like_api(); Can_build_flat_expression_control_flow_directly(); Can_property_test_generated_flat_expression_roundtrip_structurally(); - return 17; + Flat_lambda_parameter_ref_before_decl_preserves_identity(); + Flat_lambda_multiple_parameter_refs_all_yield_same_identity(); + Flat_block_variables_and_refs_yield_same_identity(); + Flat_nested_lambda_captures_outer_parameter_identity(); + Flat_out_of_order_decl_block_in_lambda_compiles_correctly(); + Flat_enum_constant_stored_inline_roundtrip(); + return 23; } @@ -516,5 +522,169 @@ public void Can_embed_normal_Expression_into_LightExpression_eg_as_Constructor_a Asserts.IsInstanceOf

(func()); } + + // Tests for decl vs ref nodes and out-of-order decl in lambdas/blocks + + ///

+ /// In the flat encoding, a lambda stores body first then parameters. + /// So when reading, parameter refs in the body are encountered BEFORE + /// the parameter decl node in the parameter list (out-of-order decl). + /// Both should resolve to the exact same SysParameterExpression. + /// + public void Flat_lambda_parameter_ref_before_decl_preserves_identity() + { + var fe = default(ExprTree); + var p = fe.ParameterOf("p"); + // body uses p: ref nodes come first when the lambda is encoded/read + fe.RootIndex = fe.Lambda>(fe.Add(p, fe.ConstantInt(1)), p); + + var sysLambda = (System.Linq.Expressions.LambdaExpression)fe.ToExpression(); + var add = (System.Linq.Expressions.BinaryExpression)sysLambda.Body; + + // The parameter in the params list and its ref in the body must be the same object + Asserts.AreSame(sysLambda.Parameters[0], add.Left); + } + + /// + /// A parameter referenced more than once in a lambda body (all refs are + /// out-of-order relative to the single decl at the end of the child list) + /// must all resolve to the same SysParameterExpression. + /// + public void Flat_lambda_multiple_parameter_refs_all_yield_same_identity() + { + var fe = default(ExprTree); + var p = fe.ParameterOf("p"); + // p * p + p: three independent refs to the same parameter + fe.RootIndex = fe.Lambda>( + fe.Add(fe.MakeBinary(System.Linq.Expressions.ExpressionType.Multiply, p, p), p), + p); + + var sysLambda = (System.Linq.Expressions.LambdaExpression)fe.ToExpression(); + var add = (System.Linq.Expressions.BinaryExpression)sysLambda.Body; + var mul = (System.Linq.Expressions.BinaryExpression)add.Left; + var paramDecl = sysLambda.Parameters[0]; + + Asserts.AreSame(paramDecl, mul.Left); + Asserts.AreSame(paramDecl, mul.Right); + Asserts.AreSame(paramDecl, add.Right); + } + + /// + /// Block variables are read before body expressions (normal order), + /// but each variable index is cloned whenever it appears as a child. + /// All clones must resolve to the same SysParameterExpression. + /// + public void Flat_block_variables_and_refs_yield_same_identity() + { + var fe = default(ExprTree); + var p = fe.ParameterOf("p"); + var v1 = fe.Variable(typeof(int), "v1"); + var v2 = fe.Variable(typeof(int), "v2"); + // { int v1, v2; v1 = p; v2 = v1 + 1; v2 } + var block = fe.Block(typeof(int), + new[] { v1, v2 }, + fe.Assign(v1, p), + fe.Assign(v2, fe.Add(v1, fe.ConstantInt(1))), + v2); + fe.RootIndex = fe.Lambda>(block, p); + + var sysLambda = (System.Linq.Expressions.LambdaExpression)fe.ToExpression(); + var sysBlock = (System.Linq.Expressions.BlockExpression)sysLambda.Body; + var assign1 = (System.Linq.Expressions.BinaryExpression)sysBlock.Expressions[0]; // v1 = p + var assign2 = (System.Linq.Expressions.BinaryExpression)sysBlock.Expressions[1]; // v2 = v1 + 1 + var addExpr = (System.Linq.Expressions.BinaryExpression)assign2.Right; // v1 + 1 + + // v1 decl and its ref on the left of assign1 are the same object + Asserts.AreSame(sysBlock.Variables[0], assign1.Left); + // v1 decl and its ref inside the add expression are the same object + Asserts.AreSame(sysBlock.Variables[0], addExpr.Left); + // v2 decl and its ref on the left of assign2 are the same object + Asserts.AreSame(sysBlock.Variables[1], assign2.Left); + // v2 decl and the final block result expression are the same object + Asserts.AreSame(sysBlock.Variables[1], sysBlock.Expressions[2]); + } + + /// + /// An outer lambda parameter captured in a nested lambda body creates + /// a ref node in the nested lambda scope. All three occurrences — + /// the outer params list, the inner body, and any outer body usage — + /// must resolve to the exact same SysParameterExpression. + /// + public void Flat_nested_lambda_captures_outer_parameter_identity() + { + var fe = default(ExprTree); + var x = fe.ParameterOf("x"); + // outer: x => () => x (inner lambda closes over outer param) + var inner = fe.Lambda>(x); + fe.RootIndex = fe.Lambda>>(inner, x); + + var sysOuter = (System.Linq.Expressions.LambdaExpression)fe.ToExpression(); + var sysInner = (System.Linq.Expressions.LambdaExpression)sysOuter.Body; + + // The inner lambda body (the x ref) must be the same object as the outer param decl + Asserts.AreSame(sysOuter.Parameters[0], sysInner.Body); + } + + /// + /// End-to-end compile-and-run test with a block containing two variables, + /// verifying that out-of-order parameter decls and variable refs produce + /// a correctly executing delegate. + /// + public void Flat_out_of_order_decl_block_in_lambda_compiles_correctly() + { + var fe = default(ExprTree); + var p = fe.ParameterOf("p"); + var v1 = fe.Variable(typeof(int), "v1"); + var v2 = fe.Variable(typeof(int), "v2"); + // (int p) => { int v1 = p * 2; int v2 = v1 + p; v2 } + var block = fe.Block(typeof(int), + new[] { v1, v2 }, + fe.Assign(v1, fe.MakeBinary(System.Linq.Expressions.ExpressionType.Multiply, p, fe.ConstantInt(2))), + fe.Assign(v2, fe.Add(v1, p)), + v2); + fe.RootIndex = fe.Lambda>(block, p); + + var func = (Func)((System.Linq.Expressions.LambdaExpression)fe.ToExpression()).Compile(); + // p=3 → v1 = 3*2=6, v2 = 6+3=9 + Asserts.AreEqual(9, func(3)); + // p=0 → v1 = 0, v2 = 0 + Asserts.AreEqual(0, func(0)); + } + + enum ByteEnum : byte { A = 1, B = 200 } + enum SByteEnum : sbyte { A = -1, B = 50 } + enum ShortEnum : short { A = -1000, B = 30000 } + enum UShortEnum : ushort { A = 0, B = 60000 } + enum IntEnum : int { A = int.MinValue, B = 42 } + enum UIntEnum : uint { A = 0, B = uint.MaxValue } + + public void Flat_enum_constant_stored_inline_roundtrip() + { + // Verify that enum constants with ≤32-bit underlying types are stored inline + // (no ClosureConstants entry, no boxing) and round-trip correctly. + void Check(TEnum enumValue) where TEnum : Enum + { + var fe = default(ExprTree); + var idx = fe.Constant(enumValue, typeof(TEnum)); + Asserts.AreEqual(0, fe.ClosureConstants.Count, + $"{typeof(TEnum).Name}.{enumValue} should be inline (no ClosureConstants), but got {fe.ClosureConstants.Count}"); + fe.RootIndex = fe.Lambda>(idx); + var result = (TEnum)((System.Linq.Expressions.LambdaExpression)fe.ToExpression()).Compile().DynamicInvoke()!; + Asserts.AreEqual(enumValue, result, $"Round-trip failed for {typeof(TEnum).Name}.{enumValue}"); + } + + Check(ByteEnum.A); + Check(ByteEnum.B); + Check(SByteEnum.A); + Check(SByteEnum.B); + Check(ShortEnum.A); + Check(ShortEnum.B); + Check(UShortEnum.A); + Check(UShortEnum.B); + Check(IntEnum.A); + Check(IntEnum.B); + Check(UIntEnum.A); + Check(UIntEnum.B); + } } }