diff --git a/src/literal.h b/src/literal.h index adb8e2f171e..a1f77596820 100644 --- a/src/literal.h +++ b/src/literal.h @@ -19,6 +19,7 @@ #include #include +#include #include "support/bits.h" #include "support/hash.h" @@ -212,7 +213,7 @@ class Literal { } } - static Literal makeFromMemory(void* p, Type type); + static Literal makeFromMemory(const void* p, Type type); static Literal makeSignedMin(Type type) { switch (type.getBasic()) { @@ -784,15 +785,51 @@ std::ostream& operator<<(std::ostream& o, wasm::Literals literals); // A GC Struct, Array, or String is a set of values with a type saying how it // should be interpreted. struct GCData { - // The element or field values. - Literals values; + Type type; + + // The element or field values. Primitive numeric arrays use raw byte buffers + // (std::vector), while reference arrays, structs, strings, and other + // reference allocations use Literals. + std::variant, Literals> storage; // The descriptor, if it exists, or null. Literal desc; - GCData(Literals&& values, + GCData(Type type, + Literals&& values, + const Literal& desc = Literal::makeNull(HeapType::none)) + : type(type), storage(std::move(values)), desc(desc) {} + + GCData(Type type, + std::vector&& data, const Literal& desc = Literal::makeNull(HeapType::none)) - : values(std::move(values)), desc(desc) {} + : type(type), storage(std::move(data)), desc(desc) {} + + bool isRawBytes() const { + return std::holds_alternative>(storage); + } + + const std::vector& getRawBytes() const { + return std::get>(storage); + } + + std::vector& getRawBytes() { + return std::get>(storage); + } + + const Literals& getLiterals() const { return std::get(storage); } + + Literals& getLiterals() { return std::get(storage); } + + size_t getNumElements() const; + Literal getElement(size_t index, bool signed_ = false) const; + void setElement(size_t index, Literal value); + + // Writes a field value to the byte buffer at `dest`. + static void writeField(void* dest, const Field& field, Literal value); + // Reads a field value from the byte buffer at `src`. + static Literal + readField(const void* src, const Field& field, bool signed_ = false); }; inline bool Literal::hasExternPayload() const { @@ -800,12 +837,13 @@ inline bool Literal::hasExternPayload() const { return false; } assert(type.getHeapType().isMaybeShared(HeapType::ext)); - return gcData->values[0].type == Type::i32; + return !gcData->getLiterals().empty() && + gcData->getLiterals()[0].type == Type::i32; } inline int32_t Literal::getExternPayload() const { assert(hasExternPayload()); - return gcData->values[0].geti32(); + return gcData->getLiterals()[0].geti32(); } } // namespace wasm @@ -869,7 +907,7 @@ template<> struct hash { return digest; } if (a.type.isString()) { - auto& values = a.getGCData()->values; + auto& values = a.getGCData()->getLiterals(); wasm::rehash(digest, values.size()); for (auto c : values) { wasm::rehash(digest, c.getInteger()); diff --git a/src/passes/Precompute.cpp b/src/passes/Precompute.cpp index 8f550bd27eb..a1357e6475c 100644 --- a/src/passes/Precompute.cpp +++ b/src/passes/Precompute.cpp @@ -1103,7 +1103,7 @@ struct Precompute // string. bool isValidUTF16Literal(const Literal& value) { bool expectLowSurrogate = false; - for (auto& v : value.getGCData()->values) { + for (auto& v : value.getGCData()->getLiterals()) { auto c = v.getInteger(); if (c >= 0xDC00 && c <= 0xDFFF) { if (expectLowSurrogate) { diff --git a/src/tools/wasm-ctor-eval.cpp b/src/tools/wasm-ctor-eval.cpp index f5b05c98220..a23387e4903 100644 --- a/src/tools/wasm-ctor-eval.cpp +++ b/src/tools/wasm-ctor-eval.cpp @@ -912,7 +912,6 @@ struct CtorEvalExternalInterface : EvallingModuleRunner::ExternalInterface { } else { // This is the first usage of this data. Generate a struct.new / // array.new for it. - auto& values = data->values; std::vector args; // The initial values for this allocation may themselves be GC @@ -934,8 +933,8 @@ struct CtorEvalExternalInterface : EvallingModuleRunner::ExternalInterface { definingGlobals[data] = DefiningGlobalInfo{definingGlobalName, type}; } - for (auto& value : values) { - auto* serialized = getSerialization(value); + for (size_t i = 0; i < data->getNumElements(); i++) { + auto* serialized = getSerialization(data->getElement(i)); if (!serialized) { return nullptr; } diff --git a/src/wasm-builder.h b/src/wasm-builder.h index 3b4cbb2d7ef..a5644ebf763 100644 --- a/src/wasm-builder.h +++ b/src/wasm-builder.h @@ -1504,7 +1504,7 @@ class Builder { // The string is already WTF-16, but we need to convert from `Literals` to // actual string. std::stringstream wtf16; - for (auto c : value.getGCData()->values) { + for (auto c : value.getGCData()->getLiterals()) { auto u = c.getInteger(); assert(u < 0x10000); wtf16 << uint8_t(u & 0xFF); diff --git a/src/wasm-interpreter.h b/src/wasm-interpreter.h index 08a1405ba22..e0fc20c7b23 100644 --- a/src/wasm-interpreter.h +++ b/src/wasm-interpreter.h @@ -370,10 +370,12 @@ class ExpressionRunner : public OverriddenVisitor { // this function in LSan. // // This consumes the input |data| entirely. - Literal makeGCData(Literals&& data, + template + Literal makeGCData(T&& data, Type type, Literal desc = Literal::makeNull(HeapType::none)) { - auto allocation = std::make_shared(std::move(data), desc); + auto allocation = + std::make_shared(type, std::forward(data), desc); #if __has_feature(leak_sanitizer) || __has_feature(address_sanitizer) // GC data with cycles will leak, since shared_ptrs do not handle cycles. // Binaryen is generally not used in long-running programs so we just ignore @@ -393,20 +395,6 @@ class ExpressionRunner : public OverriddenVisitor { return Literal(allocation); } - template - void writeBytes(T value, int numBytes, size_t index, Literals& values) { - if constexpr (std::is_same_v>) { - for (int i = 0; i < numBytes; ++i) { - values[index + i] = Literal(static_cast(value[i])); - } - } else { - for (int i = 0; i < numBytes; ++i) { - values[index + i] = - Literal(static_cast((value >> (i * 8)) & 0xff)); - } - } - } - static Literal applyRMW(AtomicRMWOp op, Literal lhs, Literal rhs) { switch (op) { case RMWAdd: @@ -2276,7 +2264,8 @@ class ExpressionRunner : public OverriddenVisitor { trap("null ref"); } auto field = curr->ref->type.getHeapType().getStruct().fields[curr->index]; - return extendForPacking(data->values[curr->index], field, curr->signed_); + return extendForPacking( + data->getLiterals()[curr->index], field, curr->signed_); } Flow visitStructSet(StructSet* curr) { VISIT(ref, curr->ref) @@ -2286,7 +2275,7 @@ class ExpressionRunner : public OverriddenVisitor { trap("null ref"); } auto field = curr->ref->type.getHeapType().getStruct().fields[curr->index]; - data->values[curr->index] = + data->getLiterals()[curr->index] = truncateForPacking(value.getSingleValue(), field); return Flow(); } @@ -2298,7 +2287,7 @@ class ExpressionRunner : public OverriddenVisitor { if (!data) { trap("null ref"); } - auto& field = data->values[curr->index]; + auto& field = data->getLiterals()[curr->index]; auto oldVal = field; field = applyRMW(curr->op, oldVal, value.getSingleValue()); return oldVal; @@ -2312,7 +2301,7 @@ class ExpressionRunner : public OverriddenVisitor { if (!data) { trap("null ref"); } - auto& field = data->values[curr->index]; + auto& field = data->getLiterals()[curr->index]; auto oldVal = field; if (field == expected.getSingleValue()) { field = replacement.getSingleValue(); @@ -2337,7 +2326,7 @@ class ExpressionRunner : public OverriddenVisitor { if (!waitqueue.getSingleValue().getGCData()) { trap("null ref"); } - auto& field = data->values[curr->index]; + auto& field = data->getLiterals()[curr->index]; if (field != expected.getSingleValue()) { return Literal(int32_t{1}); // not equal } @@ -2354,7 +2343,7 @@ class ExpressionRunner : public OverriddenVisitor { } Flow visitWaitqueueNew(WaitqueueNew* curr) { - return self()->makeGCData({}, + return self()->makeGCData(Literals{}, Type(HeapTypes::sharedWaitqueue, NonNullable)); } @@ -2395,17 +2384,21 @@ class ExpressionRunner : public OverriddenVisitor { if (num >= DataLimit) { hostLimit("allocation failure"); } - Literals data(num); - if (curr->isWithDefault()) { - auto zero = Literal::makeZero(element.type); + if (element.type.isRef()) { + Literals data(num); + auto val = curr->isWithDefault() ? Literal::makeZero(element.type) + : init.getSingleValue(); for (Index i = 0; i < num; i++) { - data[i] = zero; + data[i] = val; } - } else { - auto field = curr->type.getHeapType().getArray().element; - auto value = truncateForPacking(init.getSingleValue(), field); + return makeGCData(std::move(data), curr->type); + } + size_t elemBytes = element.getByteSize(); + std::vector data(num * elemBytes, 0); + if (!curr->isWithDefault()) { + Literal val = init.getSingleValue(); for (Index i = 0; i < num; i++) { - data[i] = value; + GCData::writeField(&data[i * elemBytes], element, val); } } return makeGCData(std::move(data), curr->type); @@ -2427,10 +2420,19 @@ class ExpressionRunner : public OverriddenVisitor { } auto heapType = curr->type.getHeapType(); auto field = heapType.getArray().element; - Literals data(num); + if (field.type.isRef()) { + Literals data(num); + for (Index i = 0; i < num; i++) { + VISIT(value, curr->values[i]) + data[i] = value.getSingleValue(); + } + return makeGCData(std::move(data), curr->type); + } + size_t elemBytes = field.getByteSize(); + std::vector data(num * elemBytes); for (Index i = 0; i < num; i++) { VISIT(value, curr->values[i]) - data[i] = truncateForPacking(value.getSingleValue(), field); + GCData::writeField(&data[i * elemBytes], field, value.getSingleValue()); } return makeGCData(std::move(data), curr->type); } @@ -2442,11 +2444,10 @@ class ExpressionRunner : public OverriddenVisitor { trap("null ref"); } Index i = index.getSingleValue().geti32(); - if (i >= data->values.size()) { + if (i >= data->getNumElements()) { trap("array oob"); } - auto field = curr->ref->type.getHeapType().getArray().element; - return extendForPacking(data->values[i], field, curr->signed_); + return data->getElement(i, curr->signed_); } Flow visitArraySet(ArraySet* curr) { VISIT(ref, curr->ref) @@ -2457,11 +2458,10 @@ class ExpressionRunner : public OverriddenVisitor { trap("null ref"); } Index i = index.getSingleValue().geti32(); - if (i >= data->values.size()) { + if (i >= data->getNumElements()) { trap("array oob"); } - auto field = curr->ref->type.getHeapType().getArray().element; - data->values[i] = truncateForPacking(value.getSingleValue(), field); + data->setElement(i, value.getSingleValue()); return Flow(); } Flow visitArrayLoad(ArrayLoad* curr) { @@ -2472,13 +2472,14 @@ class ExpressionRunner : public OverriddenVisitor { trap("null ref"); } Index i = index.getSingleValue().geti32(); - size_t size = data->values.size(); + size_t size = data->getRawBytes().size(); if (i >= size || curr->bytes > (size - i)) { trap("array oob"); } + const uint8_t* p = &data->getRawBytes()[i]; uint64_t val = 0; for (unsigned b = 0; b < curr->bytes; ++b) { - val |= static_cast(data->values[i + b].geti32()) << (b * 8); + val |= static_cast(p[b]) << (b * 8); } switch (curr->type.getBasic()) { case Type::i32: { @@ -2526,40 +2527,15 @@ class ExpressionRunner : public OverriddenVisitor { } Index i = index.getSingleValue().geti32(); - size_t size = data->values.size(); + size_t size = data->getRawBytes().size(); // Use subtraction to avoid overflow. if (i >= size || curr->bytes > (size - i)) { trap("array oob"); } - switch (curr->value->type.getBasic()) { - case Type::i32: - writeBytes( - value.getSingleValue().geti32(), curr->bytes, i, data->values); - break; - case Type::i64: - writeBytes( - value.getSingleValue().geti64(), curr->bytes, i, data->values); - break; - case Type::f32: - writeBytes(value.getSingleValue().reinterpreti32(), - curr->bytes, - i, - data->values); - break; - case Type::f64: - writeBytes(value.getSingleValue().reinterpreti64(), - curr->bytes, - i, - data->values); - break; - case Type::v128: - writeBytes( - value.getSingleValue().getv128(), curr->bytes, i, data->values); - break; - case Type::none: - case Type::unreachable: - WASM_UNREACHABLE("unimp basic type"); - } + uint8_t* p = &data->getRawBytes()[i]; + uint8_t buf[16]; + value.getSingleValue().getBits(buf); + memcpy(p, buf, curr->bytes); return Flow(); } Flow visitArrayLen(ArrayLen* curr) { @@ -2568,7 +2544,7 @@ class ExpressionRunner : public OverriddenVisitor { if (!data) { trap("null ref"); } - return Literal(int32_t(data->values.size())); + return Literal(int32_t(data->getNumElements())); } Flow visitArrayCopy(ArrayCopy* curr) { VISIT(destRef, curr->destRef) @@ -2587,19 +2563,29 @@ class ExpressionRunner : public OverriddenVisitor { size_t destVal = destIndex.getSingleValue().getUnsigned(); size_t srcVal = srcIndex.getSingleValue().getUnsigned(); size_t lengthVal = length.getSingleValue().getUnsigned(); - if (destVal + lengthVal > destData->values.size()) { + if (destVal + lengthVal > destData->getNumElements()) { trap("oob"); } - if (srcVal + lengthVal > srcData->values.size()) { + if (srcVal + lengthVal > srcData->getNumElements()) { trap("oob"); } + if (destData->isRawBytes() && srcData->isRawBytes()) { + if (lengthVal > 0) { + auto elemSize = + destData->type.getHeapType().getArray().element.getByteSize(); + memmove(&destData->getRawBytes()[destVal * elemSize], + &srcData->getRawBytes()[srcVal * elemSize], + lengthVal * elemSize); + } + return Flow(); + } std::vector copied; copied.resize(lengthVal); for (size_t i = 0; i < lengthVal; i++) { - copied[i] = srcData->values[srcVal + i]; + copied[i] = srcData->getElement(srcVal + i); } for (size_t i = 0; i < lengthVal; i++) { - destData->values[destVal + i] = copied[i]; + destData->setElement(destVal + i, copied[i]); } return Flow(); } @@ -2616,16 +2602,13 @@ class ExpressionRunner : public OverriddenVisitor { Literal fillVal = value.getSingleValue(); size_t sizeVal = size.getSingleValue().getUnsigned(); - auto field = curr->ref->type.getHeapType().getArray().element; - fillVal = truncateForPacking(fillVal, field); - - size_t arraySize = data->values.size(); + size_t arraySize = data->getNumElements(); if (indexVal > arraySize || sizeVal > arraySize || indexVal + sizeVal > arraySize || indexVal + sizeVal < indexVal) { trap("out of bounds array access in array.fill"); } for (size_t i = 0; i < sizeVal; ++i) { - data->values[indexVal + i] = fillVal; + data->setElement(indexVal + i, fillVal); } return {}; } @@ -2640,12 +2623,12 @@ class ExpressionRunner : public OverriddenVisitor { trap("null ref"); } size_t indexVal = index.getSingleValue().getUnsigned(); - if (indexVal >= data->values.size()) { + if (indexVal >= data->getNumElements()) { trap("array oob"); } - auto& field = data->values[indexVal]; - auto oldVal = field; - field = applyRMW(curr->op, oldVal, value.getSingleValue()); + auto oldVal = data->getElement(indexVal); + data->setElement(indexVal, + applyRMW(curr->op, oldVal, value.getSingleValue())); return oldVal; } @@ -2659,13 +2642,12 @@ class ExpressionRunner : public OverriddenVisitor { trap("null ref"); } size_t indexVal = index.getSingleValue().getUnsigned(); - if (indexVal >= data->values.size()) { + if (indexVal >= data->getNumElements()) { trap("array oob"); } - auto& field = data->values[indexVal]; - auto oldVal = field; - if (field == expected.getSingleValue()) { - field = replacement.getSingleValue(); + auto oldVal = data->getElement(indexVal); + if (oldVal == expected.getSingleValue()) { + data->setElement(indexVal, replacement.getSingleValue()); } return oldVal; } @@ -2695,18 +2677,17 @@ class ExpressionRunner : public OverriddenVisitor { if (!ptrData) { trap("null ref"); } - const auto& ptrDataValues = ptrData->values; + size_t arrayLen = ptrData->getNumElements(); size_t startVal = start.getSingleValue().getUnsigned(); size_t endVal = end.getSingleValue().getUnsigned(); - if (startVal > ptrDataValues.size() || endVal > ptrDataValues.size() || - endVal < startVal) { + if (startVal > arrayLen || endVal > arrayLen || endVal < startVal) { trap("array oob"); } Literals contents; if (endVal > startVal) { contents.reserve(endVal - startVal); for (size_t i = startVal; i < endVal; i++) { - contents.push_back(ptrDataValues[i]); + contents.push_back(ptrData->getElement(i)); } } return makeGCData(std::move(contents), curr->type); @@ -2743,7 +2724,7 @@ class ExpressionRunner : public OverriddenVisitor { trap("null ref"); } - return Literal(int32_t(data->values.size())); + return Literal(int32_t(data->getLiterals().size())); } Flow visitStringConcat(StringConcat* curr) { VISIT(flow, curr->left) @@ -2756,17 +2737,19 @@ class ExpressionRunner : public OverriddenVisitor { trap("null ref"); } - auto totalSize = leftData->values.size() + rightData->values.size(); + auto totalSize = + leftData->getLiterals().size() + rightData->getLiterals().size(); if (totalSize >= DataLimit) { hostLimit("allocation failure"); } Literals contents; - contents.reserve(leftData->values.size() + rightData->values.size()); - for (Literal& l : leftData->values) { + contents.reserve(leftData->getLiterals().size() + + rightData->getLiterals().size()); + for (Literal& l : leftData->getLiterals()) { contents.push_back(l); } - for (Literal& l : rightData->values) { + for (Literal& l : rightData->getLiterals()) { contents.push_back(l); } @@ -2788,19 +2771,19 @@ class ExpressionRunner : public OverriddenVisitor { trap("null ref"); } auto startVal = start.getSingleValue().getUnsigned(); - auto& strValues = strData->values; - auto& arrayValues = arrayData->values; + auto& strValues = strData->getLiterals(); + size_t arrayLen = arrayData->getNumElements(); size_t end; if (std::ckd_add(&end, startVal, strValues.size()) || - end > arrayValues.size()) { + end > arrayLen) { trap("oob"); } for (Index i = 0; i < strValues.size(); i++) { - arrayValues[startVal + i] = strValues[i]; + arrayData->setElement(startVal + i, strValues[i]); } - return Literal(int32_t(strData->values.size())); + return Literal(int32_t(strValues.size())); } Flow visitStringEq(StringEq* curr) { VISIT(flow, curr->left) @@ -2813,17 +2796,17 @@ class ExpressionRunner : public OverriddenVisitor { switch (curr->op) { case StringEqEqual: { // They are equal if both are null, or both are non-null and equal. - result = - (!leftData && !rightData) || - (leftData && rightData && leftData->values == rightData->values); + result = (!leftData && !rightData) || + (leftData && rightData && + leftData->getLiterals() == rightData->getLiterals()); break; } case StringEqCompare: { if (!leftData || !rightData) { trap("null ref"); } - auto& leftValues = leftData->values; - auto& rightValues = rightData->values; + auto& leftValues = leftData->getLiterals(); + auto& rightValues = rightData->getLiterals(); Index i = 0; while (1) { if (i == leftValues.size() && i == rightValues.size()) { @@ -2873,7 +2856,7 @@ class ExpressionRunner : public OverriddenVisitor { if (!data) { trap("null ref"); } - auto& values = data->values; + auto& values = data->getLiterals(); Index i = pos.getSingleValue().geti32(); if (i >= values.size()) { trap("string oob"); @@ -2890,7 +2873,7 @@ class ExpressionRunner : public OverriddenVisitor { if (!refData) { trap("null ref"); } - auto& refValues = refData->values; + auto& refValues = refData->getLiterals(); auto startVal = start.getSingleValue().getUnsigned(); auto endVal = end.getSingleValue().getUnsigned(); endVal = std::min(endVal, refValues.size()); @@ -2950,22 +2933,6 @@ class ExpressionRunner : public OverriddenVisitor { } return value; } - - Literal makeFromMemory(void* p, Field field) { - switch (field.packedType) { - case Field::NotPacked: - return Literal::makeFromMemory(p, field.type); - case Field::i8: { - return truncateForPacking(Literal(int32_t(Bits::readLE(p))), - field); - } - case Field::i16: { - return truncateForPacking(Literal(int32_t(Bits::readLE(p))), - field); - } - } - WASM_UNREACHABLE("unexpected type"); - } }; // Execute a suspected constant expression (precompute and C-API). @@ -4736,7 +4703,6 @@ class ModuleRunnerBase : public ExpressionRunner { auto heapType = curr->type.getHeapType(); const auto& element = heapType.getArray().element; - Literals contents; const auto& seg = *wasm.getDataSegment(curr->segment); auto elemBytes = element.getByteSize(); @@ -4751,10 +4717,9 @@ class ModuleRunnerBase : public ExpressionRunner { if (droppedDataSegments.contains(curr->segment) && end > 0) { trap("dropped segment access in array.new_data"); } - contents.reserve(size); - for (Index i = offset; i < end; i += elemBytes) { - auto addr = (void*)&seg.data[i]; - contents.push_back(this->makeFromMemory(addr, element)); + std::vector contents(size * elemBytes); + if (size > 0) { + memcpy(contents.data(), &seg.data[offset], size * elemBytes); } #pragma GCC diagnostic pop @@ -4798,7 +4763,7 @@ class ModuleRunnerBase : public ExpressionRunner { size_t offsetVal = offset.getSingleValue().getUnsigned(); size_t sizeVal = size.getSingleValue().getUnsigned(); - size_t arraySize = data->values.size(); + size_t arraySize = data->getNumElements(); if ((uint64_t)indexVal + sizeVal > arraySize) { trap("out of bounds array access in array.init"); } @@ -4816,9 +4781,11 @@ class ModuleRunnerBase : public ExpressionRunner { droppedDataSegments.contains(curr->segment)) { trap("out of bounds segment access in array.init_data"); } - for (size_t i = 0; i < sizeVal; i++) { - void* addr = (void*)&seg->data[offsetVal + i * elemSize]; - data->values[indexVal + i] = this->makeFromMemory(addr, elem); + assert(data->isRawBytes()); + if (sizeVal > 0) { + memcpy(&data->getRawBytes()[indexVal * elemSize], + &seg->data[offsetVal], + sizeVal * elemSize); } return {}; } @@ -4835,7 +4802,7 @@ class ModuleRunnerBase : public ExpressionRunner { size_t offsetVal = offset.getSingleValue().getUnsigned(); size_t sizeVal = size.getSingleValue().getUnsigned(); - size_t arraySize = data->values.size(); + size_t arraySize = data->getNumElements(); if ((uint64_t)indexVal + sizeVal > arraySize) { trap("out of bounds array access in array.init"); } @@ -4855,7 +4822,8 @@ class ModuleRunnerBase : public ExpressionRunner { // of references in the table! ArrayNew suffers the same problem. // Fixing it will require changing how we represent segments, at least // in the interpreter. - data->values[indexVal + i] = self()->visit(seg->data[i]).getSingleValue(); + data->setElement(indexVal + i, + self()->visit(seg->data[i]).getSingleValue()); } return {}; } diff --git a/src/wasm/literal.cpp b/src/wasm/literal.cpp index a037fef99e6..58ff8346877 100644 --- a/src/wasm/literal.cpp +++ b/src/wasm/literal.cpp @@ -67,7 +67,7 @@ Literal::Literal(Type type) : type(type) { if (type.isRef() && type.getHeapType().isMaybeShared(HeapType::ext)) { assert(type.isNonNullable()); new (&gcData) std::shared_ptr( - std::make_shared(Literals{Literal(int32_t{0})})); + std::make_shared(type, Literals{Literal(int32_t{0})})); return; } @@ -100,7 +100,9 @@ Literal Literal::makeFunc(Name func, Module& wasm) { Literal Literal::makeExtern(int32_t payload, Shareability share) { auto ext = HeapTypes::ext.getBasic(share); - return Literal(std::make_shared(Literals{Literal(payload)}), ext); + return Literal(std::make_shared(Type(ext, NonNullable), + Literals{Literal(payload)}), + ext); } Literal::Literal(std::shared_ptr gcData, HeapType type) @@ -137,7 +139,7 @@ Literal::Literal(std::string_view string) int32_t u = uint8_t(string[i]) | (uint8_t(string[i + 1]) << 8); contents.push_back(Literal(u)); } - gcData = std::make_shared(std::move(contents)); + gcData = std::make_shared(type, std::move(contents)); } Literal::Literal(const Literal& other) : type(other.type) { @@ -314,7 +316,7 @@ Literal Literal::makeNegOne(Type type) { return makeFromInt32(-1, type); } -Literal Literal::makeFromMemory(void* p, Type type) { +Literal Literal::makeFromMemory(const void* p, Type type) { assert(type.isNumber()); switch (type.getBasic()) { case Type::i32: { @@ -499,7 +501,7 @@ bool Literal::operator==(const Literal& other) const { return *funcData == *other.funcData; } if (type.isString()) { - return gcData->values == other.gcData->values; + return gcData->getLiterals() == other.gcData->getLiterals(); } if (type.isData()) { return gcData == other.gcData; @@ -775,8 +777,8 @@ std::ostream& operator<<(std::ostream& o, Literal literal) { break; case HeapType::any: { auto data = literal.getGCData(); - assert(data->values.size() == 1); - o << "internalized " << literal.getGCData()->values[0]; + assert(data->getLiterals().size() == 1); + o << "internalized " << data->getLiterals()[0]; break; } case HeapType::ext: { @@ -806,7 +808,7 @@ std::ostream& operator<<(std::ostream& o, Literal literal) { o << "string("; // Convert WTF-16 literals to WTF-16 string. std::stringstream wtf16; - for (auto c : data->values) { + for (auto c : data->getLiterals()) { auto u = c.getInteger(); assert(u < 0x10000); wtf16 << uint8_t(u & 0xFF); @@ -844,9 +846,15 @@ std::ostream& operator<<(std::ostream& o, Literal literal) { assert(literal.isData()); auto data = literal.getGCData(); assert(data); - o << "[ref " << literal.type.getHeapType() << ' ' << data->values; + o << "[ref " << literal.type.getHeapType() << ' '; + for (size_t i = 0; i < data->getNumElements(); i++) { + if (i > 0) { + o << ' '; + } + o << data->getElement(i); + } if (!data->desc.isNull()) { - if (!data->values.empty()) { + if (data->getNumElements() > 0) { o << ", "; } o << "desc=" << data->desc; @@ -3102,12 +3110,13 @@ Literal Literal::externalize() const { } if (heapType.isMaybeShared(HeapType::any)) { // This is an internalized externref or string; just unwrap it. - assert(gcData->values.size() == 1); - return gcData->values[0]; + assert(gcData->getLiterals().size() == 1); + return gcData->getLiterals()[0]; } // This is an internal reference. Wrap it. auto ext = HeapTypes::ext.getBasic(heapType.getShared()); - return Literal(std::make_shared(Literals{*this}), ext); + return Literal( + std::make_shared(Type(ext, NonNullable), Literals{*this}), ext); } Literal Literal::internalize() const { @@ -3121,11 +3130,12 @@ Literal Literal::internalize() const { if (isString() || hasExternPayload()) { // This is an external reference. Wrap it. auto any = HeapTypes::any.getBasic(heapType.getShared()); - return Literal(std::make_shared(Literals{*this}), any); + return Literal( + std::make_shared(Type(any, NonNullable), Literals{*this}), any); } // This is an externalized internal reference; just unwrap it. - assert(gcData->values.size() == 1); - return gcData->values[0]; + assert(gcData->getLiterals().size() == 1); + return gcData->getLiterals()[0]; } Literal Literal::unwrap() const { @@ -3148,7 +3158,7 @@ Literal Literal::getJSPrototype() const { assert(type.isRef()); if (auto desc = type.getHeapType().getDescriptorType(); desc && JSUtils::hasPossibleJSPrototypeField(*desc)) { - auto proto = gcData->desc.getGCData()->values[0].unwrap(); + auto proto = gcData->desc.getGCData()->getLiterals()[0].unwrap(); // Strings and numbers are not valid prototypes, so they appear as null. // Externref nulls are also converted to nullref. auto protoType = proto.type.getHeapType(); @@ -3161,4 +3171,72 @@ Literal Literal::getJSPrototype() const { return Literal::makeNull(HeapType::none); } +size_t GCData::getNumElements() const { + if (isRawBytes()) { + auto field = type.getHeapType().getArray().element; + return getRawBytes().size() / field.getByteSize(); + } + return getLiterals().size(); +} + +Literal GCData::getElement(size_t index, bool signed_) const { + if (isRawBytes()) { + auto field = type.getHeapType().getArray().element; + size_t elemSize = field.getByteSize(); + assert((index + 1) * elemSize <= getRawBytes().size()); + return readField(&getRawBytes()[index * elemSize], field, signed_); + } + return getLiterals()[index]; +} + +void GCData::setElement(size_t index, Literal value) { + if (isRawBytes()) { + auto field = type.getHeapType().getArray().element; + size_t elemSize = field.getByteSize(); + assert((index + 1) * elemSize <= getRawBytes().size()); + writeField(&getRawBytes()[index * elemSize], field, value); + return; + } + getLiterals()[index] = value; +} + +void GCData::writeField(void* dest, const Field& field, Literal value) { + if (field.isPacked()) { + assert(field.type == Type::i32); + int32_t c = value.geti32(); + if (field.packedType == Field::i8) { + Bits::writeLE(static_cast(c), dest); + } else if (field.packedType == Field::i16) { + Bits::writeLE(static_cast(c), dest); + } else { + WASM_UNREACHABLE("invalid packed type"); + } + return; + } + + uint8_t buf[16]; + assert(field.getByteSize() <= sizeof(buf)); + value.getBits(buf); + memcpy(dest, buf, field.getByteSize()); +} + +Literal GCData::readField(const void* src, const Field& field, bool signed_) { + if (field.isPacked()) { + assert(field.type == Type::i32); + if (field.packedType == Field::i8) { + int8_t val = Bits::readLE(src); + return Literal(signed_ ? int32_t(val) + : int32_t(static_cast(val))); + } else if (field.packedType == Field::i16) { + int16_t val = Bits::readLE(src); + return Literal(signed_ ? int32_t(val) + : int32_t(static_cast(val))); + } else { + WASM_UNREACHABLE("invalid packed type"); + } + } + + return Literal::makeFromMemory(src, field.type); +} + } // namespace wasm