From f4e42c02831a61604be5b8cb106922c933b9577b Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Thu, 9 Jul 2026 16:38:33 -0700 Subject: [PATCH 01/17] Start spill pointers pass --- gen/optimizer.cpp | 18 +++ gen/passes/Passes.h | 2 + gen/passes/WasmPointersSpill.cpp | 209 +++++++++++++++++++++++++++++++ gen/passes/WasmPointersSpill.h | 25 ++++ 4 files changed, 254 insertions(+) create mode 100644 gen/passes/WasmPointersSpill.cpp create mode 100644 gen/passes/WasmPointersSpill.h diff --git a/gen/optimizer.cpp b/gen/optimizer.cpp index 849d32b3d58..60041383578 100644 --- a/gen/optimizer.cpp +++ b/gen/optimizer.cpp @@ -26,6 +26,7 @@ #include "gen/passes/GarbageCollect2Stack.h" #include "gen/passes/StripExternals.h" #include "gen/passes/SimplifyDRuntimeCalls.h" +#include "gen/passes/WasmPointersSpill.h" #include "gen/passes/Passes.h" #ifndef IN_JITRT @@ -333,6 +334,19 @@ static void addGarbageCollect2StackPass(ModulePassManager &mpm, } } +static void addWasmPointersSpillPass(ModulePassManager &mpm, + OptimizationLevel level +#if LLVM_VERSION_MAJOR >= 20 + , + ThinOrFullLTOPhase +#endif +) { + mpm.addPass(createModuleToFunctionPassAdaptor(WasmPointersSpillPass())); + if (verifyEach) { + mpm.addPass(VerifierPass()); + } +} + #ifndef IN_JITRT static std::optional getPGOOptions() { // FIXME: Do we have these anywhere? @@ -511,6 +525,10 @@ void runOptimizationPasses(llvm::Module *M, llvm::TargetMachine *TM) { pb.registerOptimizerLastEPCallback(addStripExternalsPass); + if (global.params.targetTriple->isWasm()) { + pb.registerOptimizerLastEPCallback(addWasmPointersSpillPass); + } + #ifndef IN_JITRT registerAllPluginsWithPassBuilder(pb); #endif diff --git a/gen/passes/Passes.h b/gen/passes/Passes.h index a112714dc97..e1b34cda1b5 100644 --- a/gen/passes/Passes.h +++ b/gen/passes/Passes.h @@ -23,6 +23,8 @@ llvm::FunctionPass *createSimplifyDRuntimeCalls(); llvm::FunctionPass *createGarbageCollect2Stack(); +llvm::FunctionPass *createWasmPointersSpillPass(); + llvm::ModulePass *createStripExternalsPass(); llvm::ModulePass *createDLLImportRelocationPass(); diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp new file mode 100644 index 00000000000..8b581991fde --- /dev/null +++ b/gen/passes/WasmPointersSpill.cpp @@ -0,0 +1,209 @@ +//===-- WasmPointersSpill.cpp - Spill pointer onto stack for Wasm ---------===// +// +// LDC – the LLVM D compiler +// +// This file is distributed under the University of Illinois Open Source +// License. See the LICENSE file for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements a pass to scan vregs and dump potential GC pointers that +// are live across calls (which may trigger GC) back to the stack so they can +// be scanned in Wasm environments. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "wasm-ptrs-spill" + +#include "gen/passes/Passes.h" +#include "gen/passes/WasmPointersSpill.h" +#include "gen/llvmhelpers.h" +#include "gen/tollvm.h" +#include "gen/runtime.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Module.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/Pass.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/ADT/PostOrderIterator.h" + + +using namespace llvm; + +class LLVM_LIBRARY_VISIBILITY WasmPointersSpillLegacyPass : public FunctionPass { + WasmPointersSpill pass; + +public: + static char ID; // Pass identification + WasmPointersSpillLegacyPass() : FunctionPass(ID) {} + + bool runOnFunction(Function &F) override { + return pass.run(F); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + //AU.addRequired(); + } +}; +char WasmPointersSpillLegacyPass::ID = 0; + +static RegisterPass + X("wasm-ptrs-spill", "Spill ptr vreg to `alloca` for Wasm"); + +// Public interface to the pass. +FunctionPass *createWasmPointersSpill() { + return new WasmPointersSpillLegacyPass(); +} + +static bool TypeContainsPointers(LLType *ty) { + switch (ty->getTypeID()) { + case LLType::PointerTyID: + return true; + + case LLType::ArrayTyID: + return TypeContainsPointers(ty->getArrayElementType()); + + case LLType::StructTyID: { + unsigned NumElements = ty->getStructNumElements(); + for (unsigned I = 0; I < NumElements; ++I) { + if (TypeContainsPointers(ty->getStructElementType(I))) { + return true; + } + } + return false; + } + + default: + return false; + } +} + +bool WasmPointersSpill::run(Function &F) { + IRBuilder<> Builder(F.getContext()); +dbgs() << F.getName() << "\n"; + bool Changed = false; + + /*for (auto &BB : F) { + for (Instruction &I : BB) { + if (I.getType()->isPointerTy() || TypeContainsPointers(I.getType())) SpillCandidates.push_back(&I); + } + }*/ + + auto *entryBB = &F.getEntryBlock(); + + DenseMap InstIdx; + uint64_t NextInstIdx = 0; + for (auto &BB : F) { + for (Instruction &I : BB) { + if ( + !I.getType()->isVoidTy() // we only care about vreg/values + && !isa(I) // the pointer returned by `alloca` isn't in the GC heap + ) { + InstIdx[&I] = NextInstIdx++; + } + if (NextInstIdx >= (uint64_t(1) << 32)) reportFatalInternalError("More vregs than can fit in 32-bits!"); + } + } + + size_t NumVRegs = NextInstIdx; + + //SmallSetVector Worklist; + //DenseMap LiveIn; + //DenseMap LiveOut; + DenseMap Defs; + DenseMap Uses; + + + ReversePostOrderTraversal RPOT(&F); + for (BasicBlock* BB : make_range(RPOT.begin(), RPOT.end())) { + //LiveIn[BB].resize(NumVRegs); + //LiveOut[BB].resize(NumVRegs); + Defs[BB].resize(NumVRegs); + Uses[BB].resize(NumVRegs); + + auto &BBUses = Uses[BB]; + auto &BBDefs = Defs[BB]; + + BitVector TmpUses(NumVRegs); + + for (Instruction &I : make_range(BB->rbegin(), BB->rend())) { + for (Value *Op : I.operands()) { + if (auto *OpInst = dyn_cast(Op)) { + if (InstIdx.contains(OpInst)) TmpUses.set(InstIdx[OpInst]); + } + } + + if (InstIdx.contains(&I)) { + BBDefs.set(InstIdx[&I]); + TmpUses.reset(InstIdx[&I]); + } + + if (isa(&I)) BBUses |= TmpUses; + } + //Worklist.insert(BB); + } + + /* + while (!Worklist.empty()) { + auto *BB = Worklist.back(); + Worklist.pop_back(); + + BitVector NewLiveOut(NumVRegs); + + for (BasicBlock *Succ : successors(BB)) { + NewLiveOut |= LiveIn[Succ]; + } + + LiveOut[BB] = NewLiveOut; + + BitVector NewLiveIn = NewLiveOut; + NewLiveIn |= Uses[BB]; + NewLiveIn.reset(Defs[BB]); + + if (NewLiveIn != LiveIn[BB]) { + LiveIn[BB] = NewLiveIn; + + for (BasicBlock *Pred : predecessors(BB)) { + Worklist.insert(Pred); + } + } + } + */ + + BitVector NeedsSpill(NumVRegs); + for (auto &BB : F) { + NeedsSpill |= Uses[&BB]; + } + + + auto *allocaBB = BasicBlock::Create(F.getContext(), "stackSpillAllocas"); + allocaBB->insertInto(&F, entryBB); + + SmallVector InstToSpill = to_vector(InstIdx.keys()); + std::sort(InstToSpill.begin(), InstToSpill.end(), [&](Instruction *a, Instruction *b) + { + return InstIdx[a] < InstIdx[b]; + } + ); + + for (Instruction *I : InstToSpill) { + if (!NeedsSpill[InstIdx[I]]) continue; + + auto *ai = new AllocaInst(I->getType(), 0, "stackSpill." + (I->hasName() ? I->getName() : "unnamedVreg"), allocaBB); + + auto *store = new StoreInst(I, ai, true, ai->getAlign(), nullptr); + store->insertAfter(I); + } + + UncondBrInst::Create(entryBB, allocaBB); + + return Changed; +} diff --git a/gen/passes/WasmPointersSpill.h b/gen/passes/WasmPointersSpill.h new file mode 100644 index 00000000000..0dcdda2457d --- /dev/null +++ b/gen/passes/WasmPointersSpill.h @@ -0,0 +1,25 @@ +#pragma once +#include "gen/llvm.h" +#include "gen/passes/Passes.h" + +struct LLVM_LIBRARY_VISIBILITY WasmPointersSpill { + bool run(llvm::Function &F); + + static llvm::StringRef getPassName() { return "WasmPointersSpill"; } +}; + +struct LLVM_LIBRARY_VISIBILITY WasmPointersSpillPass : public llvm::PassInfoMixin { + llvm::PreservedAnalyses run(llvm::Function &F, llvm::FunctionAnalysisManager &fam) { + if (pass.run(F)) { + return llvm::PreservedAnalyses::none(); + } + else { + return llvm::PreservedAnalyses::all(); + } + } + static llvm::StringRef name() { return WasmPointersSpill::getPassName(); } + + WasmPointersSpillPass() : pass() {} +private: + WasmPointersSpill pass; +}; From 8a16c688a89c26777df59730c5eef1742afe7889 Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Fri, 10 Jul 2026 13:28:35 -0700 Subject: [PATCH 02/17] Make it work for the most part. --- gen/passes/WasmPointersSpill.cpp | 198 ++++++++++++++++++++++--------- 1 file changed, 142 insertions(+), 56 deletions(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index 8b581991fde..f75d935809a 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -15,6 +15,7 @@ #define DEBUG_TYPE "wasm-ptrs-spill" +#include #include "gen/passes/Passes.h" #include "gen/passes/WasmPointersSpill.h" #include "gen/llvmhelpers.h" @@ -50,7 +51,6 @@ class LLVM_LIBRARY_VISIBILITY WasmPointersSpillLegacyPass : public FunctionPass } void getAnalysisUsage(AnalysisUsage &AU) const override { - //AU.addRequired(); } }; char WasmPointersSpillLegacyPass::ID = 0; @@ -81,129 +81,215 @@ static bool TypeContainsPointers(LLType *ty) { return false; } + case LLType::FixedVectorTyID: + return TypeContainsPointers(cast(ty)->getElementType()); + default: return false; } } bool WasmPointersSpill::run(Function &F) { - IRBuilder<> Builder(F.getContext()); -dbgs() << F.getName() << "\n"; + F.renumberBlocks(); + bool Changed = false; - /*for (auto &BB : F) { - for (Instruction &I : BB) { - if (I.getType()->isPointerTy() || TypeContainsPointers(I.getType())) SpillCandidates.push_back(&I); + // for SetVector determinstic iteration + SmallSetVector PotentialPointers; + + { + SmallPtrSet WorklistSeen; + SmallSetVector Worklist; + + for (auto &BB : F) { + for (Instruction &I : BB) { + if (TypeContainsPointers(I.getType())) { + PotentialPointers.insert(&I); + if (IntToPtrInst *IntToPtr = dyn_cast(&I)) { + if (auto *OpInst = dyn_cast(IntToPtr->getOperand(0))) { + WorklistSeen.insert(OpInst); + Worklist.insert(OpInst); + } + } + } + } } - }*/ - auto *entryBB = &F.getEntryBlock(); + while (!Worklist.empty()) { + auto *I = Worklist.back(); + Worklist.pop_back(); - DenseMap InstIdx; - uint64_t NextInstIdx = 0; - for (auto &BB : F) { - for (Instruction &I : BB) { - if ( - !I.getType()->isVoidTy() // we only care about vreg/values - && !isa(I) // the pointer returned by `alloca` isn't in the GC heap - ) { - InstIdx[&I] = NextInstIdx++; + if (I->getType()->isVoidTy()) continue; // we don't care about non-values + + // small integers (and bools) can't hold values large enough to + // be in the GC heap; assuming we have at least 64K of stack space + data + if (I->getType()->isIntegerTy() + && I->getType()->getScalarSizeInBits() <= 16) continue; + + PotentialPointers.insert(I); + + if (isa(I)) continue; // the operand of the load is unrelated to its result + + // We can safely assume the return of the function is + // unrelated to any non-pointer operands. + // + // However, we can only assume that for D function calls + // Not intrinsics LLVM might insert. + if (isa(I) && !isa(I)) continue; + + for (Use &Op : I->operands()) { + Value *V = Op.get(); + + // If the op is/has a pointer, it's not a concern for + // hidding values anymore; stop the back-tracking. + if (TypeContainsPointers(V->getType())) continue; + + if (auto *OpInst = dyn_cast(V)) { + if (WorklistSeen.insert(OpInst).second) Worklist.insert(OpInst); + } } - if (NextInstIdx >= (uint64_t(1) << 32)) reportFatalInternalError("More vregs than can fit in 32-bits!"); } } - size_t NumVRegs = NextInstIdx; + assert(PotentialPointers.size() < std::numeric_limits::max()); + + DenseMap InstIdx; + unsigned NumVRegs = 0; + for (Instruction *I : PotentialPointers) { + InstIdx[I] = NumVRegs++; + } - //SmallSetVector Worklist; - //DenseMap LiveIn; - //DenseMap LiveOut; + SmallSetVector Worklist; + DenseMap LiveIn; + DenseMap LiveInAcrossCall; + DenseMap LiveOut; + DenseMap LiveOutAcrossCall; DenseMap Defs; DenseMap Uses; - + DenseMap UsesAcrossCall; + DenseMap DefsAfterAllCalls; + BitVector HasCalls(F.size()); ReversePostOrderTraversal RPOT(&F); for (BasicBlock* BB : make_range(RPOT.begin(), RPOT.end())) { - //LiveIn[BB].resize(NumVRegs); - //LiveOut[BB].resize(NumVRegs); + LiveIn[BB].resize(NumVRegs); + LiveInAcrossCall[BB].resize(NumVRegs); + LiveOut[BB].resize(NumVRegs); + LiveOutAcrossCall[BB].resize(NumVRegs); Defs[BB].resize(NumVRegs); Uses[BB].resize(NumVRegs); + UsesAcrossCall[BB].resize(NumVRegs); + DefsAfterAllCalls[BB].resize(NumVRegs); auto &BBUses = Uses[BB]; + auto &BBUsesAcrossCall = UsesAcrossCall[BB]; auto &BBDefs = Defs[BB]; + auto &BBDefsAfterAllCalls = DefsAfterAllCalls[BB]; BitVector TmpUses(NumVRegs); + bool SeenCall = false; for (Instruction &I : make_range(BB->rbegin(), BB->rend())) { - for (Value *Op : I.operands()) { - if (auto *OpInst = dyn_cast(Op)) { - if (InstIdx.contains(OpInst)) TmpUses.set(InstIdx[OpInst]); + for (Use &Op : I.operands()) { + Value *V = Op.get(); + + if (auto *OpInst = dyn_cast(V)) { + if (InstIdx.contains(OpInst)) { + auto Idx = InstIdx[OpInst]; + BBUses.set(Idx); + TmpUses.set(Idx); + } } } if (InstIdx.contains(&I)) { - BBDefs.set(InstIdx[&I]); - TmpUses.reset(InstIdx[&I]); + auto Idx = InstIdx[&I]; + BBDefs.set(Idx); + if (!SeenCall) BBDefsAfterAllCalls.set(Idx); + BBUses.reset(Idx); + TmpUses.reset(Idx); } - if (isa(&I)) BBUses |= TmpUses; + if (isa(&I)) { + SeenCall = true; + BBUsesAcrossCall |= TmpUses; + } } - //Worklist.insert(BB); + HasCalls[BB->getNumber()] = SeenCall; + Worklist.insert(BB); } - /* + while (!Worklist.empty()) { auto *BB = Worklist.back(); Worklist.pop_back(); - BitVector NewLiveOut(NumVRegs); + auto &BBLiveOut = LiveOut[BB]; + auto &BBLiveOutAcrossCall = LiveOutAcrossCall[BB]; + + BBLiveOut.reset(); + BBLiveOutAcrossCall.reset(); for (BasicBlock *Succ : successors(BB)) { - NewLiveOut |= LiveIn[Succ]; + BBLiveOut |= LiveIn[Succ]; + BBLiveOutAcrossCall |= LiveInAcrossCall[Succ]; } - LiveOut[BB] = NewLiveOut; - - BitVector NewLiveIn = NewLiveOut; + // Normal liveliness + BitVector NewLiveIn = BBLiveOut; NewLiveIn |= Uses[BB]; NewLiveIn.reset(Defs[BB]); - if (NewLiveIn != LiveIn[BB]) { + + // Liveliness specifically of call crossing + BitVector NewLiveInAcrossCall = BBLiveOutAcrossCall; + NewLiveInAcrossCall |= UsesAcrossCall[BB]; + NewLiveInAcrossCall.reset(Defs[BB]); + + if (HasCalls[BB->getNumber()]) { + BitVector Tmp = BBLiveOut; + Tmp.reset(DefsAfterAllCalls[BB]); + NewLiveInAcrossCall |= Tmp; + } + + + if (NewLiveIn != LiveIn[BB] || NewLiveInAcrossCall != LiveInAcrossCall[BB]) { LiveIn[BB] = NewLiveIn; + LiveInAcrossCall[BB] = NewLiveInAcrossCall; for (BasicBlock *Pred : predecessors(BB)) { Worklist.insert(Pred); } } } - */ BitVector NeedsSpill(NumVRegs); for (auto &BB : F) { - NeedsSpill |= Uses[&BB]; + NeedsSpill |= UsesAcrossCall[&BB]; + NeedsSpill |= LiveOutAcrossCall[&BB]; } + // TODO: mark lifetimes and/or reuse alloca to reduce stack usage + auto &entryBB = F.getEntryBlock(); + auto allocaPoint = entryBB.getFirstInsertionPt(); - auto *allocaBB = BasicBlock::Create(F.getContext(), "stackSpillAllocas"); - allocaBB->insertInto(&F, entryBB); - - SmallVector InstToSpill = to_vector(InstIdx.keys()); - std::sort(InstToSpill.begin(), InstToSpill.end(), [&](Instruction *a, Instruction *b) - { - return InstIdx[a] < InstIdx[b]; - } - ); - - for (Instruction *I : InstToSpill) { + for (Instruction *I : PotentialPointers) { if (!NeedsSpill[InstIdx[I]]) continue; - auto *ai = new AllocaInst(I->getType(), 0, "stackSpill." + (I->hasName() ? I->getName() : "unnamedVreg"), allocaBB); + Changed = true; + + auto *ai = new AllocaInst(I->getType(), 0, "stackSpill." + (I->hasName() ? I->getName() : "unnamedVreg"), allocaPoint); auto *store = new StoreInst(I, ai, true, ai->getAlign(), nullptr); - store->insertAfter(I); - } - UncondBrInst::Create(entryBB, allocaBB); + std::optional After = I->getInsertionPointAfterDef(); + if (!After.has_value()) + reportFatalInternalError( + "Cannot spill value as it has no valid insertion point after it." + ); + + store->insertBefore(*After); + } return Changed; } From 44d51120a58648dabc20c034735e5a566f3abd29 Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Fri, 10 Jul 2026 13:46:49 -0700 Subject: [PATCH 03/17] `assert` instead of reportFatalInternalError --- gen/passes/WasmPointersSpill.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index f75d935809a..588e0bece64 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -283,10 +283,7 @@ bool WasmPointersSpill::run(Function &F) { auto *store = new StoreInst(I, ai, true, ai->getAlign(), nullptr); std::optional After = I->getInsertionPointAfterDef(); - if (!After.has_value()) - reportFatalInternalError( - "Cannot spill value as it has no valid insertion point after it." - ); + assert(After.has_value() && "Cannot spill value as it has no valid insertion point after it."); store->insertBefore(*After); } From 52c4f85c406b00293663bd49d0a397b9195b6d68 Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Fri, 10 Jul 2026 14:00:14 -0700 Subject: [PATCH 04/17] Fix block numbering on old LLVM --- gen/passes/WasmPointersSpill.cpp | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index 588e0bece64..6a71ca29f9e 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -90,7 +90,15 @@ static bool TypeContainsPointers(LLType *ty) { } bool WasmPointersSpill::run(Function &F) { - F.renumberBlocks(); +#if LLVM_VERSION_MAJOR >= 20 + unsigned BBCount = F.getMaxBlockNumber(); +#else + DenseMap BBIdx; + unsigned BBCount = 0; + for (auto &BB : F) { + BBIdx[&BB] = BBCount++; + } +#endif bool Changed = false; @@ -168,7 +176,7 @@ bool WasmPointersSpill::run(Function &F) { DenseMap Uses; DenseMap UsesAcrossCall; DenseMap DefsAfterAllCalls; - BitVector HasCalls(F.size()); + BitVector HasCalls(BBCount); ReversePostOrderTraversal RPOT(&F); for (BasicBlock* BB : make_range(RPOT.begin(), RPOT.end())) { @@ -215,7 +223,13 @@ bool WasmPointersSpill::run(Function &F) { BBUsesAcrossCall |= TmpUses; } } - HasCalls[BB->getNumber()] = SeenCall; + +#if LLVM_VERSION_MAJOR >= 20 + unsigned BBNum = BB->getNumber(); +#else + unsigned BBNum = BBIdx[BB]; +#endif + HasCalls[BBNum] = SeenCall; Worklist.insert(BB); } @@ -246,7 +260,12 @@ bool WasmPointersSpill::run(Function &F) { NewLiveInAcrossCall |= UsesAcrossCall[BB]; NewLiveInAcrossCall.reset(Defs[BB]); - if (HasCalls[BB->getNumber()]) { +#if LLVM_VERSION_MAJOR >= 20 + unsigned BBNum = BB->getNumber(); +#else + unsigned BBNum = BBIdx[BB]; +#endif + if (HasCalls[BBNum]) { BitVector Tmp = BBLiveOut; Tmp.reset(DefsAfterAllCalls[BB]); NewLiveInAcrossCall |= Tmp; From dcb11ac71e0f32a2c1c03bb22badcdff67bd675d Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Fri, 10 Jul 2026 16:21:48 -0700 Subject: [PATCH 05/17] Fix `AllocaInst` on old LLVM and use `TM` instead of `global` for triple. --- gen/optimizer.cpp | 2 +- gen/passes/WasmPointersSpill.cpp | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/gen/optimizer.cpp b/gen/optimizer.cpp index 60041383578..95d6cf32d1b 100644 --- a/gen/optimizer.cpp +++ b/gen/optimizer.cpp @@ -525,7 +525,7 @@ void runOptimizationPasses(llvm::Module *M, llvm::TargetMachine *TM) { pb.registerOptimizerLastEPCallback(addStripExternalsPass); - if (global.params.targetTriple->isWasm()) { + if (TM->getTargetTriple().isWasm()) { pb.registerOptimizerLastEPCallback(addWasmPointersSpillPass); } diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index 6a71ca29f9e..400ae45e666 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -28,8 +28,6 @@ #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Module.h" -#include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Analysis/ValueTracking.h" #include "llvm/Pass.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" @@ -297,7 +295,15 @@ bool WasmPointersSpill::run(Function &F) { Changed = true; - auto *ai = new AllocaInst(I->getType(), 0, "stackSpill." + (I->hasName() ? I->getName() : "unnamedVreg"), allocaPoint); + auto *ai = new AllocaInst( + I->getType(), 0, + "stackSpill." + (I->hasName() ? I->getName() : "unnamedVreg"), +#if LLVM_VERSION_MAJOR >= 19 + allocaPoint +#else + &*allocaPoint +#endif + ); auto *store = new StoreInst(I, ai, true, ai->getAlign(), nullptr); From 6f2bef0072649f421ebcee7d4bd725dde83d92b0 Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Fri, 10 Jul 2026 16:30:18 -0700 Subject: [PATCH 06/17] Fix `StoreInst` on old LLVM --- gen/passes/WasmPointersSpill.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index 400ae45e666..cea5c63c69f 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -305,12 +305,17 @@ bool WasmPointersSpill::run(Function &F) { #endif ); - auto *store = new StoreInst(I, ai, true, ai->getAlign(), nullptr); + std::optional After = I->getInsertionPointAfterDef(); + assert(After.has_value() && "Cannot spill value as it has no valid insertion point after it."); - std::optional After = I->getInsertionPointAfterDef(); - assert(After.has_value() && "Cannot spill value as it has no valid insertion point after it."); - - store->insertBefore(*After); + new StoreInst( + I, ai, true, ai->getAlign(), +#if LLVM_VERSION_MAJOR >= 19 + *After +#else + &**After +#endif + ); } return Changed; From 2e5ef09fa5229fd465fd30f35adb2044d0292761 Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Fri, 10 Jul 2026 21:46:53 -0700 Subject: [PATCH 07/17] Trace through `alloca` loads back through its stores --- gen/passes/WasmPointersSpill.cpp | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index cea5c63c69f..733c441aa87 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -22,8 +22,6 @@ #include "gen/tollvm.h" #include "gen/runtime.h" #include "llvm/ADT/Statistic.h" -#include "llvm/ADT/StringMap.h" -#include "llvm/IR/DataLayout.h" #include "llvm/IR/Function.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IRBuilder.h" @@ -33,6 +31,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/ADT/PostOrderIterator.h" +#include "llvm/Analysis/ValueTracking.h" using namespace llvm; @@ -134,7 +133,28 @@ bool WasmPointersSpill::run(Function &F) { PotentialPointers.insert(I); - if (isa(I)) continue; // the operand of the load is unrelated to its result + // trace `alloca` loads back through stores to the same + if (auto *Load = dyn_cast(I)) { + Value *V = Load->getPointerOperand(); + + AllocaInst *Alloca = findAllocaForValue(V); + if (Alloca) { + for (User *User : Alloca->users()) { + if (auto *Store = dyn_cast(User)) { + Value *StoreValue = Store->getValueOperand(); + StoreValue->dump(); + + if (TypeContainsPointers(StoreValue->getType())) continue; + + if (auto *OpInst = dyn_cast(StoreValue)) { + if (WorklistSeen.insert(OpInst).second) Worklist.insert(OpInst); + } + } + } + } + + continue; + } // We can safely assume the return of the function is // unrelated to any non-pointer operands. From 02e73e34d662f419ce492af9757a7afccca38fef Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Fri, 10 Jul 2026 21:58:01 -0700 Subject: [PATCH 08/17] Skip marking `alloca` --- gen/passes/WasmPointersSpill.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index 733c441aa87..762a5174f19 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -108,6 +108,8 @@ bool WasmPointersSpill::run(Function &F) { for (auto &BB : F) { for (Instruction &I : BB) { + if (isa(I)) continue; // the result of `alloca` can't be a GC pointer + if (TypeContainsPointers(I.getType())) { PotentialPointers.insert(&I); if (IntToPtrInst *IntToPtr = dyn_cast(&I)) { @@ -125,6 +127,7 @@ bool WasmPointersSpill::run(Function &F) { Worklist.pop_back(); if (I->getType()->isVoidTy()) continue; // we don't care about non-values + if (isa(I)) continue; // the result of `alloca` can't be a GC pointer // small integers (and bools) can't hold values large enough to // be in the GC heap; assuming we have at least 64K of stack space + data @@ -142,7 +145,6 @@ bool WasmPointersSpill::run(Function &F) { for (User *User : Alloca->users()) { if (auto *Store = dyn_cast(User)) { Value *StoreValue = Store->getValueOperand(); - StoreValue->dump(); if (TypeContainsPointers(StoreValue->getType())) continue; From 96a35edb23ce62c63524b9915cbd41f8b769e5fd Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Sat, 11 Jul 2026 00:23:48 -0700 Subject: [PATCH 09/17] Improve tracking across `load` --- gen/passes/WasmPointersSpill.cpp | 64 ++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index 762a5174f19..31523f4c05b 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -104,24 +104,57 @@ bool WasmPointersSpill::run(Function &F) { { SmallPtrSet WorklistSeen; - SmallSetVector Worklist; + SmallVector Worklist; + + DenseMap> ValuesStoredInAlloca; for (auto &BB : F) { for (Instruction &I : BB) { + if (auto *Store = dyn_cast(&I)) { + Value *StorePtr = Store->getPointerOperand(); + AllocaInst *Alloca = findAllocaForValue(StorePtr); + if (Alloca) ValuesStoredInAlloca[Alloca].insert(Store->getValueOperand()); + } + if (isa(I)) continue; // the result of `alloca` can't be a GC pointer if (TypeContainsPointers(I.getType())) { PotentialPointers.insert(&I); if (IntToPtrInst *IntToPtr = dyn_cast(&I)) { + // if we have `inttoptr` it's transitive uses are themselves + // potential pointers + if (auto *OpInst = dyn_cast(IntToPtr->getOperand(0))) { WorklistSeen.insert(OpInst); - Worklist.insert(OpInst); + Worklist.push_back(OpInst); } + } else if (isa(&I)) { + // if we have a direct `load ptr`, treat it as + // e.g. `load i32` and `inttoptr` + // + // if it's a load from an `alloca`, any stores + // into said `alloca` might be hidden pointers + + if (WorklistSeen.insert(&I).second) Worklist.push_back(&I); } } } } + + auto markValue = [&](Value *V) { + // If the op is/has a pointer, it's not a concern for + // hidding values anymore; stop the back-tracking. + // + // However, we want to still trace loads back through memory + // regardless to help catch `store`s with mismatching type. + if (TypeContainsPointers(V->getType()) && !isa(V)) return; + + if (auto *OpInst = dyn_cast(V)) { + if (WorklistSeen.insert(OpInst).second) Worklist.push_back(OpInst); + } + }; + while (!Worklist.empty()) { auto *I = Worklist.back(); Worklist.pop_back(); @@ -139,21 +172,10 @@ bool WasmPointersSpill::run(Function &F) { // trace `alloca` loads back through stores to the same if (auto *Load = dyn_cast(I)) { Value *V = Load->getPointerOperand(); - AllocaInst *Alloca = findAllocaForValue(V); - if (Alloca) { - for (User *User : Alloca->users()) { - if (auto *Store = dyn_cast(User)) { - Value *StoreValue = Store->getValueOperand(); - - if (TypeContainsPointers(StoreValue->getType())) continue; - - if (auto *OpInst = dyn_cast(StoreValue)) { - if (WorklistSeen.insert(OpInst).second) Worklist.insert(OpInst); - } - } - } - } + if (Alloca) + for (Value *StoreValue : ValuesStoredInAlloca[Alloca]) + markValue(StoreValue); continue; } @@ -167,14 +189,7 @@ bool WasmPointersSpill::run(Function &F) { for (Use &Op : I->operands()) { Value *V = Op.get(); - - // If the op is/has a pointer, it's not a concern for - // hidding values anymore; stop the back-tracking. - if (TypeContainsPointers(V->getType())) continue; - - if (auto *OpInst = dyn_cast(V)) { - if (WorklistSeen.insert(OpInst).second) Worklist.insert(OpInst); - } + markValue(V); } } } @@ -188,6 +203,7 @@ bool WasmPointersSpill::run(Function &F) { } SmallSetVector Worklist; + DenseMap LiveIn; DenseMap LiveInAcrossCall; DenseMap LiveOut; From 2458f241680ba4b62eaca40c10031376a3d5b142 Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Sun, 12 Jul 2026 01:15:51 -0700 Subject: [PATCH 10/17] Mark spill lifetimes --- gen/passes/WasmPointersSpill.cpp | 79 +++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index 31523f4c05b..5ac80fdd7ec 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -20,16 +20,13 @@ #include "gen/passes/WasmPointersSpill.h" #include "gen/llvmhelpers.h" #include "gen/tollvm.h" -#include "gen/runtime.h" -#include "llvm/ADT/Statistic.h" +#include "llvm/IR/Intrinsics.h" #include "llvm/IR/Function.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Module.h" #include "llvm/Pass.h" #include "llvm/Support/Compiler.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/Analysis/ValueTracking.h" @@ -319,41 +316,79 @@ bool WasmPointersSpill::run(Function &F) { } BitVector NeedsSpill(NumVRegs); + DenseMap SpillKills; for (auto &BB : F) { NeedsSpill |= UsesAcrossCall[&BB]; NeedsSpill |= LiveOutAcrossCall[&BB]; + + BitVector Tmp = LiveInAcrossCall[&BB]; + Tmp |= Defs[&BB]; + Tmp.reset(LiveOutAcrossCall[&BB]); + + SpillKills[&BB] = Tmp; } - // TODO: mark lifetimes and/or reuse alloca to reduce stack usage auto &entryBB = F.getEntryBlock(); auto allocaPoint = entryBB.getFirstInsertionPt(); + Function *LifetimeStartFn = + llvm::Intrinsic::getOrInsertDeclaration(F.getParent(), llvm::Intrinsic::lifetime_start, {F.getDataLayout().getAllocaPtrType(F.getContext())}); + Function *LifetimeEndFn = + llvm::Intrinsic::getOrInsertDeclaration(F.getParent(), llvm::Intrinsic::lifetime_end, {F.getDataLayout().getAllocaPtrType(F.getContext())}); + + IRBuilder<> Builder(&entryBB, allocaPoint); + for (Instruction *I : PotentialPointers) { - if (!NeedsSpill[InstIdx[I]]) continue; + unsigned Idx = InstIdx[I]; + if (!NeedsSpill[Idx]) continue; Changed = true; - auto *ai = new AllocaInst( - I->getType(), 0, - "stackSpill." + (I->hasName() ? I->getName() : "unnamedVreg"), -#if LLVM_VERSION_MAJOR >= 19 - allocaPoint -#else - &*allocaPoint -#endif + Builder.SetInsertPoint(allocaPoint); + AllocaInst *ai = Builder.CreateAlloca( + I->getType(), nullptr, + "stackSpill." + (I->hasName() ? I->getName() : "unnamedVreg") ); std::optional After = I->getInsertionPointAfterDef(); assert(After.has_value() && "Cannot spill value as it has no valid insertion point after it."); - new StoreInst( - I, ai, true, ai->getAlign(), -#if LLVM_VERSION_MAJOR >= 19 - *After -#else - &**After -#endif - ); + + Builder.SetInsertPoint(*After); + Builder.CreateCall(LifetimeStartFn, {ai}); + Builder.CreateStore(I, ai, true); + + + for (auto &BB : F) { + if(!SpillKills[&BB][Idx]) continue; + + if (UsesAcrossCall[&BB][Idx]) { + bool Found = false; + for (Instruction &IterI : make_range(BB.rbegin(), BB.rend())) { + for (Use &Op : IterI.operands()) { + Value *V = Op.get(); + + if (V == I) { + Found = true; + break; + } + } + + if (Found && isa(&IterI)) { + if (auto *Invoke = dyn_cast(&IterI)) { + Builder.SetInsertPoint(Invoke->getNormalDest()->getFirstInsertionPt()); + } else { + assert(!IterI.isTerminator()); + Builder.SetInsertPoint(std::next(IterI.getIterator())); + } + break; + } + } + } else { + Builder.SetInsertPoint(&BB.front()); + } + Builder.CreateCall(LifetimeEndFn, {ai}); + } } return Changed; From d9bfbea706f485253be0cfa4993dc81c245bf165 Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Sun, 12 Jul 2026 11:01:46 -0700 Subject: [PATCH 11/17] Fix `getOrInsertDeclaration` on old LLVM --- gen/passes/WasmPointersSpill.cpp | 39 ++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index 5ac80fdd7ec..1538a681756 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -331,10 +331,45 @@ bool WasmPointersSpill::run(Function &F) { auto &entryBB = F.getEntryBlock(); auto allocaPoint = entryBB.getFirstInsertionPt(); +#if LLVM_VERSION_MAJOR >= 20 + Function *LifetimeStartFn = + llvm::Intrinsic::getOrInsertDeclaration( + F.getParent(), + llvm::Intrinsic::lifetime_start, + {F.getParent()->getDataLayout().getAllocaPtrType(F.getContext())} + ); + Function *LifetimeEndFn = + llvm::Intrinsic::getOrInsertDeclaration( + F.getParent(), + llvm::Intrinsic::lifetime_end, + {F.getParent()->getDataLayout().getAllocaPtrType(F.getContext())} + ); +#elif LLVM_VERSION_MAJOR >= 19 Function *LifetimeStartFn = - llvm::Intrinsic::getOrInsertDeclaration(F.getParent(), llvm::Intrinsic::lifetime_start, {F.getDataLayout().getAllocaPtrType(F.getContext())}); + llvm::Intrinsic::getDeclaration( + F.getParent(), + llvm::Intrinsic::lifetime_start, + {F.getParent()->getDataLayout().getAllocaPtrType(F.getContext())} + ); Function *LifetimeEndFn = - llvm::Intrinsic::getOrInsertDeclaration(F.getParent(), llvm::Intrinsic::lifetime_end, {F.getDataLayout().getAllocaPtrType(F.getContext())}); + llvm::Intrinsic::getDeclaration( + F.getParent(), + llvm::Intrinsic::lifetime_end, + {F.getParent()->getDataLayout().getAllocaPtrType(F.getContext())} + ); +#else + Function *LifetimeStartFn = + llvm::Intrinsic::getDeclaration( + F.getParent(), + llvm::Intrinsic::lifetime_start + ); + Function *LifetimeEndFn = + llvm::Intrinsic::getDeclaration( + F.getParent(), + llvm::Intrinsic::lifetime_end + ); +#endif + IRBuilder<> Builder(&entryBB, allocaPoint); From 324cf6dde16450619942d3aa2667b2e1ad9e5e91 Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Sun, 12 Jul 2026 12:38:30 -0700 Subject: [PATCH 12/17] Rearrange lifetime placement to make them more disjoint. --- gen/passes/WasmPointersSpill.cpp | 75 ++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 23 deletions(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index 1538a681756..0ddc6f5cc2a 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -321,7 +321,8 @@ bool WasmPointersSpill::run(Function &F) { NeedsSpill |= UsesAcrossCall[&BB]; NeedsSpill |= LiveOutAcrossCall[&BB]; - BitVector Tmp = LiveInAcrossCall[&BB]; + BitVector Tmp(NumVRegs); + Tmp |= LiveInAcrossCall[&BB]; Tmp |= Defs[&BB]; Tmp.reset(LiveOutAcrossCall[&BB]); @@ -333,66 +334,70 @@ bool WasmPointersSpill::run(Function &F) { #if LLVM_VERSION_MAJOR >= 20 Function *LifetimeStartFn = - llvm::Intrinsic::getOrInsertDeclaration( + Intrinsic::getOrInsertDeclaration( F.getParent(), - llvm::Intrinsic::lifetime_start, + Intrinsic::lifetime_start, {F.getParent()->getDataLayout().getAllocaPtrType(F.getContext())} ); Function *LifetimeEndFn = - llvm::Intrinsic::getOrInsertDeclaration( + Intrinsic::getOrInsertDeclaration( F.getParent(), - llvm::Intrinsic::lifetime_end, + Intrinsic::lifetime_end, {F.getParent()->getDataLayout().getAllocaPtrType(F.getContext())} ); #elif LLVM_VERSION_MAJOR >= 19 Function *LifetimeStartFn = - llvm::Intrinsic::getDeclaration( + Intrinsic::getDeclaration( F.getParent(), - llvm::Intrinsic::lifetime_start, + Intrinsic::lifetime_start, {F.getParent()->getDataLayout().getAllocaPtrType(F.getContext())} ); Function *LifetimeEndFn = - llvm::Intrinsic::getDeclaration( + Intrinsic::getDeclaration( F.getParent(), - llvm::Intrinsic::lifetime_end, + Intrinsic::lifetime_end, {F.getParent()->getDataLayout().getAllocaPtrType(F.getContext())} ); #else Function *LifetimeStartFn = - llvm::Intrinsic::getDeclaration( + Intrinsic::getDeclaration( F.getParent(), - llvm::Intrinsic::lifetime_start + Intrinsic::lifetime_start ); Function *LifetimeEndFn = - llvm::Intrinsic::getDeclaration( + Intrinsic::getDeclaration( F.getParent(), - llvm::Intrinsic::lifetime_end + Intrinsic::lifetime_end ); #endif IRBuilder<> Builder(&entryBB, allocaPoint); + SmallVector SpillPointers; + SmallVector SpillAllocas; + + Builder.SetInsertPoint(allocaPoint); + for (Instruction *I : PotentialPointers) { unsigned Idx = InstIdx[I]; if (!NeedsSpill[Idx]) continue; Changed = true; - Builder.SetInsertPoint(allocaPoint); - AllocaInst *ai = Builder.CreateAlloca( + SpillAllocas.push_back(Builder.CreateAlloca( I->getType(), nullptr, "stackSpill." + (I->hasName() ? I->getName() : "unnamedVreg") - ); - - std::optional After = I->getInsertionPointAfterDef(); - assert(After.has_value() && "Cannot spill value as it has no valid insertion point after it."); - + )); + SpillPointers.push_back(I); + } - Builder.SetInsertPoint(*After); - Builder.CreateCall(LifetimeStartFn, {ai}); - Builder.CreateStore(I, ai, true); + unsigned NextSpillPointerIdx = 0; + // do `lifetime.end` first + for (Instruction *I : SpillPointers) { + unsigned Idx = InstIdx[I]; + AllocaInst *ai = SpillAllocas[NextSpillPointerIdx++]; for (auto &BB : F) { if(!SpillKills[&BB][Idx]) continue; @@ -426,5 +431,29 @@ bool WasmPointersSpill::run(Function &F) { } } + // then insert `lifetime.start` and store, + // skipping after any `lifetime.end` + NextSpillPointerIdx = 0; + for (Instruction *I : SpillPointers) { + AllocaInst *ai = SpillAllocas[NextSpillPointerIdx++]; + + BasicBlock::iterator After; + { + std::optional MaybeAfter = I->getInsertionPointAfterDef(); + assert(MaybeAfter.has_value() && "Cannot spill value as it has no valid insertion point after it."); + After = *MaybeAfter; + } + + while (auto *Intrinsic = dyn_cast(&*After)) { + if (Intrinsic->getIntrinsicID() != Intrinsic::lifetime_end) break; + After = std::next(After); + } + + + Builder.SetInsertPoint(After); + Builder.CreateCall(LifetimeStartFn, {ai}); + Builder.CreateStore(I, ai, true); + } + return Changed; } From a21c4dff75f34deb8b6424711fe9bfebfb027369 Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Sun, 12 Jul 2026 13:15:40 -0700 Subject: [PATCH 13/17] Fix lifetime intrinsic signatures on old LLVM --- gen/passes/WasmPointersSpill.cpp | 44 +++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index 0ddc6f5cc2a..6a5ecab600d 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -332,31 +332,33 @@ bool WasmPointersSpill::run(Function &F) { auto &entryBB = F.getEntryBlock(); auto allocaPoint = entryBB.getFirstInsertionPt(); + auto &DL = F.getParent()->getDataLayout(); + #if LLVM_VERSION_MAJOR >= 20 Function *LifetimeStartFn = Intrinsic::getOrInsertDeclaration( F.getParent(), Intrinsic::lifetime_start, - {F.getParent()->getDataLayout().getAllocaPtrType(F.getContext())} + {DL.getAllocaPtrType(F.getContext())} ); Function *LifetimeEndFn = Intrinsic::getOrInsertDeclaration( F.getParent(), Intrinsic::lifetime_end, - {F.getParent()->getDataLayout().getAllocaPtrType(F.getContext())} + {DL.getAllocaPtrType(F.getContext())} ); #elif LLVM_VERSION_MAJOR >= 19 Function *LifetimeStartFn = Intrinsic::getDeclaration( F.getParent(), Intrinsic::lifetime_start, - {F.getParent()->getDataLayout().getAllocaPtrType(F.getContext())} + {DL.getAllocaPtrType(F.getContext())} ); Function *LifetimeEndFn = Intrinsic::getDeclaration( F.getParent(), Intrinsic::lifetime_end, - {F.getParent()->getDataLayout().getAllocaPtrType(F.getContext())} + {DL.getAllocaPtrType(F.getContext())} ); #else Function *LifetimeStartFn = @@ -376,9 +378,15 @@ bool WasmPointersSpill::run(Function &F) { SmallVector SpillPointers; SmallVector SpillAllocas; +#if LLVM_VERSION_MAJOR < 22 + SmallVector SpillSizes; +#endif Builder.SetInsertPoint(allocaPoint); +#if LLVM_VERSION_MAJOR < 22 + IntegerType *I64Type = IntegerType::get(F.getContext(), 64); +#endif for (Instruction *I : PotentialPointers) { unsigned Idx = InstIdx[I]; if (!NeedsSpill[Idx]) continue; @@ -390,6 +398,10 @@ bool WasmPointersSpill::run(Function &F) { "stackSpill." + (I->hasName() ? I->getName() : "unnamedVreg") )); SpillPointers.push_back(I); + +#if LLVM_VERSION_MAJOR < 22 + SpillSizes.push_back(ConstantInt::get(I64Type, DL.getTypeAllocSize(I->getType()))); +#endif } unsigned NextSpillPointerIdx = 0; @@ -397,6 +409,9 @@ bool WasmPointersSpill::run(Function &F) { // do `lifetime.end` first for (Instruction *I : SpillPointers) { unsigned Idx = InstIdx[I]; +#if LLVM_VERSION_MAJOR < 22 + ConstantInt *size = SpillSizes[NextSpillPointerIdx]; +#endif AllocaInst *ai = SpillAllocas[NextSpillPointerIdx++]; for (auto &BB : F) { @@ -427,7 +442,14 @@ bool WasmPointersSpill::run(Function &F) { } else { Builder.SetInsertPoint(&BB.front()); } - Builder.CreateCall(LifetimeEndFn, {ai}); + Builder.CreateCall( + LifetimeEndFn, +#if LLVM_VERSION_MAJOR >= 22 + {ai}, +#else + {size, ai} +#endif + ); } } @@ -435,6 +457,9 @@ bool WasmPointersSpill::run(Function &F) { // skipping after any `lifetime.end` NextSpillPointerIdx = 0; for (Instruction *I : SpillPointers) { +#if LLVM_VERSION_MAJOR < 22 + ConstantInt *size = SpillSizes[NextSpillPointerIdx]; +#endif AllocaInst *ai = SpillAllocas[NextSpillPointerIdx++]; BasicBlock::iterator After; @@ -451,7 +476,14 @@ bool WasmPointersSpill::run(Function &F) { Builder.SetInsertPoint(After); - Builder.CreateCall(LifetimeStartFn, {ai}); + Builder.CreateCall( + LifetimeStartFn, +#if LLVM_VERSION_MAJOR >= 22 + {ai}, +#else + {size, ai} +#endif + ); Builder.CreateStore(I, ai, true); } From eade33cbf30c9806c8d613827e14259d9b6a6f7c Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Sun, 12 Jul 2026 13:34:55 -0700 Subject: [PATCH 14/17] Fix lifetime intrinsic declarations...again... --- gen/passes/WasmPointersSpill.cpp | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index 6a5ecab600d..07419d33a8c 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -347,7 +347,7 @@ bool WasmPointersSpill::run(Function &F) { Intrinsic::lifetime_end, {DL.getAllocaPtrType(F.getContext())} ); -#elif LLVM_VERSION_MAJOR >= 19 +#else Function *LifetimeStartFn = Intrinsic::getDeclaration( F.getParent(), @@ -360,20 +360,8 @@ bool WasmPointersSpill::run(Function &F) { Intrinsic::lifetime_end, {DL.getAllocaPtrType(F.getContext())} ); -#else - Function *LifetimeStartFn = - Intrinsic::getDeclaration( - F.getParent(), - Intrinsic::lifetime_start - ); - Function *LifetimeEndFn = - Intrinsic::getDeclaration( - F.getParent(), - Intrinsic::lifetime_end - ); #endif - IRBuilder<> Builder(&entryBB, allocaPoint); SmallVector SpillPointers; @@ -445,7 +433,7 @@ bool WasmPointersSpill::run(Function &F) { Builder.CreateCall( LifetimeEndFn, #if LLVM_VERSION_MAJOR >= 22 - {ai}, + {ai} #else {size, ai} #endif @@ -479,7 +467,7 @@ bool WasmPointersSpill::run(Function &F) { Builder.CreateCall( LifetimeStartFn, #if LLVM_VERSION_MAJOR >= 22 - {ai}, + {ai} #else {size, ai} #endif From 8042198ffa98d4e4ef85c6584948c267eb9d9517 Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Thu, 16 Jul 2026 16:29:03 -0700 Subject: [PATCH 15/17] `getFirstInsertionPt`, not `front`! (lifetime.end placement) --- gen/passes/WasmPointersSpill.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index 07419d33a8c..7e1b723449d 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -428,7 +428,7 @@ bool WasmPointersSpill::run(Function &F) { } } } else { - Builder.SetInsertPoint(&BB.front()); + Builder.SetInsertPoint(BB.getFirstInsertionPt()); } Builder.CreateCall( LifetimeEndFn, From 60f5ddd9808122b27d3deb8e13d7db8540a4b358 Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Thu, 16 Jul 2026 21:56:29 -0700 Subject: [PATCH 16/17] Fix spill insert point for `invoke` and `catchswitch` --- gen/passes/WasmPointersSpill.cpp | 55 ++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index 7e1b723449d..5ff1a469106 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -450,20 +450,57 @@ bool WasmPointersSpill::run(Function &F) { #endif AllocaInst *ai = SpillAllocas[NextSpillPointerIdx++]; - BasicBlock::iterator After; - { - std::optional MaybeAfter = I->getInsertionPointAfterDef(); - assert(MaybeAfter.has_value() && "Cannot spill value as it has no valid insertion point after it."); - After = *MaybeAfter; + BasicBlock::iterator SpillInsertPoint; + + assert(!I->isTerminator() || isa(I)); + + BasicBlock *BB = I->getParent(); + + if (auto *Invoke = dyn_cast(I)) { + if (Invoke->getNormalDest()->hasNPredecessors(1)) { + SpillInsertPoint = Invoke->getNormalDest()->getFirstInsertionPt(); + } else { + // The destination block has predecessors other + // than this invoke, so split the CFG edge + // (to make the invoke/def dominate the spill) + + BasicBlock *newBB = BasicBlock::Create(F.getContext(), ai->getName() + ".bb", &F); + Builder.SetInsertPoint(newBB); + Builder.CreateBr(Invoke->getNormalDest()); + newBB->moveAfter(BB); + + Invoke->getNormalDest()->replacePhiUsesWith(BB, newBB); + Invoke->replaceSuccessorWith(Invoke->getNormalDest(), newBB); + + SpillInsertPoint = newBB->begin(); + } + } else if (isa(I)) { + if (isa(BB->getTerminator())) { + // Having a catchswitch for a terminator means no instructions other + // than PHI can be here. And since all successors must be catchpad + // we can't split the edge either. + // + // However, Wasm EH associates each catchswitch with a single catchpad. + // So spill at the start of that single successor. + + BasicBlock *Succ = BB->getSingleSuccessor(); + assert(Succ && "catchswitch with multiple catchpads?"); + + SpillInsertPoint = Succ->getFirstInsertionPt(); + } else { + // After all the PHI + SpillInsertPoint = BB->getFirstInsertionPt(); + } + } else { + SpillInsertPoint = std::next(I->getIterator()); } - while (auto *Intrinsic = dyn_cast(&*After)) { + while (auto *Intrinsic = dyn_cast(&*SpillInsertPoint)) { if (Intrinsic->getIntrinsicID() != Intrinsic::lifetime_end) break; - After = std::next(After); + SpillInsertPoint = std::next(SpillInsertPoint); } - - Builder.SetInsertPoint(After); + Builder.SetInsertPoint(SpillInsertPoint); Builder.CreateCall( LifetimeStartFn, #if LLVM_VERSION_MAJOR >= 22 From 7e89bd2bd7ea4b03a18b92a81739941cb3369799 Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Thu, 16 Jul 2026 22:48:23 -0700 Subject: [PATCH 17/17] Fix AcrossCall liveliness propogation --- gen/passes/WasmPointersSpill.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/gen/passes/WasmPointersSpill.cpp b/gen/passes/WasmPointersSpill.cpp index 5ff1a469106..28b1d56a968 100644 --- a/gen/passes/WasmPointersSpill.cpp +++ b/gen/passes/WasmPointersSpill.cpp @@ -287,12 +287,6 @@ bool WasmPointersSpill::run(Function &F) { NewLiveIn |= Uses[BB]; NewLiveIn.reset(Defs[BB]); - - // Liveliness specifically of call crossing - BitVector NewLiveInAcrossCall = BBLiveOutAcrossCall; - NewLiveInAcrossCall |= UsesAcrossCall[BB]; - NewLiveInAcrossCall.reset(Defs[BB]); - #if LLVM_VERSION_MAJOR >= 20 unsigned BBNum = BB->getNumber(); #else @@ -301,9 +295,14 @@ bool WasmPointersSpill::run(Function &F) { if (HasCalls[BBNum]) { BitVector Tmp = BBLiveOut; Tmp.reset(DefsAfterAllCalls[BB]); - NewLiveInAcrossCall |= Tmp; + BBLiveOutAcrossCall |= Tmp; } + // Liveliness specifically of call crossing + BitVector NewLiveInAcrossCall = BBLiveOutAcrossCall; + NewLiveInAcrossCall |= UsesAcrossCall[BB]; + NewLiveInAcrossCall.reset(Defs[BB]); + if (NewLiveIn != LiveIn[BB] || NewLiveInAcrossCall != LiveInAcrossCall[BB]) { LiveIn[BB] = NewLiveIn;