52#include <llvm/Config/llvm-config.h>
54#include "llvm/ADT/APInt.h"
55#include "llvm/ADT/DenseMap.h"
56#include "llvm/ADT/MapVector.h"
57#include "llvm/ADT/SmallPtrSet.h"
58#include "llvm/ADT/SmallVector.h"
60#include "llvm/IR/BasicBlock.h"
61#include "llvm/IR/Constants.h"
62#include "llvm/IR/DataLayout.h"
63#include "llvm/IR/Dominators.h"
65#include "llvm/IR/CFG.h"
66#include "llvm/IR/Function.h"
67#include "llvm/IR/GetElementPtrTypeIterator.h"
68#include "llvm/IR/IRBuilder.h"
69#include "llvm/IR/InstrTypes.h"
70#include "llvm/IR/Instruction.h"
71#include "llvm/IR/Instructions.h"
72#include "llvm/IR/Value.h"
74#include "llvm/Analysis/AliasAnalysis.h"
75#include "llvm/Analysis/MemoryLocation.h"
76#include "llvm/Analysis/TargetLibraryInfo.h"
77#include "llvm/Analysis/ValueTracking.h"
79#include "llvm/IR/LegacyPassManager.h"
81#include "llvm/Support/Debug.h"
82#include "llvm/Support/raw_ostream.h"
84#include "llvm/Transforms/Utils/Local.h"
94#define DEBUG_TYPE "simple-gvn"
99Value *extractValue(IRBuilder<> &Builder, Value *StoredVal, Type *LoadType,
100 const DataLayout &DL, APInt LoadOffset, APInt StoreOffset,
102 Type *StoreType = StoredVal->getType();
103 uint64_t StoreSize = DL.getTypeStoreSize(StoreType);
106 int64_t RelativeOffset = (LoadOffset - StoreOffset).getSExtValue();
109 if (RelativeOffset < 0 || (uint64_t)RelativeOffset + LoadSize > StoreSize) {
114 if (RelativeOffset == 0 && LoadType == StoreType) {
118 if (RelativeOffset == 0 && isa<PointerType>(LoadType) &&
119 isa<PointerType>(StoreType)) {
120 auto FromAS = cast<PointerType>(StoreType)->getAddressSpace();
121 auto ToAS = cast<PointerType>(LoadType)->getAddressSpace();
122 if (FromAS == 10 && ToAS == 0) {
123 Module *mod = Builder.GetInsertBlock()->getModule();
124 Function *F = mod->getFunction(
"julia.pointer_from_objref");
126 Type *T_jlvalue = StructType::get(mod->getContext());
129 FunctionType *FTy = FunctionType::get(T_pjlvalue, {T_prjlvalue},
false);
130 F = Function::Create(FTy, Function::ExternalLinkage,
131 "julia.pointer_from_objref", mod);
134 Type *T_jlvalue = StructType::get(mod->getContext());
136 Builder.CreateAddrSpaceCast(StoredVal,
getPointerType(T_jlvalue, 11));
137 Value *RawPtr = Builder.CreateCall(F, {TrackedVal});
138 return Builder.CreatePointerCast(RawPtr, LoadType);
140 return Builder.CreatePointerCast(StoredVal, LoadType);
143 if (RelativeOffset == 0 && StoreSize >= LoadSize &&
144 StoreType->isAggregateType()) {
145 auto first = Builder.CreateExtractValue(StoredVal, 0);
146 auto res = extractValue(Builder, first, LoadType, DL, LoadOffset,
147 StoreOffset, LoadSize);
151 if (
auto I = dyn_cast<Instruction>(first))
152 I->eraseFromParent();
158 if (!StoreType->isIntegerTy()) {
159 IntegerType *IntTy = Builder.getIntNTy(StoreSize * 8);
160 if (!CastInst::castIsValid(Instruction::BitCast, StoredVal->getType(),
164 StoredVal = Builder.CreateBitCast(StoredVal, IntTy);
168 if (RelativeOffset > 0) {
169 uint64_t ShiftBits = RelativeOffset * 8;
170 StoredVal = Builder.CreateLShr(StoredVal, ShiftBits);
174 IntegerType *LoadIntTy = Builder.getIntNTy(LoadSize * 8);
175 if (StoredVal->getType() != LoadIntTy) {
176 unsigned StoredWidth = StoredVal->getType()->getIntegerBitWidth();
177 unsigned LoadWidth = LoadIntTy->getIntegerBitWidth();
178 if (StoredWidth > LoadWidth) {
179 StoredVal = Builder.CreateTrunc(StoredVal, LoadIntTy);
180 }
else if (StoredWidth < LoadWidth) {
181 StoredVal = Builder.CreateZExt(StoredVal, LoadIntTy);
186 if (LoadIntTy != LoadType) {
187 if (LoadType->isPointerTy()) {
188 if (cast<PointerType>(LoadType)->getAddressSpace() == 10) {
189 Type *EmptyStructTy = StructType::get(LoadType->getContext());
191 Value *Ptr0 = Builder.CreateIntToPtr(StoredVal, PtrAddrSpace0);
192 StoredVal = Builder.CreateAddrSpaceCast(Ptr0, LoadType);
194 StoredVal = Builder.CreateIntToPtr(StoredVal, LoadType);
197 if (!CastInst::castIsValid(Instruction::BitCast, StoredVal->getType(),
201 StoredVal = Builder.CreateBitCast(StoredVal, LoadType);
212static bool dominatesAndCovers(Instruction *Source, Instruction *Target,
213 const APInt &SourceOffset,
214 const APInt &TargetOffset, uint64_t TargetSize,
215 const DataLayout &DL, DominatorTree &DT) {
216 if (!DT.dominates(Source, Target))
221 if (
auto *SI = dyn_cast<StoreInst>(Source)) {
222 SourceSize = DL.getTypeStoreSize(SI->getValueOperand()->getType());
223 }
else if (
auto *LI = dyn_cast<LoadInst>(Source)) {
224 SourceSize = DL.getTypeStoreSize(LI->getType());
229 int64_t RelOffset = (TargetOffset - SourceOffset).getSExtValue();
230 return RelOffset >= 0 && (uint64_t)RelOffset + TargetSize <= SourceSize;
236static bool memoryRangesAlias(
const APInt &Offset1, uint64_t Size1,
237 const APInt &Offset2, uint64_t Size2) {
239 if ((Offset2 + Size2).sle(Offset1))
243 if ((Offset1 + Size1).sle(Offset2))
255collectMemoryOps(Value *Arg,
const DataLayout &DL,
256 SmallVectorImpl<std::pair<StoreInst *, APInt>> &Stores,
257 SmallVectorImpl<std::pair<LoadInst *, APInt>> &Loads,
258 SmallVectorImpl<std::pair<CallInst *, APInt>> &Calls) {
260 SmallVector<std::pair<Value *, APInt>, 16> ToProcess;
261 SmallPtrSet<Value *, 16> Visited;
263 APInt ZeroOffset(DL.getIndexTypeSizeInBits(Arg->getType()), 0);
264 ToProcess.push_back({Arg, ZeroOffset});
266 while (!ToProcess.empty()) {
267 auto [V, CurrentOffset] = ToProcess.pop_back_val();
270 if (!Visited.insert(V).second)
273 for (Use &U : V->uses()) {
274 User *Usr = U.getUser();
275 if (
auto *LI = dyn_cast<LoadInst>(Usr)) {
276 Loads.push_back({LI, CurrentOffset});
277 }
else if (
auto *SI = dyn_cast<StoreInst>(Usr)) {
280 if (SI->getPointerOperand() == V) {
281 Stores.push_back({SI, CurrentOffset});
286 }
else if (
auto *GEP = dyn_cast<GetElementPtrInst>(Usr)) {
288 APInt GEPOffset(DL.getIndexTypeSizeInBits(GEP->getType()), 0);
289 if (!GEP->accumulateConstantOffset(DL, GEPOffset)) {
294 APInt NewOffset = CurrentOffset + GEPOffset;
295 ToProcess.push_back({GEP, NewOffset});
296 }
else if (
auto *CI = dyn_cast<CastInst>(Usr)) {
298 ToProcess.push_back({CI, CurrentOffset});
299 }
else if (
auto *
Call = dyn_cast<CallInst>(Usr)) {
301 unsigned ArgIdx = U.getOperandNo();
303 Calls.push_back({
Call, CurrentOffset});
318static bool hasInterveningMod(Instruction *Start, Instruction *End,
319 LoadInst *LI, AAResults &AA,
320 TargetLibraryInfo &TLI) {
321 assert(Start->getParent() == End->getParent());
322 for (Instruction *I = Start->getNextNode(); I != End; I = I->getNextNode()) {
323 if (I->mayWriteToMemory()) {
333bool simplifyGVN(Function &F, DominatorTree &DT,
const DataLayout &DL,
334 AAResults &AA, TargetLibraryInfo &TLI) {
335 bool Changed =
false;
338 SmallVector<std::pair<Value *, bool>, 4> CandidateArgs;
339 for (Argument &Arg : F.args()) {
340 if (Arg.getType()->isPointerTy()) {
341 CandidateArgs.push_back({&Arg, Arg.hasNoAliasAttr()});
345 for (BasicBlock &BB : F) {
346 for (Instruction &I : BB) {
347 if (isa<AllocaInst>(&I)) {
348 CandidateArgs.push_back({&I,
true});
349 }
else if (
auto *CB = dyn_cast<CallBase>(&I)) {
350 if (CB->getType()->isPointerTy()) {
351 CandidateArgs.push_back({CB, CB->hasRetAttr(Attribute::NoAlias)});
357 if (CandidateArgs.empty())
361 for (
auto &pair : CandidateArgs) {
362 Value *Arg = pair.first;
366 SmallVector<std::pair<StoreInst *, APInt>, 8> Stores;
367 SmallVector<std::pair<LoadInst *, APInt>, 8> Loads;
368 SmallVector<std::pair<CallInst *, APInt>, 8> Calls;
372 if (!collectMemoryOps(Arg, DL, Stores, Loads, Calls)) {
377 APInt ZeroOffset(DL.getIndexTypeSizeInBits(Arg->getType()), 0);
381 for (
auto &[LI, LoadOffset] : Loads) {
382 uint64_t LoadSize = DL.getTypeStoreSize(LI->getType());
385 SmallVector<std::tuple<Instruction *, APInt, uint64_t>, 8> AliasingStores;
386 for (
auto &[SI, StoreOffset] : Stores) {
388 DL.getTypeStoreSize(SI->getValueOperand()->getType());
389 if (memoryRangesAlias(LoadOffset, LoadSize, StoreOffset, StoreSize)) {
390 AliasingStores.push_back({SI, StoreOffset, StoreSize});
396 for (
auto &[CI, CallOffset] : Calls) {
397 AliasingStores.push_back({CI, LoadOffset, LoadSize});
403 SmallVector<std::tuple<Instruction *, APInt, Value *>, 8>
404 DominatingCoveringStores;
405 for (
auto &[I, StoreOffset, StoreSize] : AliasingStores) {
406 if (
auto SI = dyn_cast<StoreInst>(I))
407 if (dominatesAndCovers(SI, LI, StoreOffset, LoadOffset, LoadSize, DL,
409 DominatingCoveringStores.push_back(
410 {SI, StoreOffset, SI->getValueOperand()});
416 if (AliasingStores.size() == 1 && DominatingCoveringStores.size() == 1) {
417 Instruction *SI = std::get<0>(DominatingCoveringStores[0]);
418 APInt StoreOffset = std::get<1>(DominatingCoveringStores[0]);
420 IRBuilder<> Builder(LI);
421 Value *StoredVal = std::get<2>(DominatingCoveringStores[0]);
422 Value *ExtractedVal =
423 extractValue(Builder, StoredVal, LI->getType(), DL, LoadOffset,
424 StoreOffset, LoadSize);
429 if (SI->getParent() != LI->getParent()) {
432 Safe = !hasInterveningMod(SI, LI, LI, AA, TLI);
436 LLVM_DEBUG(dbgs() <<
"SimpleGVN: Forwarding (single alias)\n"
437 <<
" Store: " << *SI <<
"\n"
438 <<
" Load: " << *LI <<
"\n");
439 LI->replaceAllUsesWith(ExtractedVal);
440 LI->eraseFromParent();
449 for (
auto &[LI2, LoadOffset2] : Loads) {
450 if (!LI2 || LI2 == LI)
452 if (dominatesAndCovers(LI2, LI, LoadOffset2, LoadOffset, LoadSize, DL,
454 DominatingCoveringStores.emplace_back(LI2, LoadOffset2, LI2);
459 if (DominatingCoveringStores.empty()) {
464 DenseMap<BasicBlock *, std::tuple<Instruction *, APInt, uint64_t>>
465 LastStoreInBlockBeforeLI;
466 for (
auto &[SI, StoreOffset, Size] : AliasingStores) {
467 BasicBlock *BB = SI->getParent();
468 if (BB == LI->getParent()) {
470 if (SI->comesBefore(LI)) {
471 auto &Entry = LastStoreInBlockBeforeLI[BB];
472 if (!std::get<0>(Entry) || std::get<0>(Entry)->comesBefore(SI)) {
473 Entry = {SI, StoreOffset, Size};
478 auto &Entry = LastStoreInBlockBeforeLI[BB];
479 if (!std::get<0>(Entry) || std::get<0>(Entry)->comesBefore(SI)) {
480 Entry = {SI, StoreOffset, Size};
486 BasicBlock *LIBlock = LI->getParent();
487 auto It = LastStoreInBlockBeforeLI.find(LIBlock);
488 if (It != LastStoreInBlockBeforeLI.end()) {
489 Instruction *SI = std::get<0>(It->second);
491 for (
auto &&[DCS, StoreOffset, StoredVal] : DominatingCoveringStores) {
493 (DCS->getParent() == LI->getParent() && SI->comesBefore(DCS))) {
495 IRBuilder<> Builder(LI);
496 Value *ExtractedVal =
497 extractValue(Builder, StoredVal, LI->getType(), DL, LoadOffset,
498 StoreOffset, LoadSize);
503 Safe = !hasInterveningMod(DCS, LI, LI, AA, TLI);
506 LLVM_DEBUG(dbgs() <<
"SimpleGVN: Forwarding (same block)\n"
507 <<
" Store: " << *DCS <<
"\n"
508 <<
" Load: " << *LI <<
"\n");
509 LI->replaceAllUsesWith(ExtractedVal);
510 LI->eraseFromParent();
520 for (
auto &&[DCS, StoreOffset, StoredVal] : DominatingCoveringStores) {
521 if (DCS->getParent() == LI->getParent()) {
523 IRBuilder<> Builder(LI);
524 Value *ExtractedVal =
525 extractValue(Builder, StoredVal, LI->getType(), DL, LoadOffset,
526 StoreOffset, LoadSize);
531 Safe = !hasInterveningMod(DCS, LI, LI, AA, TLI);
534 LLVM_DEBUG(dbgs() <<
"SimpleGVN: Forwarding (same block)\n"
535 <<
" Store: " << *DCS <<
"\n"
536 <<
" Load: " << *LI <<
"\n");
537 LI->replaceAllUsesWith(ExtractedVal);
538 LI->eraseFromParent();
553 SmallPtrSet<BasicBlock *, 32> Visited;
554 SmallVector<BasicBlock *, 16> Worklist;
555 StoreInst *Candidate =
nullptr;
556 APInt CandidateOffset = ZeroOffset;
559 for (BasicBlock *Pred : predecessors(LIBlock)) {
560 if (Visited.insert(Pred).second)
561 Worklist.push_back(Pred);
564 while (!Worklist.empty()) {
565 BasicBlock *BB = Worklist.pop_back_val();
567 auto It = LastStoreInBlockBeforeLI.find(BB);
568 if (It != LastStoreInBlockBeforeLI.end()) {
569 StoreInst *SI = dyn_cast<StoreInst>(std::get<0>(It->second));
570 APInt StoreOffset = std::get<1>(It->second);
572 if (!SI || !dominatesAndCovers(SI, LI, StoreOffset, LoadOffset,
582 CandidateOffset = StoreOffset;
583 }
else if (Candidate != SI) {
591 for (BasicBlock *Pred : predecessors(BB)) {
592 if (Visited.insert(Pred).second)
593 Worklist.push_back(Pred);
599 IRBuilder<> Builder(LI);
600 Value *StoredVal = Candidate->getValueOperand();
601 Value *ExtractedVal =
602 extractValue(Builder, StoredVal, LI->getType(), DL, LoadOffset,
603 CandidateOffset, LoadSize);
606 LLVM_DEBUG(dbgs() <<
"SimpleGVN: Forwarding (BFS candidate)\n"
607 <<
" Store: " << *Candidate <<
"\n"
608 <<
" Load: " << *LI <<
"\n");
609 LI->replaceAllUsesWith(ExtractedVal);
610 LI->eraseFromParent();
621class SimpleGVN final :
public FunctionPass {
624 SimpleGVN() : FunctionPass(ID) {}
626 void getAnalysisUsage(AnalysisUsage &AU)
const override {
627 AU.addRequired<DominatorTreeWrapperPass>();
628 AU.addRequired<AAResultsWrapperPass>();
629 AU.addRequired<TargetLibraryInfoWrapperPass>();
632 bool runOnFunction(Function &F)
override {
633 auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
634 auto &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
635 auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
636 const DataLayout &DL = F.getParent()->getDataLayout();
637 return simplifyGVN(F, DT, DL, AA, TLI);
649char SimpleGVN::ID = 0;
651static RegisterPass<SimpleGVN>
X(
"simple-gvn",
652 "GVN-like load forwarding optimization");
655 FunctionAnalysisManager &FAM) {
656 bool Changed =
false;
657 const DataLayout &DL = F.getParent()->getDataLayout();
658 Changed = simplifyGVN(F, FAM.getResult<DominatorTreeAnalysis>(F), DL,
659 FAM.getResult<AAManager>(F),
660 FAM.getResult<TargetLibraryAnalysis>(F));
661 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
664llvm::AnalysisKey SimpleGVNNewPM::Key;
static RegisterPass< SimpleGVN > X("simple-gvn", "GVN-like load forwarding optimization")
FunctionPass * createSimpleGVNPass()
void LLVMAddSimpleGVNPass(LLVMPassManagerRef PM)
bool writesToMemoryReadBy(const TypeResults *TR, llvm::AAResults &AA, llvm::TargetLibraryInfo &TLI, llvm::Instruction *maybeReader, llvm::Instruction *maybeWriter)
Return whether maybeReader can read from memory written to by maybeWriter.
static llvm::PointerType * getPointerType(llvm::Type *T, unsigned AddressSpace=0)
Pointer into the given address space.
static bool isNoAlias(const llvm::CallBase *call)
static bool isNoCapture(const llvm::CallBase *call, size_t idx)
llvm::PreservedAnalyses Result
Result run(llvm::Function &F, llvm::FunctionAnalysisManager &FAM)