FunctionUtils.cpp File Reference

#include "FunctionUtils.h"
#include "DiffeGradientUtils.h"
#include "EnzymeLogic.h"
#include "GradientUtils.h"
#include "LibraryFuncs.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Passes/PassBuilder.h"
#include "llvm/ADT/APSInt.h"
#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/SetOperations.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/LazyValueInfo.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryDependenceAnalysis.h"
#include "llvm/Analysis/MemorySSA.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include <set>
#include "llvm/Analysis/CFLSteensAliasAnalysis.h"
#include "llvm/Analysis/DependenceAnalysis.h"
#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
#include "llvm/CodeGen/UnreachableBlockElim.h"
#include "llvm/Analysis/PhiValues.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScopedNoAliasAA.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Support/TimeProfiler.h"
#include "llvm/Transforms/IPO/FunctionAttrs.h"
#include "llvm/Transforms/Utils/Mem2Reg.h"
#include "llvm/Transforms/Utils.h"
#include "llvm/Transforms/InstCombine/InstCombine.h"
#include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
#include "llvm/Transforms/Scalar/DCE.h"
#include "llvm/Transforms/Scalar/DeadStoreElimination.h"
#include "llvm/Transforms/Scalar/EarlyCSE.h"
#include "llvm/Transforms/Scalar/GVN.h"
#include "llvm/Transforms/Scalar/IndVarSimplify.h"
#include "llvm/Transforms/Scalar/InstSimplifyPass.h"
#include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
#include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
#include "llvm/Transforms/Scalar/SROA.h"
#include "llvm/Transforms/Scalar/SimplifyCFG.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/LCSSA.h"
#include "llvm/Transforms/Utils/LowerInvoke.h"
#include "llvm/Transforms/Scalar/LoopDeletion.h"
#include "llvm/Transforms/Scalar/LoopRotation.h"
#include "llvm/Transforms/Utils/CodeExtractor.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/IR/LegacyPassManager.h"
#include "llvm/Transforms/IPO/PassManagerBuilder.h"
#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include <optional>
#include "CacheUtility.h"
#include "Utils.h"

Go to the source code of this file.

Classes
struct	compare_insts
class	DominatorOrderSet
struct	ConstraintComparator
struct	ConstraintContext
class	Constraints

Macros
#define	addAttribute   addAttributeAtIndex
#define	removeAttribute   removeAttributeAtIndex
#define	getAttribute   getAttributeAtIndex
#define	hasAttribute   hasAttributeAtIndex
#define	DEBUG_TYPE   "enzyme"

Typedefs
typedef DominatorOrderSet	QueueType

Enumerations
enum	RecurType { MaybeRecursive = 1 , NotRecursive = 2 , DefinitelyRecursive = 3 }

Functions
cl::opt< bool >	EnzymePreopt ("enzyme-preopt", cl::init(true), cl::Hidden, cl::desc("Run enzyme preprocessing optimizations"))
cl::opt< bool >	EnzymeInline ("enzyme-inline", cl::init(false), cl::Hidden, cl::desc("Force inlining of autodiff"))
cl::opt< int >	EnzymePostInlineOpt ("enzyme-post-inline-opt", cl::init(0), cl::Hidden, cl::desc("Force inlining of autodiff"))
cl::opt< bool >	EnzymeNoAlias ("enzyme-noalias", cl::init(false), cl::Hidden, cl::desc("Force noalias of autodiff"))
cl::opt< bool >	EnzymeAggressiveAA ("enzyme-aggressive-aa", cl::init(false), cl::Hidden, cl::desc("Use more unstable but aggressive LLVM AA"))
cl::opt< bool >	EnzymeLowerGlobals ("enzyme-lower-globals", cl::init(false), cl::Hidden, cl::desc("Lower globals to locals assuming the global values are not " "needed outside of this gradient"))
cl::opt< int >	EnzymeInlineCount ("enzyme-inline-count", cl::init(10000), cl::Hidden, cl::desc("Limit of number of functions to inline"))
cl::opt< bool >	EnzymeCoalese ("enzyme-coalese", cl::init(false), cl::Hidden, cl::desc("Whether to coalese memory allocations"))
static cl::opt< bool >	EnzymePHIRestructure ("enzyme-phi-restructure", cl::init(false), cl::Hidden, cl::desc("Whether to restructure phi's to have better unwrap behavior"))
cl::opt< bool >	EnzymeNameInstructions ("enzyme-name-instructions", cl::init(false), cl::Hidden, cl::desc("Have enzyme name all instructions"))
cl::opt< bool >	EnzymeSelectOpt ("enzyme-select-opt", cl::init(true), cl::Hidden, cl::desc("Run Enzyme select optimization"))
cl::opt< bool >	EnzymeAutoSparsity ("enzyme-auto-sparsity", cl::init(false), cl::Hidden, cl::desc("Run Enzyme auto sparsity"))
cl::opt< int >	EnzymePostOptLevel ("enzyme-post-opt-level", cl::init(0), cl::Hidden, cl::desc("Post optimization level within Enzyme differentiated function"))
cl::opt< bool >	EnzymeAlwaysInlineDiff ("enzyme-always-inline", cl::init(false), cl::Hidden, cl::desc("Mark generated functions as always-inline"))
bool	couldFunctionArgumentCapture (llvm::CallInst *CI, llvm::Value *val)
	Is the use of value val as an argument of call CI potentially captured.
static bool	IsFunctionRecursive (Function *F, std::map< const Function *, RecurType > &Results)
	Return whether this function eventually calls itself.
static bool	OnlyUsedInOMP (AllocaInst *AI)
void	RecursivelyReplaceAddressSpace (SmallVector< std::tuple< Value *, Value *, Instruction * >, 1 > &Todo, SmallVector< Instruction *, 1 > &toErase, bool legal)
void	RecursivelyReplaceAddressSpace (Value *AI, Value *rep, bool legal)
static void	UpgradeAllocasToMallocs (Function *NewF, DerivativeMode mode, SmallPtrSetImpl< llvm::BasicBlock * > &Unreachable)
	Convert necessary stack allocations into mallocs for use in the reverse pass.
static AllocaInst *	OldAllocationSize (Value *Ptr, CallInst *Loc, Function *NewF, IntegerType *T, const std::map< CallInst *, Value * > &reallocSizes)
void	simplifyExtractions (Function *NewF)
Function *	CreateMPIWrapper (Function *F)
static void	SimplifyMPIQueries (Function &NewF, FunctionAnalysisManager &FAM)
static void	ForceRecursiveInlining (Function *NewF, size_t Limit)
	Perform recursive inlinining on NewF up to the given limit.
void	CanonicalizeLoops (Function *F, FunctionAnalysisManager &FAM)
void	RemoveRedundantPHI (Function *F, FunctionAnalysisManager &FAM)
void	setFullWillReturn (Function *NewF)
void	SplitPHIs (llvm::Function &F)
bool	DetectPointerArgOfFn (llvm::Function &F, SmallPtrSetImpl< Function * > &calls_todo)
bool	DetectNoUnwindOfFn (llvm::Function &F, SmallPtrSetImpl< Function * > &calls_todo)
bool	DetectReadonlyOrThrowFn (llvm::Function &F, SmallPtrSetImpl< Function * > &calls_todo, llvm::TargetLibraryInfo &TLI, bool &local)
bool	DetectReadonlyOrThrow (Module &M)
FunctionType *	getFunctionTypeForClone (llvm::FunctionType *FTy, DerivativeMode mode, unsigned width, llvm::Type *additionalArg, llvm::ArrayRef< DIFFE_TYPE > constant_args, bool diffeReturnArg, bool returnTape, bool returnPrimal, bool returnShadow)
void	CoaleseTrivialMallocs (Function &F, DominatorTree &DT)
void	SelectOptimization (Function *F)
void	ReplaceFunctionImplementation (Module &M)
static bool	isNot (Value *a, Value *b)
bool	directlySparse (Value *z)
Function *	getProductIntrinsic (llvm::Module &M, llvm::Type *T)
Function *	getSumIntrinsic (llvm::Module &M, llvm::Type *T)
CallInst *	isProduct (llvm::Value *v)
CallInst *	isSum (llvm::Value *v)
SmallVector< Value *, 1 >	callOperands (llvm::CallBase *CB)
bool	guaranteedDataDependent (Value *z)
std::optional< std::string >	fixSparse_inner (Instruction *cur, llvm::Function &F, QueueType &Q, DominatorTree &DT, ScalarEvolution &SE, LoopInfo &LI, const DataLayout &DL)
raw_ostream &	operator<< (raw_ostream &os, const Constraints &c)
bool	cannotDependOnLoopIV (const SCEV *S, const Loop *L)
const SCEV *	evaluateAtLoopIter (const SCEV *V, ScalarEvolution &SE, const Loop *find, const SCEV *replace)
void	dump (const Constraints &c)
void	dump (std::shared_ptr< const Constraints > c)
std::shared_ptr< const Constraints >	getSparseConditions (bool &legal, Value *val, std::shared_ptr< const Constraints > defaultFloat, Instruction *scope, const ConstraintContext &ctx)
void	fixSparseIndices (llvm::Function &F, llvm::FunctionAnalysisManager &FAM, SetVector< BasicBlock * > &toDenseBlocks)
void	replaceToDense (llvm::CallBase *CI, bool replaceAll, llvm::Function *F, const llvm::DataLayout &DL)
bool	LowerSparsification (llvm::Function *F, bool replaceAll)
	Lower __enzyme_todense, returning if changed.

Variables
constexpr bool	SparseDebug = false

Macro Definition Documentation

addAttribute

#define addAttribute   addAttributeAtIndex

Definition at line 118 of file FunctionUtils.cpp.

DEBUG_TYPE

#define DEBUG_TYPE   "enzyme"

Definition at line 123 of file FunctionUtils.cpp.

getAttribute

#define getAttribute   getAttributeAtIndex

Definition at line 120 of file FunctionUtils.cpp.

hasAttribute

#define hasAttribute   hasAttributeAtIndex

Definition at line 121 of file FunctionUtils.cpp.

removeAttribute

#define removeAttribute   removeAttributeAtIndex

Definition at line 119 of file FunctionUtils.cpp.

Typedef Documentation

QueueType

typedef DominatorOrderSet QueueType

Definition at line 3575 of file FunctionUtils.cpp.

Enumeration Type Documentation

RecurType

enum RecurType

Enumerator
MaybeRecursive
NotRecursive
DefinitelyRecursive

Definition at line 221 of file FunctionUtils.cpp.

Function Documentation

callOperands()

SmallVector< Value *, 1 > callOperands

(

llvm::CallBase *

CB

)

Definition at line 3643 of file FunctionUtils.cpp.

Referenced by fixSparse_inner(), fixSparseIndices(), and guaranteedDataDependent().

cannotDependOnLoopIV()

bool cannotDependOnLoopIV	(	const SCEV *	S,
		const Loop *	L )

Definition at line 7040 of file FunctionUtils.cpp.

References cannotDependOnLoopIV().

Referenced by Constraints::andB(), cannotDependOnLoopIV(), evaluateAtLoopIter(), and getSparseConditions().

CanonicalizeLoops()

void CanonicalizeLoops	(	Function *	F,
		FunctionAnalysisManager &	FAM )

Definition at line 1452 of file FunctionUtils.cpp.

References InsertNewCanonicalIV(), and RemoveRedundantIVs().

Referenced by PreProcessCache::preprocessForClone().

CoaleseTrivialMallocs()

void CoaleseTrivialMallocs	(	Function &	F,
		DominatorTree &	DT )

Definition at line 3235 of file FunctionUtils.cpp.

References hasMetadata().

Referenced by PreProcessCache::optimizeIntermediate().

couldFunctionArgumentCapture()

bool couldFunctionArgumentCapture	(	llvm::CallInst *	CI,
		llvm::Value *	val )

Is the use of value val as an argument of call CI potentially captured.

Definition at line 182 of file FunctionUtils.cpp.

Referenced by PreProcessCache::preprocessForClone().

CreateMPIWrapper()

Function * CreateMPIWrapper

(

Function *

F

)

Definition at line 1240 of file FunctionUtils.cpp.

Referenced by SimplifyMPIQueries().

DetectNoUnwindOfFn()

bool DetectNoUnwindOfFn	(	llvm::Function &	F,
		SmallPtrSetImpl< Function * > &	calls_todo )

Definition at line 1827 of file FunctionUtils.cpp.

Referenced by DetectReadonlyOrThrow().

DetectPointerArgOfFn()

bool DetectPointerArgOfFn	(	llvm::Function &	F,
		SmallPtrSetImpl< Function * > &	calls_todo )

Definition at line 1667 of file FunctionUtils.cpp.

References addFunctionNoCapture(), getFuncNameFromCall(), isNoCapture(), isPointerArithmeticInst(), isReadOnly(), and isWriteOnly().

Referenced by DetectReadonlyOrThrow().

DetectReadonlyOrThrow()

bool DetectReadonlyOrThrow

(

Module &

M

)

Definition at line 2079 of file FunctionUtils.cpp.

References DetectNoUnwindOfFn(), DetectPointerArgOfFn(), and DetectReadonlyOrThrowFn().

Referenced by EnzymeDetectReadonlyOrThrow().

DetectReadonlyOrThrowFn()

bool DetectReadonlyOrThrowFn	(	llvm::Function &	F,
		SmallPtrSetImpl< Function * > &	calls_todo,
		llvm::TargetLibraryInfo &	TLI,
		bool &	local )

Definition at line 1867 of file FunctionUtils.cpp.

References EmitWarning(), EnzymeJuliaAddrLoad, EnzymePrintPerf, getBaseObject(), getFuncNameFromCall(), getGuaranteedUnreachable(), hasMetadata(), isAllocationCall(), isDebugFunction(), isLocalReadOnlyOrThrow(), isReadOnlyOrThrow(), and notCaptured().

Referenced by DetectReadonlyOrThrow(), and PreProcessCache::preprocessForClone().

directlySparse()

bool directlySparse

(

Value *

z

)

Definition at line 3555 of file FunctionUtils.cpp.

Referenced by fixSparse_inner().

dump() [1/2]

void dump

(

const Constraints &

c

)

Definition at line 7810 of file FunctionUtils.cpp.

dump() [2/2]

void dump

(

std::shared_ptr< const Constraints >

c

)

Definition at line 7811 of file FunctionUtils.cpp.

EnzymeAggressiveAA()

cl::opt< bool > EnzymeAggressiveAA	(	"enzyme-aggressive-aa"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Use more unstable but aggressive LLVM AA")	)

Referenced by PreProcessCache::PreProcessCache().

EnzymeAlwaysInlineDiff()

cl::opt< bool > EnzymeAlwaysInlineDiff	(	"enzyme-always-inline"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Mark generated functions as always-inline")	)

EnzymeAutoSparsity()

cl::opt< bool > EnzymeAutoSparsity	(	"enzyme-auto-sparsity"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Run Enzyme auto sparsity")	)

Referenced by LowerSparsification().

EnzymeCoalese()

cl::opt< bool > EnzymeCoalese	(	"enzyme-coalese"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Whether to coalese memory allocations")	)

Referenced by PreProcessCache::optimizeIntermediate().

EnzymeInline()

cl::opt< bool > EnzymeInline	(	"enzyme-inline"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Force inlining of autodiff")	)

Referenced by PreProcessCache::preprocessForClone().

EnzymeInlineCount()

cl::opt< int > EnzymeInlineCount	(	"enzyme-inline-count"	,
		cl::init(10000)	,
		cl::Hidden	,
		cl::desc("Limit of number of functions to inline")	)

Referenced by PreProcessCache::preprocessForClone().

EnzymeLowerGlobals()

cl::opt< bool > EnzymeLowerGlobals	(	"enzyme-lower-globals"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Lower globals to locals assuming the global values are not " "needed outside of this gradient")	)

Referenced by PreProcessCache::preprocessForClone().

EnzymeNameInstructions()

cl::opt< bool > EnzymeNameInstructions	(	"enzyme-name-instructions"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Have enzyme name all instructions")	)

Referenced by PreProcessCache::preprocessForClone().

EnzymeNoAlias()

cl::opt< bool > EnzymeNoAlias	(	"enzyme-noalias"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Force noalias of autodiff")	)

Referenced by PreProcessCache::preprocessForClone().

EnzymePHIRestructure()

static cl::opt< bool > EnzymePHIRestructure ( "enzyme-phi-restructure" , cl::init(false) , cl::Hidden , cl::desc("Whether to restructure phi's to have better unwrap behavior")  )

static

Referenced by PreProcessCache::preprocessForClone().

EnzymePostInlineOpt()

cl::opt< int > EnzymePostInlineOpt	(	"enzyme-post-inline-opt"	,
		cl::init(0)	,
		cl::Hidden	,
		cl::desc("Force inlining of autodiff")	)

Referenced by PreProcessCache::preprocessForClone().

EnzymePostOptLevel()

cl::opt< int > EnzymePostOptLevel	(	"enzyme-post-opt-level"	,
		cl::init(0)	,
		cl::Hidden	,
		cl::desc("Post optimization level within Enzyme differentiated function")	)

Referenced by PreProcessCache::optimizeIntermediate().

EnzymePreopt()

cl::opt< bool > EnzymePreopt	(	"enzyme-preopt"	,
		cl::init(true)	,
		cl::Hidden	,
		cl::desc("Run enzyme preprocessing optimizations")	)

Referenced by PreProcessCache::preprocessForClone().

EnzymeSelectOpt()

cl::opt< bool > EnzymeSelectOpt	(	"enzyme-select-opt"	,
		cl::init(true)	,
		cl::Hidden	,
		cl::desc("Run Enzyme select optimization")	)

Referenced by PreProcessCache::optimizeIntermediate().

evaluateAtLoopIter()

const SCEV * evaluateAtLoopIter	(	const SCEV *	V,
		ScalarEvolution &	SE,
		const Loop *	find,
		const SCEV *	replace )

Definition at line 7086 of file FunctionUtils.cpp.

References cannotDependOnLoopIV(), and evaluateAtLoopIter().

Referenced by Constraints::andB(), and evaluateAtLoopIter().

fixSparse_inner()

std::optional< std::string > fixSparse_inner	(	Instruction *	cur,
		llvm::Function &	F,
		QueueType &	Q,
		DominatorTree &	DT,
		ScalarEvolution &	SE,
		LoopInfo &	LI,
		const DataLayout &	DL )

Definition at line 3687 of file FunctionUtils.cpp.

References callOperands(), DominatorOrderSet::contains(), directlySparse(), getFast(), getFunctionFromCall(), getProductIntrinsic(), getSumIntrinsic(), guaranteedDataDependent(), hasMetadata(), isNot(), isProduct(), isReadOnly(), isSum(), isZero(), and DominatorOrderSet::remove().

Referenced by fixSparseIndices().

fixSparseIndices()

void fixSparseIndices	(	llvm::Function &	F,
		llvm::FunctionAnalysisManager &	FAM,
		SetVector< BasicBlock * > &	toDenseBlocks )

Definition at line 8149 of file FunctionUtils.cpp.

References Constraints::all(), callOperands(), DominatorOrderSet::contains(), EmitFailure(), fixSparse_inner(), getFast(), getSparseConditions(), isReadOnly(), Constraints::none(), DominatorOrderSet::pop_back_val(), and startsWith().

Referenced by LowerSparsification().

ForceRecursiveInlining()

static void ForceRecursiveInlining ( Function * NewF, size_t Limit )

static

Perform recursive inlinining on NewF up to the given limit.

Definition at line 1410 of file FunctionUtils.cpp.

References IsFunctionRecursive(), and startsWith().

Referenced by PreProcessCache::preprocessForClone().

getFunctionTypeForClone()

FunctionType * getFunctionTypeForClone	(	llvm::FunctionType *	FTy,
		DerivativeMode	mode,
		unsigned	width,
		llvm::Type *	additionalArg,
		llvm::ArrayRef< DIFFE_TYPE >	constant_args,
		bool	diffeReturnArg,
		bool	returnTape,
		bool	returnPrimal,
		bool	returnShadow )

Definition at line 2936 of file FunctionUtils.cpp.

References DUP_ARG, DUP_NONEED, getDefaultAnonymousTapeType(), GradientUtils::getShadowType(), OUT_DIFF, and ReverseModeCombined.

Referenced by PreProcessCache::CloneFunctionWithReturns(), and AdjointGenerator::recursivelyHandleSubfunction().

getProductIntrinsic()

Function * getProductIntrinsic	(	llvm::Module &	M,
		llvm::Type *	T )

Definition at line 3577 of file FunctionUtils.cpp.

Referenced by fixSparse_inner().

getSparseConditions()

std::shared_ptr< const Constraints > getSparseConditions	(	bool &	legal,
		Value *	val,
		std::shared_ptr< const Constraints >	defaultFloat,
		Instruction *	scope,
		const ConstraintContext &	ctx )

Definition at line 7963 of file FunctionUtils.cpp.

References Constraints::all(), cannotDependOnLoopIV(), EmitFailure(), EmitWarning(), getSparseConditions(), ConstraintContext::loopToSolve, Constraints::make_compare(), Constraints::none(), ConstraintContext::SE, ConstraintContext::seen, and SparseDebug.

Referenced by fixSparseIndices(), getSparseConditions(), and Constraints::make_compare().

getSumIntrinsic()

Function * getSumIntrinsic	(	llvm::Module &	M,
		llvm::Type *	T )

Definition at line 3602 of file FunctionUtils.cpp.

Referenced by fixSparse_inner().

guaranteedDataDependent()

bool guaranteedDataDependent

(

Value *

z

)

Definition at line 3647 of file FunctionUtils.cpp.

References callOperands(), guaranteedDataDependent(), isProduct(), and isSum().

Referenced by fixSparse_inner(), and guaranteedDataDependent().

IsFunctionRecursive()

static bool IsFunctionRecursive ( Function * F, std::map< const Function *, RecurType > & Results )

static

Return whether this function eventually calls itself.

Definition at line 228 of file FunctionUtils.cpp.

References DefinitelyRecursive, IsFunctionRecursive(), MaybeRecursive, and NotRecursive.

Referenced by ForceRecursiveInlining(), and IsFunctionRecursive().

isNot()

static bool isNot ( Value * a, Value * b )

static

Definition at line 3470 of file FunctionUtils.cpp.

Referenced by fixSparse_inner().

isProduct()

CallInst * isProduct

(

llvm::Value *

v

)

Definition at line 3627 of file FunctionUtils.cpp.

References getFunctionFromCall(), and startsWith().

Referenced by fixSparse_inner(), and guaranteedDataDependent().

isSum()

CallInst * isSum

(

llvm::Value *

v

)

Definition at line 3635 of file FunctionUtils.cpp.

References getFunctionFromCall(), and startsWith().

Referenced by fixSparse_inner(), guaranteedDataDependent(), and AdjointGenerator::handleMPI().

LowerSparsification()

bool LowerSparsification	(	llvm::Function *	F,
		bool	replaceAll )

Lower __enzyme_todense, returning if changed.

Definition at line 8936 of file FunctionUtils.cpp.

References contains(), EnzymeAutoSparsity(), fixSparseIndices(), getFuncNameFromCall(), and replaceToDense().

Referenced by EnzymeLowerSparsification().

OldAllocationSize()

static AllocaInst * OldAllocationSize ( Value * Ptr, CallInst * Loc, Function * NewF, IntegerType * T, const std::map< CallInst *, Value * > & reallocSizes )

inlinestatic

Definition at line 785 of file FunctionUtils.cpp.

References addFunctionNoCapture(), EmitFailure(), and simplifyLoad().

Referenced by PreProcessCache::ReplaceReallocs().

OnlyUsedInOMP()

static bool OnlyUsedInOMP ( AllocaInst * AI )

inlinestatic

Definition at line 266 of file FunctionUtils.cpp.

Referenced by UpgradeAllocasToMallocs().

operator<<()

raw_ostream & operator<<	(	raw_ostream &	os,
		const Constraints &	c )

Definition at line 7819 of file FunctionUtils.cpp.

References Constraints::All, Constraints::Compare, Constraints::Intersect, Constraints::isEqual, Constraints::Loop, Constraints::node, Constraints::None, Constraints::ty, Constraints::Union, and Constraints::values.

RecursivelyReplaceAddressSpace() [1/2]

void RecursivelyReplaceAddressSpace	(	SmallVector< std::tuple< Value *, Value *, Instruction * >, 1 > &	Todo,
		SmallVector< Instruction *, 1 > &	toErase,
		bool	legal )

Definition at line 289 of file FunctionUtils.cpp.

References CustomErrorHandler, EmitFailure(), EnzymeJuliaAddrLoad, getBaseObject(), getIntrinsicDeclaration(), getJuliaObjects(), InternalError, isIntelSubscriptIntrinsic(), and PostCacheStore().

Referenced by RecursivelyReplaceAddressSpace(), and UpgradeAllocasToMallocs().

RecursivelyReplaceAddressSpace() [2/2]

void RecursivelyReplaceAddressSpace	(	Value *	AI,
		Value *	rep,
		bool	legal )

Definition at line 626 of file FunctionUtils.cpp.

References RecursivelyReplaceAddressSpace().

RemoveRedundantPHI()

void RemoveRedundantPHI	(	Function *	F,
		FunctionAnalysisManager &	FAM )

Definition at line 1481 of file FunctionUtils.cpp.

Referenced by PreProcessCache::preprocessForClone().

ReplaceFunctionImplementation()

void ReplaceFunctionImplementation

(

Module &

M

)

Definition at line 3356 of file FunctionUtils.cpp.

Referenced by EnzymeReplaceFunctionImplementation().

replaceToDense()

void replaceToDense	(	llvm::CallBase *	CI,
		bool	replaceAll,
		llvm::Function *	F,
		const llvm::DataLayout &	DL )

Definition at line 8666 of file FunctionUtils.cpp.

References EmitFailure(), getBaseObject(), getFuncNameFromCall(), and getUnqual().

Referenced by LowerSparsification().

SelectOptimization()

void SelectOptimization

(

Function *

F

)

Definition at line 3330 of file FunctionUtils.cpp.

Referenced by PreProcessCache::optimizeIntermediate().

setFullWillReturn()

void setFullWillReturn

(

Function *

NewF

)

Definition at line 1581 of file FunctionUtils.cpp.

Referenced by PreProcessCache::preprocessForClone().

simplifyExtractions()

void simplifyExtractions

(

Function *

NewF

)

Definition at line 1008 of file FunctionUtils.cpp.

References GradientUtils::extractMeta().

Referenced by PreProcessCache::LowerAllocAddr().

SimplifyMPIQueries()

static void SimplifyMPIQueries ( Function & NewF, FunctionAnalysisManager & FAM )

static

Definition at line 1290 of file FunctionUtils.cpp.

References addCallSiteNoCapture(), CreateMPIWrapper(), getFuncName(), and getUnqual().

Referenced by PreProcessCache::preprocessForClone().

SplitPHIs()

void SplitPHIs

(

llvm::Function &

F

)

Definition at line 1596 of file FunctionUtils.cpp.

References GradientUtils::extractMeta().

Referenced by PreProcessCache::preprocessForClone().

UpgradeAllocasToMallocs()

static void UpgradeAllocasToMallocs ( Function * NewF, DerivativeMode mode, SmallPtrSetImpl< llvm::BasicBlock * > & Unreachable )

inlinestatic

Convert necessary stack allocations into mallocs for use in the reverse pass.

Specifically if we're not topLevel all allocations must be upgraded Even if topLevel any allocations that aren't in the entry block (and therefore may not be reachable in the reverse pass) must be upgraded.

Definition at line 643 of file FunctionUtils.cpp.

References CreateAllocation(), EnzymeZeroCache, OnlyUsedInOMP(), RecursivelyReplaceAddressSpace(), and ReverseModeCombined.

Referenced by PreProcessCache::preprocessForClone().

Variable Documentation

SparseDebug

bool SparseDebug = false

constexpr

Definition at line 7961 of file FunctionUtils.cpp.

Referenced by getSparseConditions().