#include "FunctionUtils.h"
#include "DiffeGradientUtils.h"
#include "EnzymeLogic.h"
#include "GradientUtils.h"
#include "LibraryFuncs.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Passes/PassBuilder.h"
#include "llvm/ADT/APSInt.h"
#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/SetOperations.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/LazyValueInfo.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryDependenceAnalysis.h"
#include "llvm/Analysis/MemorySSA.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include <set>
#include "llvm/Analysis/CFLSteensAliasAnalysis.h"
#include "llvm/Analysis/DependenceAnalysis.h"
#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
#include "llvm/CodeGen/UnreachableBlockElim.h"
#include "llvm/Analysis/PhiValues.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScopedNoAliasAA.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Support/TimeProfiler.h"
#include "llvm/Transforms/IPO/FunctionAttrs.h"
#include "llvm/Transforms/Utils/Mem2Reg.h"
#include "llvm/Transforms/Utils.h"
#include "llvm/Transforms/InstCombine/InstCombine.h"
#include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
#include "llvm/Transforms/Scalar/DCE.h"
#include "llvm/Transforms/Scalar/DeadStoreElimination.h"
#include "llvm/Transforms/Scalar/EarlyCSE.h"
#include "llvm/Transforms/Scalar/GVN.h"
#include "llvm/Transforms/Scalar/IndVarSimplify.h"
#include "llvm/Transforms/Scalar/InstSimplifyPass.h"
#include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
#include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
#include "llvm/Transforms/Scalar/SROA.h"
#include "llvm/Transforms/Scalar/SimplifyCFG.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/LCSSA.h"
#include "llvm/Transforms/Utils/LowerInvoke.h"
#include "llvm/Transforms/Scalar/LoopDeletion.h"
#include "llvm/Transforms/Scalar/LoopRotation.h"
#include "llvm/Transforms/Utils/CodeExtractor.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/IR/LegacyPassManager.h"
#include "llvm/Transforms/IPO/PassManagerBuilder.h"
#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include <map>
#include <optional>
#include "CacheUtility.h"
#include "Utils.h"

Classes
class	ReadOnlyOrThrowDetector

struct	compare_insts

class	DominatorOrderSet

struct	ConstraintComparator

struct	ConstraintContext

class	Constraints

Macros
#define	addAttribute addAttributeAtIndex

#define	removeAttribute removeAttributeAtIndex

#define	getAttribute getAttributeAtIndex

#define	hasAttribute hasAttributeAtIndex

#define	DEBUG_TYPE "enzyme"

Typedefs
typedef DominatorOrderSet	QueueType

Enumerations
enum	RecurType { MaybeRecursive = 1 , NotRecursive = 2 , DefinitelyRecursive = 3 }

Functions
cl::opt< bool >	EnzymePreopt ("enzyme-preopt", cl::init(true), cl::Hidden, cl::desc("Run enzyme preprocessing optimizations"))

cl::opt< bool >	EnzymeInline ("enzyme-inline", cl::init(false), cl::Hidden, cl::desc("Force inlining of autodiff"))

cl::opt< int >	EnzymePostInlineOpt ("enzyme-post-inline-opt", cl::init(0), cl::Hidden, cl::desc("Force inlining of autodiff"))

cl::opt< bool >	EnzymeNoAlias ("enzyme-noalias", cl::init(false), cl::Hidden, cl::desc("Force noalias of autodiff"))

cl::opt< bool >	EnzymeAggressiveAA ("enzyme-aggressive-aa", cl::init(false), cl::Hidden, cl::desc("Use more unstable but aggressive LLVM AA"))

cl::opt< bool >	EnzymeLowerGlobals ("enzyme-lower-globals", cl::init(false), cl::Hidden, cl::desc("Lower globals to locals assuming the global values are not " "needed outside of this gradient"))

cl::opt< int >	EnzymeInlineCount ("enzyme-inline-count", cl::init(10000), cl::Hidden, cl::desc("Limit of number of functions to inline"))

cl::opt< bool >	EnzymeCoalese ("enzyme-coalese", cl::init(false), cl::Hidden, cl::desc("Whether to coalese memory allocations"))

static cl::opt< bool >	EnzymePHIRestructure ("enzyme-phi-restructure", cl::init(false), cl::Hidden, cl::desc("Whether to restructure phi's to have better unwrap behavior"))

cl::opt< bool >	EnzymeNameInstructions ("enzyme-name-instructions", cl::init(false), cl::Hidden, cl::desc("Have enzyme name all instructions"))

cl::opt< bool >	EnzymeSelectOpt ("enzyme-select-opt", cl::init(true), cl::Hidden, cl::desc("Run Enzyme select optimization"))

cl::opt< bool >	EnzymeAutoSparsity ("enzyme-auto-sparsity", cl::init(false), cl::Hidden, cl::desc("Run Enzyme auto sparsity"))

cl::opt< int >	EnzymePostOptLevel ("enzyme-post-opt-level", cl::init(0), cl::Hidden, cl::desc("Post optimization level within Enzyme differentiated function"))

cl::opt< bool >	EnzymeAlwaysInlineDiff ("enzyme-always-inline", cl::init(false), cl::Hidden, cl::desc("Mark generated functions as always-inline"))

bool	couldFunctionArgumentCapture (llvm::CallInst CI, llvm::Value val)
	Is the use of value val as an argument of call CI potentially captured.

static bool	IsFunctionRecursive (Function F, std::map< const Function , RecurType > &Results)
	Return whether this function eventually calls itself.

static bool	OnlyUsedInOMP (AllocaInst *AI)

void	RecursivelyReplaceAddressSpace (SmallVector< std::tuple< Value , Value , Instruction * >, 1 > &Todo, SmallVector< Instruction *, 1 > &toErase, bool legal)

void	RecursivelyReplaceAddressSpace (Value AI, Value rep, bool legal)

static void	UpgradeAllocasToMallocs (Function NewF, DerivativeMode mode, SmallPtrSetImpl< llvm::BasicBlock > &Unreachable)
	Convert necessary stack allocations into mallocs for use in the reverse pass.

static AllocaInst *	OldAllocationSize (Value Ptr, CallInst Loc, Function NewF, IntegerType T, const std::map< CallInst , Value > &reallocSizes)

void	simplifyExtractions (Function *NewF)

Function *	CreateMPIWrapper (Function *F)

static void	SimplifyMPIQueries (Function &NewF, FunctionAnalysisManager &FAM)

static void	ForceRecursiveInlining (Function *NewF, size_t Limit)
	Perform recursive inlinining on NewF up to the given limit.

void	CanonicalizeLoops (Function *F, FunctionAnalysisManager &FAM)

void	RemoveRedundantPHI (Function *F, FunctionAnalysisManager &FAM)

void	setFullWillReturn (Function *NewF)

void	SplitPHIs (llvm::Function &F)

static bool	isReturnLikeParam (const Function &F, unsigned argno)

static bool	mayReadThroughCustomRule (Function &F, unsigned argno)

bool	DetectPointerArgOfFn (llvm::Function &F, SmallPtrSetImpl< Function * > &calls_todo)

bool	DetectNoUnwindOfFn (llvm::Function &F, SmallPtrSetImpl< Function * > &calls_todo)

bool	DetectReadonlyOrThrow (Module &M)

static bool	reachesAvoiding (BasicBlock From, BasicBlock To, BasicBlock *Avoid, const AbortIgnoringPostDom &PDT)
	Whether a returning path leads from the end of `From` back into `To` without passing through `Avoid`.

static bool	unifyReturnBlocks (Function &F)
	Route every `ret` of `F` through one new return block.

static MergeResult	MergeGCPreserveEnds (CallInst Begin, ArrayRef< CallInst > AllEnds, DominatorTree &DT, AbortIgnoringPostDom &PDT, bool &CFGChanged)
	Give `Begin` a single returning `llvm.julia.gc_preserve_end` that post-dominates it, replacing the several ends it currently has.

static bool	CanonicalizeGCPreserveEnds (Function &F, FunctionAnalysisManager &FAM)
	A pass such as JumpThreading is free to clone a basic block holding an `llvm.julia.gc_preserve_end`: the token it consumes may legally have many uses, and only the instruction producing a token blocks duplication.

FunctionType *	getFunctionTypeForClone (llvm::FunctionType FTy, DerivativeMode mode, unsigned width, llvm::Type additionalArg, llvm::ArrayRef< DIFFE_TYPE > constant_args, bool diffeReturnArg, bool returnTape, bool returnPrimal, bool returnShadow)

void	CoaleseTrivialMallocs (Function &F, DominatorTree &DT)

void	SelectOptimization (Function *F)

void	ReplaceFunctionImplementation (Module &M)

static bool	isNot (Value a, Value b)

bool	directlySparse (Value *z)

Function *	getProductIntrinsic (llvm::Module &M, llvm::Type *T)

Function *	getSumIntrinsic (llvm::Module &M, llvm::Type *T)

CallInst *	isProduct (llvm::Value *v)

CallInst *	isSum (llvm::Value *v)

SmallVector< Value *, 1 >	callOperands (llvm::CallBase *CB)

bool	guaranteedDataDependent (Value *z)

std::optional< std::string >	fixSparse_inner (Instruction *cur, llvm::Function &F, QueueType &Q, DominatorTree &DT, ScalarEvolution &SE, LoopInfo &LI, const DataLayout &DL)

raw_ostream &	operator<< (raw_ostream &os, const Constraints &c)

bool	cannotDependOnLoopIV (const SCEV S, const Loop L)

const SCEV *	evaluateAtLoopIter (const SCEV V, ScalarEvolution &SE, const Loop find, const SCEV *replace)

void	dump (const Constraints &c)

void	dump (std::shared_ptr< const Constraints > c)

std::shared_ptr< const Constraints >	getSparseConditions (bool &legal, Value val, std::shared_ptr< const Constraints > defaultFloat, Instruction scope, const ConstraintContext &ctx)

static void	lowerSparsePlaceholders (llvm::Function &F)

static void	eraseSparseInbounds (llvm::Function &F)

void	fixSparseIndices (llvm::Function &F, llvm::FunctionAnalysisManager &FAM, SetVector< BasicBlock * > &toDenseBlocks)

void	replaceToDense (llvm::CallBase CI, bool replaceAll, llvm::Function F, const llvm::DataLayout &DL)

bool	LowerSparsification (llvm::Function *F, bool replaceAll)
	Lower __enzyme_todense, returning if changed.

Variables
constexpr bool	SparseDebug = false

Macro Definition Documentation

◆ addAttribute

#define addAttribute addAttributeAtIndex

Definition at line 120 of file FunctionUtils.cpp.

◆ DEBUG_TYPE

#define DEBUG_TYPE "enzyme"

Definition at line 125 of file FunctionUtils.cpp.

◆ getAttribute

#define getAttribute getAttributeAtIndex

Definition at line 122 of file FunctionUtils.cpp.

◆ hasAttribute

#define hasAttribute hasAttributeAtIndex

Definition at line 123 of file FunctionUtils.cpp.

◆ removeAttribute

#define removeAttribute removeAttributeAtIndex

Definition at line 121 of file FunctionUtils.cpp.

Typedef Documentation

◆ QueueType

typedef DominatorOrderSet QueueType

Definition at line 4948 of file FunctionUtils.cpp.

Enumeration Type Documentation

◆ RecurType

enum RecurType

Enumerator
MaybeRecursive
NotRecursive
DefinitelyRecursive

Definition at line 223 of file FunctionUtils.cpp.

Function Documentation

◆ callOperands()

SmallVector< Value *, 1 > callOperands ( llvm::CallBase * CB )

Definition at line 5028 of file FunctionUtils.cpp.

Referenced by fixSparse_inner(), guaranteedDataDependent(), and lowerSparsePlaceholders().

◆ cannotDependOnLoopIV()

bool cannotDependOnLoopIV	(	const SCEV *	S,
		const Loop *	L )

Definition at line 8443 of file FunctionUtils.cpp.

References cannotDependOnLoopIV().

Referenced by Constraints::andB(), cannotDependOnLoopIV(), evaluateAtLoopIter(), and getSparseConditions().

◆ CanonicalizeGCPreserveEnds()

static bool CanonicalizeGCPreserveEnds	(	Function &	F,
		FunctionAnalysisManager &	FAM )

static

A pass such as JumpThreading is free to clone a basic block holding an llvm.julia.gc_preserve_end: the token it consumes may legally have many uses, and only the instruction producing a token blocks duplication.

That leaves a single llvm.julia.gc_preserve_begin paired with several ends.

Reverse mode mirrors a preserve region by emitting a gc_preserve_begin at the invert of the end and the matching gc_preserve_end at the invert of the begin, so it needs the end to post-dominate the begin. With several ends no mirrored begin dominates the mirrored end, and the differentiated function fails verification.

Restore the one-end invariant. The preserved region can only grow, which is always safe.

Definition at line 3532 of file FunctionUtils.cpp.

References getFuncNameFromCall(), getGuaranteedUnreachable(), MergeGCPreserveEnds(), and unifyReturnBlocks().

Referenced by PreProcessCache::preprocessForClone().

◆ CanonicalizeLoops()

void CanonicalizeLoops	(	Function *	F,
		FunctionAnalysisManager &	FAM )

Definition at line 1554 of file FunctionUtils.cpp.

References InsertNewCanonicalIV(), and RemoveRedundantIVs().

Referenced by PreProcessCache::preprocessForClone().

◆ CoaleseTrivialMallocs()

void CoaleseTrivialMallocs	(	Function &	F,
		DominatorTree &	DT )

Definition at line 4600 of file FunctionUtils.cpp.

References hasMetadata().

Referenced by PreProcessCache::optimizeIntermediate().

◆ couldFunctionArgumentCapture()

bool couldFunctionArgumentCapture	(	llvm::CallInst *	CI,
		llvm::Value *	val )

Is the use of value val as an argument of call CI potentially captured.

Definition at line 184 of file FunctionUtils.cpp.

Referenced by PreProcessCache::preprocessForClone().

◆ CreateMPIWrapper()

Function * CreateMPIWrapper ( Function * F )

Definition at line 1342 of file FunctionUtils.cpp.

Referenced by SimplifyMPIQueries().

◆ DetectNoUnwindOfFn()

bool DetectNoUnwindOfFn	(	llvm::Function &	F,
		SmallPtrSetImpl< Function * > &	calls_todo )

Definition at line 1981 of file FunctionUtils.cpp.

Referenced by DetectReadonlyOrThrow().

◆ DetectPointerArgOfFn()

bool DetectPointerArgOfFn	(	llvm::Function &	F,
		SmallPtrSetImpl< Function * > &	calls_todo )

Definition at line 1801 of file FunctionUtils.cpp.

References addFunctionNoCapture(), getFuncNameFromCall(), isNoCapture(), isPointerArithmeticInst(), isReadOnly(), isWriteOnly(), and mayReadThroughCustomRule().

Referenced by DetectReadonlyOrThrow().

◆ DetectReadonlyOrThrow()

bool DetectReadonlyOrThrow ( Module & M )

Definition at line 3061 of file FunctionUtils.cpp.

References DetectNoUnwindOfFn(), and DetectPointerArgOfFn().

Referenced by EnzymeDetectReadonlyOrThrow().

◆ directlySparse()

bool directlySparse ( Value * z )

Definition at line 4928 of file FunctionUtils.cpp.

Referenced by fixSparse_inner().

◆ dump() [1/2]

void dump ( const Constraints & c )

Definition at line 9213 of file FunctionUtils.cpp.

Referenced by mlir::enzyme::minCutCache().

◆ dump() [2/2]

void dump ( std::shared_ptr< const Constraints > c )

Definition at line 9214 of file FunctionUtils.cpp.

◆ EnzymeAggressiveAA()

cl::opt< bool > EnzymeAggressiveAA	(	"enzyme-aggressive-aa"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Use more unstable but aggressive LLVM AA")	)

Referenced by PreProcessCache::PreProcessCache().

◆ EnzymeAlwaysInlineDiff()

cl::opt< bool > EnzymeAlwaysInlineDiff	(	"enzyme-always-inline"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Mark generated functions as always-inline")	)

◆ EnzymeAutoSparsity()

cl::opt< bool > EnzymeAutoSparsity	(	"enzyme-auto-sparsity"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Run Enzyme auto sparsity")	)

Referenced by LowerSparsification().

◆ EnzymeCoalese()

cl::opt< bool > EnzymeCoalese	(	"enzyme-coalese"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Whether to coalese memory allocations")	)

Referenced by PreProcessCache::optimizeIntermediate().

◆ EnzymeInline()

cl::opt< bool > EnzymeInline	(	"enzyme-inline"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Force inlining of autodiff")	)

Referenced by PreProcessCache::preprocessForClone().

◆ EnzymeInlineCount()

cl::opt< int > EnzymeInlineCount	(	"enzyme-inline-count"	,
		cl::init(10000)	,
		cl::Hidden	,
		cl::desc("Limit of number of functions to inline")	)

Referenced by PreProcessCache::preprocessForClone().

◆ EnzymeLowerGlobals()

cl::opt< bool > EnzymeLowerGlobals	(	"enzyme-lower-globals"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Lower globals to locals assuming the global values are not " "needed outside of this gradient")	)

Referenced by PreProcessCache::preprocessForClone().

◆ EnzymeNameInstructions()

cl::opt< bool > EnzymeNameInstructions	(	"enzyme-name-instructions"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Have enzyme name all instructions")	)

Referenced by PreProcessCache::preprocessForClone().

◆ EnzymeNoAlias()

cl::opt< bool > EnzymeNoAlias	(	"enzyme-noalias"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Force noalias of autodiff")	)

Referenced by PreProcessCache::preprocessForClone().

◆ EnzymePHIRestructure()

static cl::opt< bool > EnzymePHIRestructure	(	"enzyme-phi-restructure"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Whether to restructure phi's to have better unwrap behavior")	)

static

Referenced by PreProcessCache::preprocessForClone().

◆ EnzymePostInlineOpt()

cl::opt< int > EnzymePostInlineOpt	(	"enzyme-post-inline-opt"	,
		cl::init(0)	,
		cl::Hidden	,
		cl::desc("Force inlining of autodiff")	)

Referenced by PreProcessCache::preprocessForClone().

◆ EnzymePostOptLevel()

cl::opt< int > EnzymePostOptLevel	(	"enzyme-post-opt-level"	,
		cl::init(0)	,
		cl::Hidden	,
		cl::desc("Post optimization level within Enzyme differentiated function")	)

Referenced by PreProcessCache::optimizeIntermediate().

◆ EnzymePreopt()

cl::opt< bool > EnzymePreopt	(	"enzyme-preopt"	,
		cl::init(true)	,
		cl::Hidden	,
		cl::desc("Run enzyme preprocessing optimizations")	)

Referenced by PreProcessCache::preprocessForClone().

◆ EnzymeSelectOpt()

cl::opt< bool > EnzymeSelectOpt	(	"enzyme-select-opt"	,
		cl::init(true)	,
		cl::Hidden	,
		cl::desc("Run Enzyme select optimization")	)

Referenced by PreProcessCache::optimizeIntermediate().

◆ eraseSparseInbounds()

static void eraseSparseInbounds ( llvm::Function & F )

static

Definition at line 9603 of file FunctionUtils.cpp.

Referenced by fixSparseIndices().

◆ evaluateAtLoopIter()

const SCEV * evaluateAtLoopIter	(	const SCEV *	V,
		ScalarEvolution &	SE,
		const Loop *	find,
		const SCEV *	replace )

Definition at line 8489 of file FunctionUtils.cpp.

References cannotDependOnLoopIV(), evaluateAtIterationWithoutExt(), and evaluateAtLoopIter().

Referenced by Constraints::andB(), and evaluateAtLoopIter().

◆ fixSparse_inner()

std::optional< std::string > fixSparse_inner	(	Instruction *	cur,
		llvm::Function &	F,
		QueueType &	Q,
		DominatorTree &	DT,
		ScalarEvolution &	SE,
		LoopInfo &	LI,
		const DataLayout &	DL )

Definition at line 5072 of file FunctionUtils.cpp.

References callOperands(), DominatorOrderSet::contains(), directlySparse(), getBranchCondition(), getFast(), getFirstNonPHI(), getFunctionFromCall(), getProductIntrinsic(), getSumIntrinsic(), guaranteedDataDependent(), hasMetadata(), isAnyBranch(), isConditionalBranch(), isNot(), isProduct(), isReadOnly(), isSum(), isZero(), moveBeforeInst(), and DominatorOrderSet::remove().

Referenced by fixSparseIndices().

◆ fixSparseIndices()

void fixSparseIndices	(	llvm::Function &	F,
		llvm::FunctionAnalysisManager &	FAM,
		SetVector< BasicBlock * > &	toDenseBlocks )

Definition at line 9615 of file FunctionUtils.cpp.

References Constraints::all(), DominatorOrderSet::contains(), EmitFailure(), eraseSparseInbounds(), fixSparse_inner(), getBranchCondition(), getFirstNonPHI(), getSparseConditions(), isAnyBranch(), isConditionalBranch(), isReadOnly(), isUnconditionalBranch(), lowerSparsePlaceholders(), Constraints::none(), DominatorOrderSet::pop_back_val(), and setBranchCondition().

Referenced by LowerSparsification().

◆ ForceRecursiveInlining()

static void ForceRecursiveInlining	(	Function *	NewF,
		size_t	Limit )

static

Perform recursive inlinining on NewF up to the given limit.

Definition at line 1512 of file FunctionUtils.cpp.

References IsFunctionRecursive(), and startsWith().

Referenced by PreProcessCache::preprocessForClone().

◆ getFunctionTypeForClone()

FunctionType * getFunctionTypeForClone	(	llvm::FunctionType *	FTy,
		DerivativeMode	mode,
		unsigned	width,
		llvm::Type *	additionalArg,
		llvm::ArrayRef< DIFFE_TYPE >	constant_args,
		bool	diffeReturnArg,
		bool	returnTape,
		bool	returnPrimal,
		bool	returnShadow )

Definition at line 4300 of file FunctionUtils.cpp.

References DUP_ARG, DUP_NONEED, getDefaultAnonymousTapeType(), GradientUtils::getShadowType(), OUT_DIFF, and ReverseModeCombined.

Referenced by PreProcessCache::CloneFunctionWithReturns(), and AdjointGenerator::recursivelyHandleSubfunction().

◆ getProductIntrinsic()

Function * getProductIntrinsic	(	llvm::Module &	M,
		llvm::Type *	T )

Definition at line 4950 of file FunctionUtils.cpp.

Referenced by fixSparse_inner().

◆ getSparseConditions()

std::shared_ptr< const Constraints > getSparseConditions	(	bool &	legal,
		Value *	val,
		std::shared_ptr< const Constraints >	defaultFloat,
		Instruction *	scope,
		const ConstraintContext &	ctx )

Definition at line 9366 of file FunctionUtils.cpp.

References Constraints::all(), cannotDependOnLoopIV(), EmitFailure(), EmitWarning(), getSparseConditions(), ConstraintContext::loopToSolve, Constraints::make_compare(), Constraints::none(), ConstraintContext::SE, ConstraintContext::seen, and SparseDebug.

Referenced by fixSparseIndices(), getSparseConditions(), and Constraints::make_compare().

◆ getSumIntrinsic()

Function * getSumIntrinsic	(	llvm::Module &	M,
		llvm::Type *	T )

Definition at line 4981 of file FunctionUtils.cpp.

Referenced by fixSparse_inner().

◆ guaranteedDataDependent()

bool guaranteedDataDependent ( Value * z )

Definition at line 5032 of file FunctionUtils.cpp.

References callOperands(), guaranteedDataDependent(), isProduct(), and isSum().

Referenced by fixSparse_inner(), and guaranteedDataDependent().

◆ IsFunctionRecursive()

static bool IsFunctionRecursive	(	Function *	F,
		std::map< const Function *, RecurType > &	Results )

static

Return whether this function eventually calls itself.

Definition at line 230 of file FunctionUtils.cpp.

References DefinitelyRecursive, IsFunctionRecursive(), MaybeRecursive, and NotRecursive.

Referenced by ForceRecursiveInlining(), and IsFunctionRecursive().

◆ isNot()

static bool isNot	(	Value *	a,
		Value *	b )

static

Definition at line 4843 of file FunctionUtils.cpp.

Referenced by fixSparse_inner().

◆ isProduct()

CallInst * isProduct ( llvm::Value * v )

Definition at line 5012 of file FunctionUtils.cpp.

References getFunctionFromCall(), and startsWith().

Referenced by fixSparse_inner(), and guaranteedDataDependent().

◆ isReturnLikeParam()

static bool isReturnLikeParam	(	const Function &	F,
		unsigned	argno )

static

Definition at line 1772 of file FunctionUtils.cpp.

Referenced by mayReadThroughCustomRule().

◆ isSum()

CallInst * isSum ( llvm::Value * v )

Definition at line 5020 of file FunctionUtils.cpp.

References getFunctionFromCall(), and startsWith().

Referenced by fixSparse_inner(), guaranteedDataDependent(), and AdjointGenerator::handleMPI().

◆ lowerSparsePlaceholders()

static void lowerSparsePlaceholders ( llvm::Function & F )

static

Definition at line 9555 of file FunctionUtils.cpp.

References callOperands(), getFast(), and startsWith().

Referenced by fixSparseIndices().

◆ LowerSparsification()

bool LowerSparsification	(	llvm::Function *	F,
		bool	replaceAll )

Lower __enzyme_todense, returning if changed.

Definition at line 10363 of file FunctionUtils.cpp.

References contains(), EnzymeAutoSparsity(), fixSparseIndices(), getFuncNameFromCall(), and replaceToDense().

Referenced by EnzymeLowerSparsification().

◆ mayReadThroughCustomRule()

static bool mayReadThroughCustomRule	(	Function &	F,
		unsigned	argno )

static

Definition at line 1793 of file FunctionUtils.cpp.

References getFuncName(), hasCustomRuleMetadata(), and isReturnLikeParam().

Referenced by DetectPointerArgOfFn().

◆ MergeGCPreserveEnds()

static MergeResult MergeGCPreserveEnds	(	CallInst *	Begin,
		ArrayRef< CallInst * >	AllEnds,
		DominatorTree &	DT,
		AbortIgnoringPostDom &	PDT,
		bool &	CFGChanged )

static

Give Begin a single returning llvm.julia.gc_preserve_end that post-dominates it, replacing the several ends it currently has.

Ends on abort-only paths are left as they are: those paths never return, and the reverse pass drops their blocks wholesale rather than visiting them. Returns NoMergePoint when only the function exit post-dominates the ends, and Failed for any other reason not to merge; the function is then left untouched but for a possible edge split.

Definition at line 3432 of file FunctionUtils.cpp.

References Args, and reachesAvoiding().

Referenced by CanonicalizeGCPreserveEnds().

◆ OldAllocationSize()

static AllocaInst * OldAllocationSize	(	Value *	Ptr,
		CallInst *	Loc,
		Function *	NewF,
		IntegerType *	T,
		const std::map< CallInst , Value > &	reallocSizes )

inlinestatic

Definition at line 877 of file FunctionUtils.cpp.

References addFunctionNoCapture(), EmitFailure(), and simplifyLoad().

Referenced by PreProcessCache::ReplaceReallocs().

◆ OnlyUsedInOMP()

static bool OnlyUsedInOMP ( AllocaInst * AI )

inlinestatic

Definition at line 268 of file FunctionUtils.cpp.

Referenced by UpgradeAllocasToMallocs().

◆ operator<<()

raw_ostream & operator<<	(	raw_ostream &	os,
		const Constraints &	c )

Definition at line 9222 of file FunctionUtils.cpp.

References Constraints::All, Constraints::Compare, Constraints::Intersect, Constraints::isEqual, Constraints::Loop, Constraints::node, Constraints::None, Constraints::ty, Constraints::Union, and Constraints::values.

◆ reachesAvoiding()

static bool reachesAvoiding	(	BasicBlock *	From,
		BasicBlock *	To,
		BasicBlock *	Avoid,
		const AbortIgnoringPostDom &	PDT )

static

Whether a returning path leads from the end of From back into To without passing through Avoid.

Definition at line 3380 of file FunctionUtils.cpp.

Referenced by MergeGCPreserveEnds().

◆ RecursivelyReplaceAddressSpace() [1/2]

void RecursivelyReplaceAddressSpace	(	SmallVector< std::tuple< Value , Value , Instruction * >, 1 > &	Todo,
		SmallVector< Instruction *, 1 > &	toErase,
		bool	legal )

Definition at line 291 of file FunctionUtils.cpp.

References changePointerAddrSpace(), CustomErrorHandler, EmitFailure(), EnzymeJuliaAddrLoad, getBaseObject(), getIntrinsicDeclaration(), getJuliaObjects(), getPointerType(), InternalError, isIntelSubscriptIntrinsic(), and PostCacheStore().

Referenced by RecursivelyReplaceAddressSpace(), and UpgradeAllocasToMallocs().

◆ RecursivelyReplaceAddressSpace() [2/2]

void RecursivelyReplaceAddressSpace	(	Value *	AI,
		Value *	rep,
		bool	legal )

Definition at line 714 of file FunctionUtils.cpp.

References RecursivelyReplaceAddressSpace().

◆ RemoveRedundantPHI()

void RemoveRedundantPHI	(	Function *	F,
		FunctionAnalysisManager &	FAM )

Definition at line 1583 of file FunctionUtils.cpp.

Referenced by PreProcessCache::preprocessForClone().

◆ ReplaceFunctionImplementation()

void ReplaceFunctionImplementation ( Module & M )

Definition at line 4729 of file FunctionUtils.cpp.

Referenced by EnzymeReplaceFunctionImplementation().

◆ replaceToDense()

void replaceToDense	(	llvm::CallBase *	CI,
		bool	replaceAll,
		llvm::Function *	F,
		const llvm::DataLayout &	DL )

Definition at line 10102 of file FunctionUtils.cpp.

References changePointerAddrSpace(), EmitFailure(), getBaseObject(), and getFuncNameFromCall().

Referenced by LowerSparsification().

◆ SelectOptimization()

void SelectOptimization ( Function * F )

Definition at line 4701 of file FunctionUtils.cpp.

References getBranchCondition(), isAnyBranch(), and isConditionalBranch().

Referenced by PreProcessCache::optimizeIntermediate().

◆ setFullWillReturn()

void setFullWillReturn ( Function * NewF )

Definition at line 1683 of file FunctionUtils.cpp.

Referenced by PreProcessCache::preprocessForClone().

◆ simplifyExtractions()

void simplifyExtractions ( Function * NewF )

Definition at line 1100 of file FunctionUtils.cpp.

References GradientUtils::extractMeta().

Referenced by PreProcessCache::LowerAllocAddr().

◆ SimplifyMPIQueries()

static void SimplifyMPIQueries	(	Function &	NewF,
		FunctionAnalysisManager &	FAM )

static

Definition at line 1392 of file FunctionUtils.cpp.

References addCallSiteNoCapture(), CreateMPIWrapper(), getFirstNonPHI(), getFuncName(), getPointerType(), and getUnqual().

Referenced by PreProcessCache::preprocessForClone().

◆ SplitPHIs()

void SplitPHIs ( llvm::Function & F )

Definition at line 1698 of file FunctionUtils.cpp.

References GradientUtils::extractMeta().

Referenced by PreProcessCache::preprocessForClone().

◆ unifyReturnBlocks()

static bool unifyReturnBlocks ( Function & F )

static

Route every ret of F through one new return block.

Unlike UnifyFunctionExitNodesPass this leaves the unreachables alone, which the augmented primal relies on to spot calls that never return.

Definition at line 3398 of file FunctionUtils.cpp.

Referenced by CanonicalizeGCPreserveEnds().

◆ UpgradeAllocasToMallocs()

static void UpgradeAllocasToMallocs	(	Function *	NewF,
		DerivativeMode	mode,
		SmallPtrSetImpl< llvm::BasicBlock * > &	Unreachable )

inlinestatic

Convert necessary stack allocations into mallocs for use in the reverse pass.

Specifically if we're not topLevel all allocations must be upgraded Even if topLevel any allocations that aren't in the entry block (and therefore may not be reachable in the reverse pass) must be upgraded.

Definition at line 733 of file FunctionUtils.cpp.

References CreateAllocation(), EnzymeZeroCache, OnlyUsedInOMP(), RecursivelyReplaceAddressSpace(), and ReverseModeCombined.

Referenced by PreProcessCache::preprocessForClone().

Variable Documentation

◆ SparseDebug

bool SparseDebug = false

constexpr

Definition at line 9364 of file FunctionUtils.cpp.

Referenced by getSparseConditions().

Classes

Macros

Typedefs

Enumerations

Functions

Variables

Macro Definition Documentation

◆ addAttribute

◆ DEBUG_TYPE

◆ getAttribute

◆ hasAttribute

◆ removeAttribute

Typedef Documentation

◆ QueueType

Enumeration Type Documentation

◆ RecurType

Function Documentation

◆ callOperands()

◆ cannotDependOnLoopIV()

◆ CanonicalizeGCPreserveEnds()

◆ CanonicalizeLoops()

◆ CoaleseTrivialMallocs()

◆ couldFunctionArgumentCapture()

◆ CreateMPIWrapper()

◆ DetectNoUnwindOfFn()

◆ DetectPointerArgOfFn()

◆ DetectReadonlyOrThrow()

◆ directlySparse()

◆ dump() [1/2]

◆ dump() [2/2]

◆ EnzymeAggressiveAA()

◆ EnzymeAlwaysInlineDiff()

◆ EnzymeAutoSparsity()

◆ EnzymeCoalese()

◆ EnzymeInline()

◆ EnzymeInlineCount()

◆ EnzymeLowerGlobals()

◆ EnzymeNameInstructions()

◆ EnzymeNoAlias()

◆ EnzymePHIRestructure()

◆ EnzymePostInlineOpt()

◆ EnzymePostOptLevel()

◆ EnzymePreopt()

◆ EnzymeSelectOpt()

◆ eraseSparseInbounds()

◆ evaluateAtLoopIter()

◆ fixSparse_inner()

◆ fixSparseIndices()

◆ ForceRecursiveInlining()

◆ getFunctionTypeForClone()

◆ getProductIntrinsic()

◆ getSparseConditions()

◆ getSumIntrinsic()

◆ guaranteedDataDependent()

◆ IsFunctionRecursive()

◆ isNot()

◆ isProduct()

◆ isReturnLikeParam()

◆ isSum()

◆ lowerSparsePlaceholders()

◆ LowerSparsification()

◆ mayReadThroughCustomRule()

◆ MergeGCPreserveEnds()

◆ OldAllocationSize()

◆ OnlyUsedInOMP()

◆ operator<<()

◆ reachesAvoiding()

◆ RecursivelyReplaceAddressSpace() [1/2]

◆ RecursivelyReplaceAddressSpace() [2/2]

◆ RemoveRedundantPHI()

◆ ReplaceFunctionImplementation()

◆ replaceToDense()

◆ SelectOptimization()

◆ setFullWillReturn()

◆ simplifyExtractions()

◆ SimplifyMPIQueries()

◆ SplitPHIs()

◆ unifyReturnBlocks()

◆ UpgradeAllocasToMallocs()

Variable Documentation