GradientUtils Class Reference

#include "MLIR/Interfaces/GradientUtils.h"

Inheritance diagram for GradientUtils:

Collaboration diagram for GradientUtils:

Classes
struct	LoadLikeCall
struct	Rematerializer
struct	ShadowRematerializer

Public Member Functions
llvm::SmallVector< llvm::OperandBundleDef, 2 >	getInvertedBundles (llvm::CallInst *orig, llvm::ArrayRef< ValueType > types, llvm::IRBuilder<> &Builder2, bool lookup, const llvm::ValueToValueMapTy &available=llvm::ValueToValueMapTy())
bool	usedInRooting (const llvm::CallBase *orig, llvm::ArrayRef< ValueType > types, const llvm::Value *val, bool shadow) const
llvm::Value *	getNewIfOriginal (llvm::Value *originst) const
llvm::Value *	ompThreadId ()
llvm::Value *	ompNumThreads ()
llvm::Value *	getOrInsertTotalMultiplicativeProduct (llvm::Value *val, LoopContext &lc)
llvm::Value *	getOrInsertConditionalIndex (llvm::Value *val, LoopContext &lc, bool pickTrue)
bool	assumeDynamicLoopOfSizeOne (llvm::Loop *L) const override
llvm::DebugLoc	getNewFromOriginal (const llvm::DebugLoc L) const
llvm::Value *	getNewFromOriginal (const llvm::Value *originst) const
llvm::Instruction *	getNewFromOriginal (const llvm::Instruction *newinst) const
llvm::BasicBlock *	getNewFromOriginal (const llvm::BasicBlock *newinst) const
llvm::Value *	hasUninverted (const llvm::Value *inverted) const
llvm::BasicBlock *	getOriginalFromNew (const llvm::BasicBlock *newinst) const
llvm::Value *	isOriginal (const llvm::Value *newinst) const
llvm::Instruction *	isOriginal (const llvm::Instruction *newinst) const
llvm::BasicBlock *	isOriginal (const llvm::BasicBlock *newinst) const
void	computeForwardingProperties (llvm::Instruction *V)
void	computeGuaranteedFrees ()
void	replaceAndRemoveUnwrapCacheFor (llvm::Value *A, llvm::Value *B)
llvm::BasicBlock *	addReverseBlock (llvm::BasicBlock *currentBlock, llvm::Twine const &name, bool forkCache=true, bool push=true)
std::map< UsageKey, bool >	populateSeenFromKnownRecompute () const
bool	legalRecompute (const llvm::Value *val, const llvm::ValueToValueMapTy &available, llvm::IRBuilder<> *BuilderM, bool reverse=false, bool legalRecomputeCache=true) const
bool	shouldRecompute (const llvm::Value *val, const llvm::ValueToValueMapTy &available, llvm::IRBuilder<> *BuilderM)
	Given the option to recompute a value or re-use an old one, return true if it is faster to recompute this value from scratch.
void	replaceAWithB (llvm::Value *A, llvm::Value *B, bool storeInCache=false) override
	Replace this instruction both in LLVM modules and any local data-structures.
void	erase (llvm::Instruction *I) override
	Erase this instruction both from LLVM modules and any local data-structures.
void	eraseWithPlaceholder (llvm::Instruction *I, llvm::Instruction *orig, const llvm::Twine &suffix="_replacementA", bool erase=true)
void	setTape (llvm::Value *newtape)
void	dumpPointers ()
int	getIndex (std::pair< llvm::Instruction *, CacheType > idx, const std::map< std::pair< llvm::Instruction *, CacheType >, int > &mapping, llvm::IRBuilder<> &)
int	getIndex (std::pair< llvm::Instruction *, CacheType > idx, std::map< std::pair< llvm::Instruction *, CacheType >, int > &mapping, llvm::IRBuilder<> &)
llvm::Value *	cacheForReverse (llvm::IRBuilder<> &BuilderQ, llvm::Value *malloc, int idx, bool replace=true)
llvm::ArrayRef< llvm::WeakTrackingVH >	getTapeValues () const
unsigned	getWidth ()
	GradientUtils (EnzymeLogic &Logic, llvm::Function *newFunc_, llvm::Function *oldFunc_, llvm::TargetLibraryInfo &TLI_, TypeAnalysis &TA_, TypeResults TR_, llvm::ValueToValueMapTy &invertedPointers_, const llvm::SmallPtrSetImpl< llvm::Value * > &constantvalues_, const llvm::SmallPtrSetImpl< llvm::Value * > &activevals_, DIFFE_TYPE ReturnActivity, bool shadowReturnUsed, llvm::ArrayRef< DIFFE_TYPE > ArgDiffeTypes_, llvm::ValueMap< const llvm::Value *, AssertingReplacingVH > &originalToNewFn_, DerivativeMode mode, bool runtimeActivity, bool strongZero, unsigned width, bool omp)
DIFFE_TYPE	getDiffeType (llvm::Value *v, bool foreignFunction) const
DIFFE_TYPE	getReturnDiffeType (llvm::Value *orig, bool *primalReturnUsedP, bool *shadowReturnUsedP, DerivativeMode cmode) const
bool	augmentedPrimalReturnUsed (llvm::Value *orig) const
	Whether the augmented pass used the primal of orig.
DIFFE_TYPE	getReturnDiffeType (llvm::Value *orig, bool *primalReturnUsedP, bool *shadowReturnUsedP) const
llvm::MDNode *	getDerivativeAliasScope (const llvm::Value *origptr, ssize_t newptr)
void	setPtrDiffe (llvm::Instruction *orig, llvm::Value *ptr, llvm::Value *newval, llvm::IRBuilder<> &BuilderM, llvm::MaybeAlign align, unsigned start, unsigned size, bool isVolatile, llvm::AtomicOrdering ordering, llvm::SyncScope::ID syncScope, llvm::Value *mask, llvm::ArrayRef< llvm::Metadata * > noAlias, llvm::ArrayRef< llvm::Metadata * > scopes, bool needs_post_cache=false)
llvm::BasicBlock *	getReverseOrLatchMerge (llvm::BasicBlock *BB, llvm::BasicBlock *branchingBlock)
	Given an edge from BB to branchingBlock get the corresponding block to branch to in the reverse pass.
void	forceContexts ()
void	computeMinCache ()
bool	isOriginalBlock (const llvm::BasicBlock &BB) const
void	eraseFictiousPHIs ()
void	forceActiveDetection ()
bool	isConstantValue (llvm::Value *val) const
bool	isConstantInstruction (const llvm::Instruction *inst) const
bool	getContext (llvm::BasicBlock *BB, LoopContext &lc)
void	forceAugmentedReturns ()
llvm::Value *	unwrapM (llvm::Value *const val, llvm::IRBuilder<> &BuilderM, const llvm::ValueToValueMapTy &available, UnwrapMode unwrapMode, llvm::BasicBlock *scope=nullptr, bool permitCache=true) override final
	if full unwrap, don't just unwrap this instruction, but also its operands, etc
void	ensureLookupCached (llvm::Instruction *inst, bool shouldFree=true, llvm::BasicBlock *scope=nullptr, llvm::MDNode *TBAA=nullptr)
llvm::Value *	fixLCSSA (llvm::Instruction *inst, llvm::BasicBlock *forwardBlock, bool legalInBlock=false)
llvm::Value *	lookupM (llvm::Value *val, llvm::IRBuilder<> &BuilderM, const llvm::ValueToValueMapTy &incoming_availalble=llvm::ValueToValueMapTy(), bool tryLegalRecomputeCheck=true, llvm::BasicBlock *scope=nullptr) override
	High-level utility to get the value an instruction at a new location specified by BuilderM.
llvm::Value *	invertPointerM (llvm::Value *val, llvm::IRBuilder<> &BuilderM)
llvm::Value *	invertPointerM (llvm::Value *val, llvm::IRBuilder<> &BuilderM, TypeTree look)
bool	isAtomic (llvm::Value *origptr) const
void	branchToCorrespondingTarget (llvm::BasicBlock *ctx, llvm::IRBuilder<> &BuilderM, const std::map< llvm::BasicBlock *, std::vector< std::pair< llvm::BasicBlock *, llvm::BasicBlock * > > > &targetToPreds, const std::map< llvm::BasicBlock *, llvm::PHINode * > *replacePHIs=nullptr)
	Given a map of edges we could have taken to desired target, compute a value that determines which target should be branched to.
void	getReverseBuilder (llvm::IRBuilder<> &Builder2, bool original=true)
void	getForwardBuilder (llvm::IRBuilder<> &Builder2)
llvm::Type *	getShadowType (llvm::Type *ty)
template<typename Func , typename... Args>
llvm::Value *	applyChainRule (llvm::Type *diffType, llvm::IRBuilder<> &Builder, Func rule, Args... args)
	Unwraps a vector derivative from its internal representation and applies a function f to each element.
template<typename Func , typename... Args>
void	applyChainRule (llvm::IRBuilder<> &Builder, Func rule, Args... args)
	Unwraps a vector derivative from its internal representation and applies a function f to each element.
template<typename Func >
llvm::Value *	applyChainRule (llvm::Type *diffType, llvm::ArrayRef< llvm::Constant * > diffs, llvm::IRBuilder<> &Builder, Func rule)
	Unwraps an collection of constant vector derivatives from their internal representations and applies a function f to each element.
bool	needsCacheWholeAllocation (const llvm::Value *V) const
Public Member Functions inherited from CacheUtility
virtual	~CacheUtility ()
bool	getContext (llvm::BasicBlock *BB, LoopContext &loopContext, bool ReverseLimit)
	Given a BasicBlock BB in newFunc, set loopContext to the relevant contained loop and return true.
std::pair< const llvm::SCEV *, const llvm::SCEV * >	computeLoopLimit (llvm::Loop *L)
	Compute the exact backedge-taken count of L (CouldNotCompute if it is not statically known) and the maximum number of iterations.
bool	isInstructionUsedInLoopInduction (llvm::Instruction &I)
	Return whether the given instruction is used as necessary as part of a loop context This includes as the canonical induction variable or increment.
llvm::AllocaInst *	getDynamicLoopLimit (llvm::Loop *L, bool ReverseLimit=true)
void	dumpScope ()
	Print out all currently cached values.
unsigned	getCacheAlignment (unsigned bsize) const
SubLimitType	getSubLimits (bool inForwardPass, llvm::IRBuilder<> *RB, LimitContext ctx, llvm::Value *extraSize=nullptr)
	Given a LimitContext ctx, representing a location inside a loop nest, break each of the loops up into chunks of loops where each chunk's number of iterations can be computed at the chunk preheader.
llvm::AllocaInst *	createCacheForScope (LimitContext ctx, llvm::Type *T, llvm::StringRef name, bool shouldFree, bool allocateInternal=true, llvm::Value *extraSize=nullptr)
	Create a cache of Type T at the given LimitContext.
virtual llvm::CallInst *	freeCache (llvm::BasicBlock *forwardPreheader, const SubLimitType &antimap, int i, llvm::AllocaInst *alloc, llvm::Type *myType, llvm::ConstantInt *byteSizeOfType, llvm::Value *storeInto, llvm::MDNode *InvariantMD)
	If an allocation is requested to be freed, this subclass will be called to chose how and where to free it.
void	storeInstructionInCache (LimitContext ctx, llvm::IRBuilder<> &BuilderM, llvm::Value *val, llvm::AllocaInst *cache, llvm::MDNode *TBAA=nullptr)
	Given an allocation defined at a particular ctx, store the value val in the cache at the location defined in the given builder.
void	storeInstructionInCache (LimitContext ctx, llvm::Instruction *inst, llvm::AllocaInst *cache, llvm::MDNode *TBAA=nullptr)
	Given an allocation defined at a particular ctx, store the instruction in the cache right after the instruction is executed.
llvm::Value *	getCachePointer (llvm::Type *T, bool inForwardPass, llvm::IRBuilder<> &BuilderM, LimitContext ctx, llvm::Value *cache, bool storeInInstructionsMap, const llvm::ValueToValueMapTy &available, llvm::Value *extraSize)
	Given an allocation specified by the LimitContext ctx and cache, compute a pointer that can hold the underlying type being cached.
llvm::Value *	lookupValueFromCache (llvm::Type *T, bool inForwardPass, llvm::IRBuilder<> &BuilderM, LimitContext ctx, llvm::Value *cache, bool isi1, const llvm::ValueToValueMapTy &available, llvm::Value *extraSize=nullptr, llvm::Value *extraOffset=nullptr)
	Given an allocation specified by the LimitContext ctx and cache, lookup the underlying cached value.

Static Public Member Functions
static GradientUtils *	CreateFromClone (EnzymeLogic &Logic, bool runtimeActivity, bool strongZero, unsigned width, llvm::Function *todiff, llvm::TargetLibraryInfo &TLI, TypeAnalysis &TA, FnTypeInfo &oldTypeInfo, DIFFE_TYPE retType, llvm::ArrayRef< DIFFE_TYPE > constant_args, bool returnUsed, bool shadowReturnUsed, std::map< AugmentedStruct, int > &returnMapping, bool omp)
static llvm::Constant *	GetOrCreateShadowConstant (RequestContext context, EnzymeLogic &Logic, llvm::TargetLibraryInfo &TLI, TypeAnalysis &TA, llvm::Constant *F, DerivativeMode mode, bool runtimeActivity, bool strongZero, unsigned width, bool AtomicAdd)
static llvm::Constant *	GetOrCreateShadowFunction (RequestContext context, EnzymeLogic &Logic, llvm::TargetLibraryInfo &TLI, TypeAnalysis &TA, llvm::Function *F, DerivativeMode mode, bool runtimeActivity, bool strongZero, unsigned width, bool AtomicAdd)
static llvm::Type *	getShadowType (llvm::Type *ty, unsigned width)
static llvm::Value *	extractMeta (llvm::IRBuilder<> &Builder, llvm::Value *Agg, unsigned off, const llvm::Twine &name="")
	Helper routine to extract a nested element from a struct/array. This is.
static llvm::Value *	extractMeta (llvm::IRBuilder<> &Builder, llvm::Value *Agg, llvm::ArrayRef< unsigned > off, const llvm::Twine &name="", bool fallback=true)
	Helper routine to extract a nested element from a struct/array. Unlike the.
static llvm::Type *	extractMeta (llvm::Type *T, llvm::ArrayRef< unsigned > off)
	Helper routine to get the type of an extraction.
static llvm::Value *	recursiveFAdd (llvm::IRBuilder<> &B, llvm::Value *lhs, llvm::Value *rhs, llvm::ArrayRef< unsigned > lhs_off={}, llvm::ArrayRef< unsigned > rhs_off={}, llvm::Value *prev=nullptr, bool vectorLayer=false)

Public Attributes
EnzymeLogic &	Logic
bool	AtomicAdd
DerivativeMode	mode
llvm::Function *	oldFunc
llvm::ValueMap< const llvm::Value *, InvertedPointerVH >	invertedPointers
llvm::DominatorTree *	OrigDT
llvm::PostDominatorTree *	OrigPDT
llvm::LoopInfo *	OrigLI
llvm::ScalarEvolution *	OrigSE
llvm::SmallPtrSet< llvm::BasicBlock *, 4 >	BlocksDominatingAllReturns
	(Original) Blocks which dominate all returns
llvm::SmallPtrSet< llvm::BasicBlock *, 4 >	notForAnalysis
std::shared_ptr< ActivityAnalyzer >	ATA
llvm::SmallVector< llvm::BasicBlock *, 12 >	originalBlocks
llvm::ValueMap< const llvm::CallInst *, llvm::SmallPtrSet< const llvm::CallInst *, 1 > >	allocationsWithGuaranteedFree
	Allocations which are known to always be freed before the reverse, to the list of frees that must apply to this allocation.
llvm::SmallPtrSet< const llvm::CallInst *, 1 >	postDominatingFrees
	Frees which can always be eliminated as the post dominate an allocation (which will itself be freed).
llvm::SmallPtrSet< const llvm::CallInst *, 1 >	forwardDeallocations
	Deallocations that should be kept in the forward pass because they deallocation memory which isn't necessary for the reverse pass.
std::map< llvm::BasicBlock *, llvm::SmallVector< llvm::BasicBlock *, 4 > >	reverseBlocks
	Map of primal block to corresponding block(s) in reverse.
std::map< llvm::BasicBlock *, llvm::BasicBlock * >	reverseBlockToPrimal
	Map of block in reverse to corresponding primal block.
llvm::SmallPtrSet< llvm::Instruction *, 4 >	TapesToPreventRecomputation
	A set of tape extractions to enforce a cache of rather than attempting to recompute.
llvm::ValueMap< llvm::PHINode *, llvm::WeakTrackingVH >	fictiousPHIs
llvm::ValueMap< const llvm::Value *, AssertingReplacingVH >	originalToNewFn
llvm::ValueMap< const llvm::Value *, AssertingReplacingVH >	newToOriginalFn
llvm::SmallVector< llvm::CallInst *, 4 >	originalCalls
llvm::SmallPtrSet< llvm::Instruction *, 4 >	unnecessaryIntermediates
const std::map< llvm::Instruction *, bool > *	can_modref_map
const std::map< llvm::CallInst *, std::pair< bool, const std::vector< bool > > > *	overwritten_args_map_ptr
const llvm::SmallPtrSetImpl< const llvm::Value * > *	unnecessaryValuesP
const std::set< const llvm::Value * > *	primalUnnecessaryValuesP = nullptr
	In the reverse pass of split mode, the augmented pass's unnecessary values and recompute decisions, to answer whether the primal pass used a value.
const std::map< const llvm::Value *, bool > *	primalKnownRecomputeHeuristicP
llvm::Value *	tid
llvm::Value *	numThreads
llvm::ValueMap< llvm::Value *, Rematerializer >	rematerializableAllocations
llvm::SmallPtrSet< llvm::Value *, 4 >	allocationsToBeRematerialized
llvm::ValueMap< llvm::Value *, ShadowRematerializer >	backwardsOnlyShadows
	Only loaded from and stored to (not captured), mapped to the stores (and memset).
std::vector< bool >	nowrite_shadows
std::map< const llvm::Value *, bool >	knownRecomputeHeuristic
llvm::ValueMap< const llvm::Instruction *, AssertingReplacingVH >	unwrappedLoads
llvm::AAResults *	OrigAA
TypeAnalysis &	TA
TypeResults	TR
bool	omp
bool	runtimeActivity
bool	strongZero
bool	shadowReturnUsed
llvm::ArrayRef< DIFFE_TYPE >	ArgDiffeTypes
llvm::ValueMap< const llvm::Value *, llvm::MDNode * >	differentialAliasScopeDomains
llvm::ValueMap< const llvm::Value *, llvm::DenseMap< ssize_t, llvm::MDNode * > >	differentialAliasScope
std::map< std::tuple< llvm::BasicBlock *, llvm::BasicBlock * >, llvm::BasicBlock * >	newBlocksForLoop_cache
	This cache stores blocks we may insert as part of getReverseOrLatchMerge to handle inverse iv iteration.
std::map< const llvm::BasicBlock *, llvm::BasicBlock * >	rematerializedLoops_cache
	This cache stores a rematerialized forward pass in the loop specified.
llvm::SmallVector< llvm::Instruction *, 1 >	rematerializedPrimalOrShadowAllocations
std::map< llvm::Instruction *, llvm::ValueMap< llvm::BasicBlock *, llvm::WeakTrackingVH > >	lcssaFixes
std::map< llvm::PHINode *, llvm::WeakTrackingVH >	lcssaPHIToOrig
Public Attributes inherited from CacheUtility
llvm::Function *const	newFunc
	The function whose instructions we are caching.
llvm::TargetLibraryInfo &	TLI
	Various analysis results of newFunc.
llvm::DominatorTree	DT
llvm::BasicBlock *	inversionAllocs

Additional Inherited Members
Public Types inherited from CacheUtility
typedef llvm::SmallVector< std::pair< llvm::Value *, llvm::SmallVector< std::pair< LoopContext, llvm::Value * >, 4 > >, 0 >	SubLimitType
	Given a LimitContext ctx, representing a location inside a loop nest, break each of the loops up into chunks of loops where each chunk's number of iterations can be computed at the chunk preheader.
Protected Member Functions inherited from CacheUtility
	CacheUtility (llvm::TargetLibraryInfo &TLI, llvm::Function *newFunc)
llvm::Value *	loadFromCachePointer (llvm::Type *T, llvm::IRBuilder<> &BuilderM, llvm::Value *cptr, llvm::Value *cache)
	Perform the final load from the cache, applying requisite invariant group and alignment.
Protected Attributes inherited from CacheUtility
llvm::LoopInfo	LI
llvm::AssumptionCache	AC
MustExitScalarEvolution	SE
std::map< llvm::Loop *, LoopContext >	loopContexts
	Map of Loop to requisite loop information needed for AD (forward/reverse induction/etc)
std::map< llvm::Value *, std::pair< llvm::AssertingVH< llvm::AllocaInst >, LimitContext > >	scopeMap
	A map of values being cached to their underlying allocation/limit context.
std::map< llvm::AllocaInst *, llvm::SmallVector< llvm::AssertingVH< llvm::Instruction >, 4 > >	scopeInstructions
	A map of allocations to a vector of instruction used to create by the allocation Keeping track of these values is useful for deallocation.
std::map< llvm::AllocaInst *, std::set< llvm::AssertingVH< llvm::CallInst > > >	scopeFrees
	A map of allocations to a set of instructions which free memory as part of the cache.
std::map< llvm::AllocaInst *, llvm::SmallVector< llvm::AssertingVH< llvm::CallInst >, 4 > >	scopeAllocs
	A map of allocations to a set of instructions which allocate memory as part of the cache.
llvm::SmallPtrSet< llvm::LoadInst *, 10 >	CacheLookups

Detailed Description

Definition at line 137 of file GradientUtils.h.

Constructor & Destructor Documentation

GradientUtils()

GradientUtils::GradientUtils	(	EnzymeLogic &	Logic,
		llvm::Function *	newFunc_,
		llvm::Function *	oldFunc_,
		llvm::TargetLibraryInfo &	TLI_,
		TypeAnalysis &	TA_,
		TypeResults	TR_,
		llvm::ValueToValueMapTy &	invertedPointers_,
		const llvm::SmallPtrSetImpl< llvm::Value * > &	constantvalues_,
		const llvm::SmallPtrSetImpl< llvm::Value * > &	activevals_,
		DIFFE_TYPE	ReturnActivity,
		bool	shadowReturnUsed,
		llvm::ArrayRef< DIFFE_TYPE >	ArgDiffeTypes_,
		llvm::ValueMap< const llvm::Value *, AssertingReplacingVH > &	originalToNewFn_,
		DerivativeMode	mode,
		bool	runtimeActivity,
		bool	strongZero,
		unsigned	width,
		bool	omp )

Definition at line 157 of file GradientUtils.cpp.

Member Function Documentation

addReverseBlock()

BasicBlock * GradientUtils::addReverseBlock	(	llvm::BasicBlock *	currentBlock,
		llvm::Twine const &	name,
		bool	forkCache = true,
		bool	push = true )

Definition at line 9797 of file GradientUtils.cpp.

Referenced by EnzymeGradientUtilsAddReverseBlock(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), and AdjointGenerator::visitCommonStore().

applyChainRule() [1/3]

template<typename Func , typename... Args>

void GradientUtils::applyChainRule ( llvm::IRBuilder<> & Builder, Func rule, Args... args )

inline

Unwraps a vector derivative from its internal representation and applies a function f to each element.

Return values of f are collected and wrapped.

Definition at line 640 of file GradientUtils.h.

applyChainRule() [2/3]

template<typename Func >

llvm::Value * GradientUtils::applyChainRule ( llvm::Type * diffType, llvm::ArrayRef< llvm::Constant * > diffs, llvm::IRBuilder<> & Builder, Func rule )

inline

Unwraps an collection of constant vector derivatives from their internal representations and applies a function f to each element.

Definition at line 663 of file GradientUtils.h.

applyChainRule() [3/3]

template<typename Func , typename... Args>

llvm::Value * GradientUtils::applyChainRule ( llvm::Type * diffType, llvm::IRBuilder<> & Builder, Func rule, Args... args )

inline

Unwraps a vector derivative from its internal representation and applies a function f to each element.

Return values of f are collected and wrapped.

Definition at line 608 of file GradientUtils.h.

assumeDynamicLoopOfSizeOne()

bool GradientUtils::assumeDynamicLoopOfSizeOne ( llvm::Loop * L ) const

overridevirtual

Implements CacheUtility.

Definition at line 595 of file GradientUtils.cpp.

augmentedPrimalReturnUsed()

bool GradientUtils::augmentedPrimalReturnUsed

(

llvm::Value *

orig

)

const

Whether the augmented pass used the primal of orig.

In the reverse pass of split mode this is the augmented pass's answer rather than one from this pass's own analysis, so it can rebuild how a call was augmented.

Definition at line 4590 of file GradientUtils.cpp.

Referenced by EnzymeGradientUtilsGetAugmentedPrimalReturnUsed().

branchToCorrespondingTarget()

void GradientUtils::branchToCorrespondingTarget	(	llvm::BasicBlock *	ctx,
		llvm::IRBuilder<> &	BuilderM,
		const std::map< llvm::BasicBlock *, std::vector< std::pair< llvm::BasicBlock *, llvm::BasicBlock * > > > &	targetToPreds,
		const std::map< llvm::BasicBlock *, llvm::PHINode * > *	replacePHIs = nullptr )

Given a map of edges we could have taken to desired target, compute a value that determines which target should be branched to.

Definition at line 8016 of file GradientUtils.cpp.

cacheForReverse()

Value * GradientUtils::cacheForReverse	(	llvm::IRBuilder<> &	BuilderQ,
		llvm::Value *	malloc,
		int	idx,
		bool	replace = true )

Definition at line 2699 of file GradientUtils.cpp.

Referenced by AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitIntrinsicInst(), AdjointGenerator::visitLoadLike(), and AdjointGenerator::visitOMPCall().

computeForwardingProperties()

void GradientUtils::computeForwardingProperties

(

llvm::Instruction *

V

)

Definition at line 9458 of file GradientUtils.cpp.

computeGuaranteedFrees()

void GradientUtils::computeGuaranteedFrees

(

)

Definition at line 10009 of file GradientUtils.cpp.

computeMinCache()

void GradientUtils::computeMinCache

(

)

Definition at line 8596 of file GradientUtils.cpp.

CreateFromClone()

GradientUtils * GradientUtils::CreateFromClone ( EnzymeLogic & Logic, bool runtimeActivity, bool strongZero, unsigned width, llvm::Function * todiff, llvm::TargetLibraryInfo & TLI, TypeAnalysis & TA, FnTypeInfo & oldTypeInfo, DIFFE_TYPE retType, llvm::ArrayRef< DIFFE_TYPE > constant_args, bool returnUsed, bool shadowReturnUsed, std::map< AugmentedStruct, int > & returnMapping, bool omp )

static

Definition at line 4444 of file GradientUtils.cpp.

dumpPointers()

void GradientUtils::dumpPointers

(

)

Definition at line 9952 of file GradientUtils.cpp.

ensureLookupCached()

void GradientUtils::ensureLookupCached	(	llvm::Instruction *	inst,
		bool	shouldFree = true,
		llvm::BasicBlock *	scope = nullptr,
		llvm::MDNode *	TBAA = nullptr )

Definition at line 2521 of file GradientUtils.cpp.

erase()

void GradientUtils::erase ( llvm::Instruction * I )

overridevirtual

Erase this instruction both from LLVM modules and any local data-structures.

Reimplemented from CacheUtility.

Definition at line 9870 of file GradientUtils.cpp.

Referenced by AdjointGenerator::createBinaryOperatorDual(), EnzymeGradientUtilsErase(), AdjointGenerator::forwardModeInvertedPointerFallback(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::resolveShadowPlaceholder(), DiffeGradientUtils::setDiffe(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitLoadLike(), and AdjointGenerator::visitOMPCall().

eraseFictiousPHIs()

void GradientUtils::eraseFictiousPHIs

(

)

Definition at line 8966 of file GradientUtils.cpp.

eraseWithPlaceholder()

void GradientUtils::eraseWithPlaceholder	(	llvm::Instruction *	I,
		llvm::Instruction *	orig,
		const llvm::Twine &	suffix = "_replacementA",
		bool	erase = true )

Definition at line 9913 of file GradientUtils.cpp.

Referenced by EnzymeGradientUtilsEraseWithPlaceholder(), and AdjointGenerator::eraseIfUnused().

extractMeta() [1/3]

static llvm::Value * GradientUtils::extractMeta ( llvm::IRBuilder<> & Builder, llvm::Value * Agg, llvm::ArrayRef< unsigned > off, const llvm::Twine & name = "", bool fallback = true )

static

Helper routine to extract a nested element from a struct/array. Unlike the.

extractMeta() [2/3]

static llvm::Value * GradientUtils::extractMeta ( llvm::IRBuilder<> & Builder, llvm::Value * Agg, unsigned off, const llvm::Twine & name = "" )

static

Helper routine to extract a nested element from a struct/array. This is.

Referenced by createMPIForwardCall(), EnzymeFixupBatchedJuliaCallingConvention(), AdjointGenerator::handleKnownCallDerivatives(), moveSRetToFromRoots(), simplifyExtractions(), simplifyLoad(), SplitPHIs(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitExtractElementInst(), AdjointGenerator::visitInsertElementInst(), and AdjointGenerator::visitShuffleVectorInst().

extractMeta() [3/3]

static llvm::Type * GradientUtils::extractMeta ( llvm::Type * T, llvm::ArrayRef< unsigned > off )

static

Helper routine to get the type of an extraction.

fixLCSSA()

Value * GradientUtils::fixLCSSA	(	llvm::Instruction *	inst,
		llvm::BasicBlock *	forwardBlock,
		bool	legalInBlock = false )

Definition at line 2544 of file GradientUtils.cpp.

forceActiveDetection()

void GradientUtils::forceActiveDetection

(

)

Definition at line 9014 of file GradientUtils.cpp.

forceAugmentedReturns()

void GradientUtils::forceAugmentedReturns

(

)

op->getType()->isPointerTy() && !op->getType()->isIntegerTy()) {

Definition at line 9093 of file GradientUtils.cpp.

forceContexts()

void GradientUtils::forceContexts

(

)

Definition at line 3966 of file GradientUtils.cpp.

getContext()

bool GradientUtils::getContext	(	llvm::BasicBlock *	BB,
		LoopContext &	lc )

Definition at line 9088 of file GradientUtils.cpp.

Referenced by AdjointGenerator::createSelectInstAdjoint(), and AdjointGenerator::visitBinaryOperator().

getDerivativeAliasScope()

MDNode * GradientUtils::getDerivativeAliasScope	(	const llvm::Value *	origptr,
		ssize_t	newptr )

Definition at line 4414 of file GradientUtils.cpp.

Referenced by AdjointGenerator::visitCommonStore(), and AdjointGenerator::visitLoadLike().

getDiffeType()

DIFFE_TYPE GradientUtils::getDiffeType	(	llvm::Value *	v,
		bool	foreignFunction ) const

Definition at line 4609 of file GradientUtils.cpp.

Referenced by EnzymeGradientUtilsGetDiffeType(), AdjointGenerator::recursivelyHandleSubfunction(), and AdjointGenerator::visitOMPCall().

getForwardBuilder()

void GradientUtils::getForwardBuilder

(

llvm::IRBuilder<> &

Builder2

)

Definition at line 5026 of file GradientUtils.cpp.

getIndex() [1/2]

int GradientUtils::getIndex	(	std::pair< llvm::Instruction *, CacheType >	idx,
		const std::map< std::pair< llvm::Instruction *, CacheType >, int > &	mapping,
		llvm::IRBuilder<> &	)

getIndex() [2/2]

int GradientUtils::getIndex	(	std::pair< llvm::Instruction *, CacheType >	idx,
		std::map< std::pair< llvm::Instruction *, CacheType >, int > &	mapping,
		llvm::IRBuilder<> &	)

getInvertedBundles()

SmallVector< OperandBundleDef, 2 > GradientUtils::getInvertedBundles	(	llvm::CallInst *	orig,
		llvm::ArrayRef< ValueType >	types,
		llvm::IRBuilder<> &	Builder2,
		bool	lookup,
		const llvm::ValueToValueMapTy &	available = llvm::ValueToValueMapTy() )

Definition at line 307 of file GradientUtils.cpp.

Referenced by createMPIForwardCall(), EnzymeGradientUtilsCallWithInvertedBundles(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), and AdjointGenerator::visitMemSetCommon().

getNewFromOriginal() [1/4]

llvm::BasicBlock * GradientUtils::getNewFromOriginal

(

const llvm::BasicBlock *

newinst

)

const

getNewFromOriginal() [2/4]

llvm::DebugLoc GradientUtils::getNewFromOriginal

(

const llvm::DebugLoc

L

)

const

Referenced by AdjointGenerator::createBinaryOperatorAdjoint(), AdjointGenerator::createBinaryOperatorDual(), createMPIForwardCall(), AdjointGenerator::createSelectInstAdjoint(), EnzymeGradientUtilsNewFromOriginal(), EnzymeGradientUtilsSetDebugLocFromOriginal(), AdjointGenerator::eraseIfUnused(), AdjointGenerator::handleAdjointForIntrinsic(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitAtomicRMWInst(), AdjointGenerator::visitBinaryOperator(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitCastInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitExtractElementInst(), AdjointGenerator::visitInsertElementInst(), AdjointGenerator::visitInstruction(), AdjointGenerator::visitIntrinsicInst(), AdjointGenerator::visitLoadLike(), AdjointGenerator::visitMemSetCommon(), AdjointGenerator::visitMemTransferCommon(), AdjointGenerator::visitMemTransferInst(), and AdjointGenerator::visitOMPCall().

getNewFromOriginal() [3/4]

llvm::Instruction * GradientUtils::getNewFromOriginal

(

const llvm::Instruction *

newinst

)

const

getNewFromOriginal() [4/4]

llvm::Value * GradientUtils::getNewFromOriginal

(

const llvm::Value *

originst

)

const

getNewIfOriginal()

Value * GradientUtils::getNewIfOriginal

(

llvm::Value *

originst

)

const

Definition at line 367 of file GradientUtils.cpp.

GetOrCreateShadowConstant()

Constant * GradientUtils::GetOrCreateShadowConstant ( RequestContext context, EnzymeLogic & Logic, llvm::TargetLibraryInfo & TLI, TypeAnalysis & TA, llvm::Constant * F, DerivativeMode mode, bool runtimeActivity, bool strongZero, unsigned width, bool AtomicAdd )

static

Definition at line 4647 of file GradientUtils.cpp.

GetOrCreateShadowFunction()

Constant * GradientUtils::GetOrCreateShadowFunction ( RequestContext context, EnzymeLogic & Logic, llvm::TargetLibraryInfo & TLI, TypeAnalysis & TA, llvm::Function * F, DerivativeMode mode, bool runtimeActivity, bool strongZero, unsigned width, bool AtomicAdd )

static

Todo allow tape propagation

Definition at line 4765 of file GradientUtils.cpp.

getOriginalFromNew()

BasicBlock * GradientUtils::getOriginalFromNew

(

const llvm::BasicBlock *

newinst

)

const

Definition at line 690 of file GradientUtils.cpp.

Referenced by AdjointGenerator::visitBinaryOperator().

getOrInsertConditionalIndex()

Value * GradientUtils::getOrInsertConditionalIndex	(	llvm::Value *	val,
		LoopContext &	lc,
		bool	pickTrue )

Definition at line 523 of file GradientUtils.cpp.

Referenced by AdjointGenerator::createSelectInstAdjoint().

getOrInsertTotalMultiplicativeProduct()

Value * GradientUtils::getOrInsertTotalMultiplicativeProduct	(	llvm::Value *	val,
		LoopContext &	lc )

Definition at line 442 of file GradientUtils.cpp.

Referenced by AdjointGenerator::visitBinaryOperator().

getReturnDiffeType() [1/2]

DIFFE_TYPE GradientUtils::getReturnDiffeType	(	llvm::Value *	orig,
		bool *	primalReturnUsedP,
		bool *	shadowReturnUsedP ) const

Definition at line 4533 of file GradientUtils.cpp.

getReturnDiffeType() [2/2]

DIFFE_TYPE GradientUtils::getReturnDiffeType	(	llvm::Value *	orig,
		bool *	primalReturnUsedP,
		bool *	shadowReturnUsedP,
		DerivativeMode	cmode ) const

Definition at line 4539 of file GradientUtils.cpp.

Referenced by EnzymeGradientUtilsGetReturnDiffeType(), AdjointGenerator::handleKnownCallDerivatives(), and AdjointGenerator::visitCallInst().

getReverseBuilder()

void GradientUtils::getReverseBuilder	(	llvm::IRBuilder<> &	Builder2,
		bool	original = true )

Definition at line 5003 of file GradientUtils.cpp.

getReverseOrLatchMerge()

BasicBlock * GradientUtils::getReverseOrLatchMerge	(	llvm::BasicBlock *	BB,
		llvm::BasicBlock *	branchingBlock )

Given an edge from BB to branchingBlock get the corresponding block to branch to in the reverse pass.

Definition at line 3821 of file GradientUtils.cpp.

getShadowType() [1/2]

llvm::Type * GradientUtils::getShadowType

(

llvm::Type *

ty

)

getShadowType() [2/2]

static llvm::Type * GradientUtils::getShadowType ( llvm::Type * ty, unsigned width )

static

Referenced by AdjointGenerator::createBinaryOperatorAdjoint(), AdjointGenerator::createBinaryOperatorDual(), AdjointGenerator::createSelectInstAdjoint(), DiffeGradientUtils::diffe(), EnzymeGetShadowType(), EnzymeGradientUtilsGetShadowType(), DiffeGradientUtils::getDifferential(), getFunctionTypeForClone(), AdjointGenerator::handleAdjointForIntrinsic(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::resolveShadowPlaceholder(), DiffeGradientUtils::setDiffe(), AdjointGenerator::visitAtomicRMWInst(), AdjointGenerator::visitCallInst(), InstructionBatcher::visitCallInst(), AdjointGenerator::visitCastInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitExtractElementInst(), AdjointGenerator::visitExtractValueInst(), AdjointGenerator::visitInsertElementInst(), AdjointGenerator::visitInsertValueInst(), AdjointGenerator::visitInstruction(), AdjointGenerator::visitLoadLike(), and AdjointGenerator::visitShuffleVectorInst().

getTapeValues()

llvm::ArrayRef< llvm::WeakTrackingVH > GradientUtils::getTapeValues ( ) const

inline

Definition at line 393 of file GradientUtils.h.

getWidth()

unsigned GradientUtils::getWidth ( )

inline

Definition at line 411 of file GradientUtils.h.

Referenced by createMPIForwardCall(), EnzymeGradientUtilsGetWidth(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitExtractElementInst(), AdjointGenerator::visitInsertElementInst(), AdjointGenerator::visitLoadLike(), AdjointGenerator::visitOMPCall(), and AdjointGenerator::visitShuffleVectorInst().

hasUninverted()

Value * GradientUtils::hasUninverted

(

const llvm::Value *

inverted

)

const

Definition at line 682 of file GradientUtils.cpp.

invertPointerM() [1/2]

llvm::Value * GradientUtils::invertPointerM	(	llvm::Value *	val,
		llvm::IRBuilder<> &	BuilderM )

Referenced by AdjointGenerator::addAtomicRMWValueAdjoint(), createMPIForwardCall(), DiffeGradientUtils::diffe(), EnzymeGradientUtilsInvertPointer(), AdjointGenerator::forwardModeInvertedPointerFallback(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitAtomicRMWInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitLoadLike(), AdjointGenerator::visitMemSetCommon(), AdjointGenerator::visitMemTransferCommon(), and AdjointGenerator::visitOMPCall().

invertPointerM() [2/2]

llvm::Value * GradientUtils::invertPointerM	(	llvm::Value *	val,
		llvm::IRBuilder<> &	BuilderM,
		TypeTree	look )

isAtomic()

bool GradientUtils::isAtomic

(

llvm::Value *

origptr

)

const

Definition at line 5487 of file GradientUtils.cpp.

Referenced by AdjointGenerator::DifferentiableMemCopyFloats().

isConstantInstruction()

bool GradientUtils::isConstantInstruction

(

const llvm::Instruction *

inst

)

const

Definition at line 9083 of file GradientUtils.cpp.

Referenced by DifferentialUseAnalysis::checkLoopyReductionPHI(), AdjointGenerator::createBinaryOperatorAdjoint(), AdjointGenerator::createBinaryOperatorDual(), AdjointGenerator::createSelectInstAdjoint(), EnzymeGradientUtilsIsConstantInstruction(), AdjointGenerator::handleAdjointForIntrinsic(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), DifferentialUseAnalysis::pushLoopyPHIPreheader(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitAtomicRMWInst(), AdjointGenerator::visitBinaryOperator(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitCastInst(), AdjointGenerator::visitExtractElementInst(), AdjointGenerator::visitExtractValueInst(), AdjointGenerator::visitInsertElementInst(), AdjointGenerator::visitInstruction(), AdjointGenerator::visitLoadLike(), AdjointGenerator::visitSelectInst(), and AdjointGenerator::visitShuffleVectorInst().

isConstantValue()

bool GradientUtils::isConstantValue

(

llvm::Value *

val

)

const

Functions must be false so we can replace function with augmentation, fallback to analysis

Definition at line 9033 of file GradientUtils.cpp.

Referenced by DifferentialUseAnalysis::__attribute__(), AdjointGenerator::addAtomicRMWValueAdjoint(), DifferentialUseAnalysis::checkLoopyReductionPHI(), AdjointGenerator::createBinaryOperatorAdjoint(), AdjointGenerator::createBinaryOperatorDual(), createMPIForwardCall(), AdjointGenerator::createSelectInstAdjoint(), DiffeGradientUtils::diffe(), EnzymeGradientUtilsIsConstantValue(), AdjointGenerator::forwardModeInvertedPointerFallback(), AdjointGenerator::handleAdjointForIntrinsic(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), DifferentialUseAnalysis::is_value_needed_in_reverse(), DifferentialUseAnalysis::pushLoopyPHIPreheader(), AdjointGenerator::recursivelyHandleSubfunction(), DiffeGradientUtils::setDiffe(), AdjointGenerator::visitAtomicRMWInst(), AdjointGenerator::visitBinaryOperator(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitCastInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitExtractElementInst(), AdjointGenerator::visitExtractValueInst(), AdjointGenerator::visitInsertElementInst(), AdjointGenerator::visitInsertValueInst(), AdjointGenerator::visitInstruction(), AdjointGenerator::visitLoadLike(), AdjointGenerator::visitMemSetCommon(), AdjointGenerator::visitMemTransferCommon(), AdjointGenerator::visitOMPCall(), and AdjointGenerator::visitShuffleVectorInst().

isOriginal() [1/3]

llvm::BasicBlock * GradientUtils::isOriginal

(

const llvm::BasicBlock *

newinst

)

const

isOriginal() [2/3]

llvm::Instruction * GradientUtils::isOriginal

(

const llvm::Instruction *

newinst

)

const

isOriginal() [3/3]

llvm::Value * GradientUtils::isOriginal

(

const llvm::Value *

newinst

)

const

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

isOriginalBlock()

bool GradientUtils::isOriginalBlock

(

const llvm::BasicBlock &

BB

)

const

Definition at line 8958 of file GradientUtils.cpp.

legalRecompute()

bool GradientUtils::legalRecompute	(	const llvm::Value *	val,
		const llvm::ValueToValueMapTy &	available,
		llvm::IRBuilder<> *	BuilderM,
		bool	reverse = false,
		bool	legalRecomputeCache = true ) const

Definition at line 3973 of file GradientUtils.cpp.

Referenced by AdjointGenerator::handleKnownCallDerivatives(), and AdjointGenerator::visitCallInst().

lookupM()

Value * GradientUtils::lookupM ( llvm::Value * val, llvm::IRBuilder<> & BuilderM, const llvm::ValueToValueMapTy & incoming_availalble = llvm::ValueToValueMapTy(), bool tryLegalityCheck = true, llvm::BasicBlock * scope = nullptr )

overridevirtual

High-level utility to get the value an instruction at a new location specified by BuilderM.

Unlike unwrap, this function can never fail – falling back to creating a cache if necessary. This function is prepopulated with a set of values that are already known to be available and may contain optimizations for getting the value in more efficient ways (e.g. unwrap'ing when legal, looking up equivalent values, etc). This high-level utility should be implemented based off the low-level caching infrastructure provided in this class.

Implements CacheUtility.

Definition at line 6835 of file GradientUtils.cpp.

Referenced by AdjointGenerator::createSelectInstAdjoint(), EnzymeGradientUtilsLookup(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::lookup(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitCallInst(), and AdjointGenerator::visitMemSetCommon().

needsCacheWholeAllocation()

bool GradientUtils::needsCacheWholeAllocation

(

const llvm::Value *

V

)

const

Definition at line 10323 of file GradientUtils.cpp.

Referenced by DifferentialUseAnalysis::is_value_needed_in_reverse().

ompNumThreads()

Value * GradientUtils::ompNumThreads

(

)

Definition at line 412 of file GradientUtils.cpp.

ompThreadId()

Value * GradientUtils::ompThreadId

(

)

Definition at line 383 of file GradientUtils.cpp.

populateSeenFromKnownRecompute()

std::map< UsageKey, bool > GradientUtils::populateSeenFromKnownRecompute ( ) const

inline

Definition at line 341 of file GradientUtils.h.

Referenced by AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitCallInst(), and AdjointGenerator::visitLoadLike().

recursiveFAdd()

llvm::Value * GradientUtils::recursiveFAdd ( llvm::IRBuilder<> & B, llvm::Value * lhs, llvm::Value * rhs, llvm::ArrayRef< unsigned > lhs_off = {}, llvm::ArrayRef< unsigned > rhs_off = {}, llvm::Value * prev = nullptr, bool vectorLayer = false )

static

Definition at line 5329 of file GradientUtils.cpp.

replaceAndRemoveUnwrapCacheFor()

void GradientUtils::replaceAndRemoveUnwrapCacheFor	(	llvm::Value *	A,
		llvm::Value *	B )

Definition at line 10446 of file GradientUtils.cpp.

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

replaceAWithB()

void GradientUtils::replaceAWithB ( llvm::Value * A, llvm::Value * B, bool storeInCache = false )

overridevirtual

Replace this instruction both in LLVM modules and any local data-structures.

Reimplemented from CacheUtility.

Definition at line 9842 of file GradientUtils.cpp.

Referenced by AdjointGenerator::createBinaryOperatorDual(), EnzymeGradientUtilsReplaceAWithB(), AdjointGenerator::forwardModeInvertedPointerFallback(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::resolveShadowPlaceholder(), DiffeGradientUtils::setDiffe(), and AdjointGenerator::visitCallInst().

setPtrDiffe()

void GradientUtils::setPtrDiffe	(	llvm::Instruction *	orig,
		llvm::Value *	ptr,
		llvm::Value *	newval,
		llvm::IRBuilder<> &	BuilderM,
		llvm::MaybeAlign	align,
		unsigned	start,
		unsigned	size,
		bool	isVolatile,
		llvm::AtomicOrdering	ordering,
		llvm::SyncScope::ID	syncScope,
		llvm::Value *	mask,
		llvm::ArrayRef< llvm::Metadata * >	noAlias,
		llvm::ArrayRef< llvm::Metadata * >	scopes,
		bool	needs_post_cache = false )

Definition at line 5038 of file GradientUtils.cpp.

Referenced by AdjointGenerator::visitCommonStore().

setTape()

void GradientUtils::setTape

(

llvm::Value *

newtape

)

Definition at line 9944 of file GradientUtils.cpp.

shouldRecompute()

bool GradientUtils::shouldRecompute	(	const llvm::Value *	val,
		const llvm::ValueToValueMapTy &	available,
		llvm::IRBuilder<> *	BuilderM )

Given the option to recompute a value or re-use an old one, return true if it is faster to recompute this value from scratch.

Definition at line 4271 of file GradientUtils.cpp.

unwrapM()

Value * GradientUtils::unwrapM ( llvm::Value *const val, llvm::IRBuilder<> & BuilderM, const llvm::ValueToValueMapTy & available, UnwrapMode unwrapMode, llvm::BasicBlock * scope = nullptr, bool permitCache = true )

finaloverridevirtual

if full unwrap, don't just unwrap this instruction, but also its operands, etc

Implements CacheUtility.

Definition at line 723 of file GradientUtils.cpp.

Referenced by DiffeGradientUtils::freeCache(), and AdjointGenerator::visitOMPCall().

usedInRooting()

bool GradientUtils::usedInRooting	(	const llvm::CallBase *	orig,
		llvm::ArrayRef< ValueType >	types,
		const llvm::Value *	val,
		bool	shadow ) const

Definition at line 257 of file GradientUtils.cpp.

Member Data Documentation

allocationsToBeRematerialized

llvm::SmallPtrSet<llvm::Value *, 4> GradientUtils::allocationsToBeRematerialized

Definition at line 310 of file GradientUtils.h.

Referenced by AdjointGenerator::handleKnownCallDerivatives(), DifferentialUseAnalysis::is_value_needed_in_reverse(), AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitStoreInst().

allocationsWithGuaranteedFree

llvm::ValueMap<const llvm::CallInst *, llvm::SmallPtrSet<const llvm::CallInst *, 1> > GradientUtils::allocationsWithGuaranteedFree

Allocations which are known to always be freed before the reverse, to the list of frees that must apply to this allocation.

Definition at line 160 of file GradientUtils.h.

ArgDiffeTypes

llvm::ArrayRef<DIFFE_TYPE> GradientUtils::ArgDiffeTypes

Definition at line 415 of file GradientUtils.h.

ATA

std::shared_ptr<ActivityAnalyzer> GradientUtils::ATA

Definition at line 153 of file GradientUtils.h.

AtomicAdd

bool GradientUtils::AtomicAdd

Definition at line 140 of file GradientUtils.h.

Referenced by EnzymeGradientUtilsGetAtomicAdd(), and AdjointGenerator::recursivelyHandleSubfunction().

backwardsOnlyShadows

llvm::ValueMap<llvm::Value *, ShadowRematerializer> GradientUtils::backwardsOnlyShadows

Only loaded from and stored to (not captured), mapped to the stores (and memset).

Boolean denotes whether the primal initializes the shadow as well (for use) as a structure which carries data.

Definition at line 315 of file GradientUtils.h.

Referenced by AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::shadowStoreSchedule(), and AdjointGenerator::visitOMPCall().

BlocksDominatingAllReturns

llvm::SmallPtrSet<llvm::BasicBlock *, 4> GradientUtils::BlocksDominatingAllReturns

(Original) Blocks which dominate all returns

Definition at line 150 of file GradientUtils.h.

can_modref_map

const std::map<llvm::Instruction *, bool>* GradientUtils::can_modref_map

Definition at line 188 of file GradientUtils.h.

Referenced by AdjointGenerator::visitLoadLike().

differentialAliasScope

llvm::ValueMap<const llvm::Value *, llvm::DenseMap<ssize_t, llvm::MDNode *> > GradientUtils::differentialAliasScope

Definition at line 458 of file GradientUtils.h.

differentialAliasScopeDomains

llvm::ValueMap<const llvm::Value *, llvm::MDNode *> GradientUtils::differentialAliasScopeDomains

Definition at line 456 of file GradientUtils.h.

fictiousPHIs

llvm::ValueMap<llvm::PHINode *, llvm::WeakTrackingVH> GradientUtils::fictiousPHIs

Definition at line 181 of file GradientUtils.h.

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

forwardDeallocations

llvm::SmallPtrSet<const llvm::CallInst *, 1> GradientUtils::forwardDeallocations

Deallocations that should be kept in the forward pass because they deallocation memory which isn't necessary for the reverse pass.

Definition at line 169 of file GradientUtils.h.

invertedPointers

llvm::ValueMap<const llvm::Value *, InvertedPointerVH> GradientUtils::invertedPointers

Definition at line 143 of file GradientUtils.h.

Referenced by AdjointGenerator::createBinaryOperatorDual(), DiffeGradientUtils::CreateFromClone(), EnzymeGradientUtilsInvertedPointersToString(), AdjointGenerator::forwardModeInvertedPointerFallback(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::resolveShadowPlaceholder(), DiffeGradientUtils::setDiffe(), AdjointGenerator::visitCallInst(), and AdjointGenerator::visitLoadLike().

knownRecomputeHeuristic

std::map<const llvm::Value *, bool> GradientUtils::knownRecomputeHeuristic

Definition at line 357 of file GradientUtils.h.

Referenced by AdjointGenerator::eraseIfUnused(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitIntrinsicInst(), and AdjointGenerator::visitLoadLike().

lcssaFixes

std::map<llvm::Instruction *, llvm::ValueMap<llvm::BasicBlock *, llvm::WeakTrackingVH> > GradientUtils::lcssaFixes

Definition at line 536 of file GradientUtils.h.

lcssaPHIToOrig

std::map<llvm::PHINode *, llvm::WeakTrackingVH> GradientUtils::lcssaPHIToOrig

Definition at line 537 of file GradientUtils.h.

Logic

EnzymeLogic& GradientUtils::Logic

Definition at line 139 of file GradientUtils.h.

Referenced by DiffeGradientUtils::CreateFromClone(), EnzymeGradientUtilsGetExternalContext(), EnzymeGradientUtilsGetLogic(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitCallInst(), and AdjointGenerator::visitOMPCall().

mode

DerivativeMode GradientUtils::mode

Definition at line 141 of file GradientUtils.h.

Referenced by DifferentialUseAnalysis::__attribute__(), DiffeGradientUtils::CreateFromClone(), DiffeGradientUtils::diffe(), EnzymeGradientUtilsGetMode(), EnzymeGradientUtilsGetUncacheableArgs(), DiffeGradientUtils::getDifferential(), DifferentialUseAnalysis::is_value_needed_in_reverse(), and DiffeGradientUtils::setDiffe().

newBlocksForLoop_cache

std::map<std::tuple<llvm::BasicBlock *, llvm::BasicBlock *>, llvm::BasicBlock *> GradientUtils::newBlocksForLoop_cache

This cache stores blocks we may insert as part of getReverseOrLatchMerge to handle inverse iv iteration.

Definition at line 480 of file GradientUtils.h.

newToOriginalFn

llvm::ValueMap<const llvm::Value *, AssertingReplacingVH> GradientUtils::newToOriginalFn

Definition at line 183 of file GradientUtils.h.

Referenced by EnzymeReplaceOriginalToNew(), and AdjointGenerator::recursivelyHandleSubfunction().

notForAnalysis

llvm::SmallPtrSet<llvm::BasicBlock *, 4> GradientUtils::notForAnalysis

Definition at line 152 of file GradientUtils.h.

Referenced by DifferentialUseAnalysis::__attribute__(), and hasSingleGCPreserveEnd().

nowrite_shadows

std::vector<bool> GradientUtils::nowrite_shadows

Definition at line 316 of file GradientUtils.h.

Referenced by AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::shadowStoreSchedule(), and AdjointGenerator::visitOMPCall().

numThreads

llvm::Value* GradientUtils::numThreads

Definition at line 212 of file GradientUtils.h.

oldFunc

llvm::Function* GradientUtils::oldFunc

Definition at line 142 of file GradientUtils.h.

Referenced by AdjointGenerator::AdjointGenerator(), calculateUnusedStores(), calculateUnusedValues(), AdjointGenerator::createBinaryOperatorAdjoint(), AdjointGenerator::createBinaryOperatorDual(), DiffeGradientUtils::CreateFromClone(), DiffeGradientUtils::diffe(), EnzymeGradientUtilsGetUncacheableArgs(), DiffeGradientUtils::getDifferential(), AdjointGenerator::handleAdjointForIntrinsic(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), DifferentialUseAnalysis::is_value_needed_in_reverse(), AdjointGenerator::recursivelyHandleSubfunction(), DiffeGradientUtils::setDiffe(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitExtractValueInst(), and AdjointGenerator::visitOMPCall().

omp

bool GradientUtils::omp

Definition at line 401 of file GradientUtils.h.

Referenced by DiffeGradientUtils::CreateFromClone().

OrigAA

llvm::AAResults* GradientUtils::OrigAA

Definition at line 398 of file GradientUtils.h.

OrigDT

llvm::DominatorTree* GradientUtils::OrigDT

Definition at line 144 of file GradientUtils.h.

originalBlocks

llvm::SmallVector<llvm::BasicBlock *, 12> GradientUtils::originalBlocks

Definition at line 154 of file GradientUtils.h.

originalCalls

llvm::SmallVector<llvm::CallInst *, 4> GradientUtils::originalCalls

Definition at line 184 of file GradientUtils.h.

originalToNewFn

llvm::ValueMap<const llvm::Value *, AssertingReplacingVH> GradientUtils::originalToNewFn

Definition at line 182 of file GradientUtils.h.

Referenced by EnzymeReplaceOriginalToNew(), and AdjointGenerator::recursivelyHandleSubfunction().

OrigLI

llvm::LoopInfo* GradientUtils::OrigLI

Definition at line 146 of file GradientUtils.h.

Referenced by DifferentialUseAnalysis::checkLoopyReductionPHI(), AdjointGenerator::createSelectInstAdjoint(), DifferentialUseAnalysis::pushLoopyPHIPreheader(), and AdjointGenerator::visitBinaryOperator().

OrigPDT

llvm::PostDominatorTree* GradientUtils::OrigPDT

Definition at line 145 of file GradientUtils.h.

OrigSE

llvm::ScalarEvolution* GradientUtils::OrigSE

Definition at line 147 of file GradientUtils.h.

overwritten_args_map_ptr

const std::map<llvm::CallInst *, std::pair<bool, const std::vector<bool> > >* GradientUtils::overwritten_args_map_ptr

Definition at line 190 of file GradientUtils.h.

Referenced by EnzymeGradientUtilsGetUncacheableArgs().

postDominatingFrees

llvm::SmallPtrSet<const llvm::CallInst *, 1> GradientUtils::postDominatingFrees

Frees which can always be eliminated as the post dominate an allocation (which will itself be freed).

Definition at line 164 of file GradientUtils.h.

primalKnownRecomputeHeuristicP

const std::map<const llvm::Value *, bool>* GradientUtils::primalKnownRecomputeHeuristicP

Initial value:

=
      nullptr

Definition at line 195 of file GradientUtils.h.

primalUnnecessaryValuesP

const std::set<const llvm::Value *>* GradientUtils::primalUnnecessaryValuesP = nullptr

In the reverse pass of split mode, the augmented pass's unnecessary values and recompute decisions, to answer whether the primal pass used a value.

Definition at line 194 of file GradientUtils.h.

rematerializableAllocations

llvm::ValueMap<llvm::Value *, Rematerializer> GradientUtils::rematerializableAllocations

Definition at line 309 of file GradientUtils.h.

Referenced by AdjointGenerator::handleKnownCallDerivatives(), DifferentialUseAnalysis::is_value_needed_in_reverse(), AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitStoreInst().

rematerializedLoops_cache

std::map<const llvm::BasicBlock *, llvm::BasicBlock *> GradientUtils::rematerializedLoops_cache

This cache stores a rematerialized forward pass in the loop specified.

The key is the loop header.

Definition at line 485 of file GradientUtils.h.

rematerializedPrimalOrShadowAllocations

llvm::SmallVector<llvm::Instruction *, 1> GradientUtils::rematerializedPrimalOrShadowAllocations

Definition at line 504 of file GradientUtils.h.

Referenced by AdjointGenerator::handleKnownCallDerivatives().

reverseBlocks

std::map<llvm::BasicBlock *, llvm::SmallVector<llvm::BasicBlock *, 4> > GradientUtils::reverseBlocks

Map of primal block to corresponding block(s) in reverse.

Definition at line 173 of file GradientUtils.h.

Referenced by EnzymeGradientUtilsSetReverseBlock(), DiffeGradientUtils::freeCache(), AdjointGenerator::handleKnownCallDerivatives(), and AdjointGenerator::handleMPI().

reverseBlockToPrimal

std::map<llvm::BasicBlock *, llvm::BasicBlock *> GradientUtils::reverseBlockToPrimal

Map of block in reverse to corresponding primal block.

Definition at line 175 of file GradientUtils.h.

Referenced by EnzymeGradientUtilsSetReverseBlock(), AdjointGenerator::handleKnownCallDerivatives(), and AdjointGenerator::handleMPI().

runtimeActivity

bool GradientUtils::runtimeActivity

Definition at line 402 of file GradientUtils.h.

Referenced by DiffeGradientUtils::CreateFromClone(), EnzymeGradientUtilsGetRuntimeActivity(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitLoadLike(), AdjointGenerator::visitMemTransferCommon(), and AdjointGenerator::visitOMPCall().

shadowReturnUsed

bool GradientUtils::shadowReturnUsed

Definition at line 413 of file GradientUtils.h.

strongZero

bool GradientUtils::strongZero

Definition at line 405 of file GradientUtils.h.

Referenced by AdjointGenerator::createBinaryOperatorAdjoint(), AdjointGenerator::createBinaryOperatorDual(), DiffeGradientUtils::CreateFromClone(), EnzymeGradientUtilsGetStrongZero(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitBinaryOperator(), and AdjointGenerator::visitOMPCall().

TA

TypeAnalysis& GradientUtils::TA

Definition at line 399 of file GradientUtils.h.

Referenced by DiffeGradientUtils::CreateFromClone().

TapesToPreventRecomputation

llvm::SmallPtrSet<llvm::Instruction *, 4> GradientUtils::TapesToPreventRecomputation

A set of tape extractions to enforce a cache of rather than attempting to recompute.

Definition at line 179 of file GradientUtils.h.

Referenced by AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), and AdjointGenerator::recursivelyHandleSubfunction().

tid

llvm::Value* GradientUtils::tid

Definition at line 209 of file GradientUtils.h.

TR

TypeResults GradientUtils::TR

Definition at line 400 of file GradientUtils.h.

Referenced by DiffeGradientUtils::CreateFromClone(), EmitNoDerivativeError(), EmitNoTypeError(), EnzymeGradientUtilsAllocAndGetTypeTree(), EnzymeGradientUtilsDumpTypeResults(), EnzymeGradientUtilsTypeAnalyzer(), and DifferentialUseAnalysis::is_value_needed_in_reverse().

unnecessaryIntermediates

llvm::SmallPtrSet<llvm::Instruction *, 4> GradientUtils::unnecessaryIntermediates

Definition at line 186 of file GradientUtils.h.

Referenced by AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitCallInst(), and AdjointGenerator::visitLoadLike().

unnecessaryValuesP

const llvm::SmallPtrSetImpl<const llvm::Value *>* GradientUtils::unnecessaryValuesP

Definition at line 191 of file GradientUtils.h.

unwrappedLoads

llvm::ValueMap<const llvm::Instruction *, AssertingReplacingVH> GradientUtils::unwrappedLoads

Definition at line 363 of file GradientUtils.h.

Referenced by AdjointGenerator::visitOMPCall().

---

The documentation for this class was generated from the following files:

• MLIR/Interfaces/GradientUtils.h
• MLIR/Interfaces/GradientUtils.cpp