AdjointGenerator Class Reference

#include "AdjointGenerator.h"

Inheritance diagram for AdjointGenerator:

Collaboration diagram for AdjointGenerator:

Public Member Functions
	AdjointGenerator (DerivativeMode Mode, GradientUtils *gutils, llvm::ArrayRef< DIFFE_TYPE > constant_args, DIFFE_TYPE retType, std::function< unsigned(llvm::Instruction *, CacheType, llvm::IRBuilder<> &)> getIndex, const std::map< llvm::CallInst *, std::pair< bool, const std::vector< bool > > > overwritten_args_map, const AugmentedReturn *augmentedReturn, const std::map< llvm::ReturnInst *, llvm::StoreInst * > *replacedReturns, const llvm::SmallPtrSetImpl< const llvm::Value * > &unnecessaryValues, const llvm::SmallPtrSetImpl< const llvm::Instruction * > &unnecessaryInstructions, const llvm::SmallPtrSetImpl< const llvm::Instruction * > &unnecessaryStores, const llvm::SmallPtrSetImpl< llvm::BasicBlock * > &oldUnreachable)
void	eraseIfUnused (llvm::Instruction &I, bool erase=true, bool check=true)
llvm::Value *	MPI_TYPE_SIZE (llvm::Value *DT, llvm::IRBuilder<> &B, llvm::Type *intType, llvm::Function *caller)
llvm::Value *	MPI_COMM_RANK (llvm::Value *comm, llvm::IRBuilder<> &B, llvm::Type *rankTy, llvm::Function *caller)
llvm::Value *	MPI_COMM_SIZE (llvm::Value *comm, llvm::IRBuilder<> &B, llvm::Type *rankTy, llvm::Function *caller)
void	visitInstruction (llvm::Instruction &inst)
void	forwardModeInvertedPointerFallback (llvm::Instruction &I)
void	visitAllocaInst (llvm::AllocaInst &I)
void	visitICmpInst (llvm::ICmpInst &I)
void	visitFCmpInst (llvm::FCmpInst &I)
void	visitLoadLike (llvm::Instruction &I, llvm::MaybeAlign alignment, bool constantval, llvm::Value *mask=nullptr, llvm::Value *orig_maskInit=nullptr)
void	visitLoadInst (llvm::LoadInst &LI)
void	visitAtomicRMWInst (llvm::AtomicRMWInst &I)
void	visitStoreInst (llvm::StoreInst &SI)
void	visitCommonStore (llvm::Instruction &I, llvm::Value *orig_ptr, llvm::Value *orig_val, llvm::MaybeAlign prevalign, bool isVolatile, llvm::AtomicOrdering ordering, llvm::SyncScope::ID syncScope, llvm::Value *mask)
void	visitGetElementPtrInst (llvm::GetElementPtrInst &gep)
void	visitPHINode (llvm::PHINode &phi)
void	visitCastInst (llvm::CastInst &I)
void	visitSelectInst (llvm::SelectInst &SI)
void	createSelectInstAdjoint (llvm::SelectInst &SI)
void	visitExtractElementInst (llvm::ExtractElementInst &EEI)
void	visitInsertElementInst (llvm::InsertElementInst &IEI)
void	visitShuffleVectorInst (llvm::ShuffleVectorInst &SVI)
void	visitExtractValueInst (llvm::ExtractValueInst &EVI)
void	visitInsertValueInst (llvm::InsertValueInst &IVI)
void	getReverseBuilder (llvm::IRBuilder<> &Builder2, bool original=true)
void	getForwardBuilder (llvm::IRBuilder<> &Builder2)
llvm::Value *	diffe (llvm::Value *val, llvm::IRBuilder<> &Builder)
void	setDiffe (llvm::Value *val, llvm::Value *dif, llvm::IRBuilder<> &Builder)
template<typename Func , typename... Args>
llvm::Value *	applyChainRule (llvm::Type *diffType, llvm::IRBuilder<> &Builder, Func rule, Args... args)
	Unwraps a vector derivative from its internal representation and applies a function f to each element.
template<typename Func , typename... Args>
void	applyChainRule (llvm::IRBuilder<> &Builder, Func rule, Args... args)
	Unwraps a vector derivative from its internal representation and applies a function f to each element.
template<typename Func >
void	applyChainRule (llvm::ArrayRef< llvm::Value * > diffs, llvm::IRBuilder<> &Builder, Func rule)
	Unwraps an collection of constant vector derivatives from their internal representations and applies a function f to each element.
bool	shouldFree ()
llvm::SmallVector< llvm::SelectInst *, 4 >	addToDiffe (llvm::Value *val, llvm::Value *dif, llvm::IRBuilder<> &Builder, llvm::Type *T, llvm::Value *mask=nullptr)
llvm::Value *	lookup (llvm::Value *val, llvm::IRBuilder<> &Builder)
void	visitBinaryOperator (llvm::BinaryOperator &BO)
void	createBinaryOperatorAdjoint (llvm::BinaryOperator &BO)
void	createBinaryOperatorDual (llvm::BinaryOperator &BO)
void	visitMemSetInst (llvm::MemSetInst &MS)
void	visitMemSetCommon (llvm::CallInst &MS)
void	visitMemTransferInst (llvm::MemTransferInst &MTI)
void	visitMemTransferCommon (llvm::Intrinsic::ID ID, llvm::MaybeAlign srcAlign, llvm::MaybeAlign dstAlign, llvm::CallInst &MTI, llvm::Value *orig_dst, llvm::Value *orig_src, llvm::Value *new_size, llvm::Value *isVolatile)
void	visitFenceInst (llvm::FenceInst &FI)
void	visitIntrinsicInst (llvm::IntrinsicInst &II)
bool	handleAdjointForIntrinsic (llvm::Intrinsic::ID ID, llvm::Instruction &I, llvm::SmallVectorImpl< llvm::Value * > &orig_ops)
void	visitOMPCall (llvm::CallInst &call)
void	DifferentiableMemCopyFloats (llvm::CallInst &call, llvm::Value *origArg, llvm::Value *dsto, llvm::Value *srco, llvm::Value *len_arg, llvm::IRBuilder<> &Builder2, llvm::ArrayRef< llvm::OperandBundleDef > ReverseDefs)
void	recursivelyHandleSubfunction (llvm::CallInst &call, llvm::Function *called, bool subsequent_calls_may_write, const std::vector< bool > &overwritten_args, bool shadowReturnUsed, DIFFE_TYPE subretType, bool subretused)
void	handleMPI (llvm::CallInst &call, llvm::Function *called, llvm::StringRef funcName)
bool	handleKnownCallDerivatives (llvm::CallInst &call, llvm::Function *called, llvm::StringRef funcName, bool subsequent_calls_may_write, const std::vector< bool > &overwritten_args, llvm::CallInst *const newCall)
void	visitCallInst (llvm::CallInst &call)

Detailed Description

Definition at line 54 of file AdjointGenerator.h.

Constructor & Destructor Documentation

AdjointGenerator()

AdjointGenerator::AdjointGenerator ( DerivativeMode Mode, GradientUtils * gutils, llvm::ArrayRef< DIFFE_TYPE > constant_args, DIFFE_TYPE retType, std::function< unsigned(llvm::Instruction *, CacheType, llvm::IRBuilder<> &)> getIndex, const std::map< llvm::CallInst *, std::pair< bool, const std::vector< bool > > > overwritten_args_map, const AugmentedReturn * augmentedReturn, const std::map< llvm::ReturnInst *, llvm::StoreInst * > * replacedReturns, const llvm::SmallPtrSetImpl< const llvm::Value * > & unnecessaryValues, const llvm::SmallPtrSetImpl< const llvm::Instruction * > & unnecessaryInstructions, const llvm::SmallPtrSetImpl< const llvm::Instruction * > & unnecessaryStores, const llvm::SmallPtrSetImpl< llvm::BasicBlock * > & oldUnreachable )

inline

Definition at line 77 of file AdjointGenerator.h.

References TypeAnalyzer::analysis, TypeResults::analyzer, TypeResults::getFunction(), and GradientUtils::oldFunc.

Member Function Documentation

addToDiffe()

llvm::SmallVector< llvm::SelectInst *, 4 > AdjointGenerator::addToDiffe ( llvm::Value * val, llvm::Value * dif, llvm::IRBuilder<> & Builder, llvm::Type * T, llvm::Value * mask = nullptr )

inline

Definition at line 2231 of file AdjointGenerator.h.

Referenced by createBinaryOperatorAdjoint(), createSelectInstAdjoint(), handleAdjointForIntrinsic(), recursivelyHandleSubfunction(), visitAtomicRMWInst(), visitBinaryOperator(), visitCastInst(), visitInsertElementInst(), and visitLoadLike().

applyChainRule() [1/3]

template<typename Func >

void AdjointGenerator::applyChainRule ( llvm::ArrayRef< llvm::Value * > diffs, llvm::IRBuilder<> & Builder, Func rule )

inline

Unwraps an collection of constant vector derivatives from their internal representations and applies a function f to each element.

Definition at line 2218 of file AdjointGenerator.h.

applyChainRule() [2/3]

template<typename Func , typename... Args>

void AdjointGenerator::applyChainRule ( llvm::IRBuilder<> & Builder, Func rule, Args... args )

inline

Unwraps a vector derivative from its internal representation and applies a function f to each element.

Definition at line 2211 of file AdjointGenerator.h.

applyChainRule() [3/3]

template<typename Func , typename... Args>

llvm::Value * AdjointGenerator::applyChainRule ( llvm::Type * diffType, llvm::IRBuilder<> & Builder, Func rule, Args... args )

inline

Unwraps a vector derivative from its internal representation and applies a function f to each element.

Return values of f are collected and wrapped.

Definition at line 2202 of file AdjointGenerator.h.

Referenced by createBinaryOperatorAdjoint(), createBinaryOperatorDual(), handleAdjointForIntrinsic(), handleKnownCallDerivatives(), visitAtomicRMWInst(), visitBinaryOperator(), visitCastInst(), visitCommonStore(), visitInsertValueInst(), visitLoadLike(), visitMemSetCommon(), and visitMemTransferCommon().

createBinaryOperatorAdjoint()

void AdjointGenerator::createBinaryOperatorAdjoint ( llvm::BinaryOperator & BO )

inline

Definition at line 2375 of file AdjointGenerator.h.

References TypeResults::addingType(), addToDiffe(), applyChainRule(), checkedMul(), containsOnlyAtMostTopBit(), CreateSelect(), diffe(), EmitNoDerivativeError(), GradientUtils::getNewFromOriginal(), getReverseBuilder(), GradientUtils::getShadowType(), TypeTree::IsAllFloat(), GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), isZero(), TypeResults::knownIntegralValues(), lookup(), looseTypeAnalysis, CacheUtility::newFunc, GradientUtils::oldFunc, TypeResults::query(), setDiffe(), TypeTree::str(), and GradientUtils::strongZero.

Referenced by visitBinaryOperator().

createBinaryOperatorDual()

void AdjointGenerator::createBinaryOperatorDual ( llvm::BinaryOperator & BO )

inline

Definition at line 2656 of file AdjointGenerator.h.

References TypeResults::addingType(), applyChainRule(), checkedMul(), containsOnlyAtMostTopBit(), CreateSelect(), diffe(), EmitNoDerivativeError(), GradientUtils::erase(), forwardModeInvertedPointerFallback(), getForwardBuilder(), GradientUtils::getNewFromOriginal(), GradientUtils::getShadowType(), GradientUtils::invertedPointers, TypeTree::IsAllFloat(), GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), isZero(), TypeResults::knownIntegralValues(), looseTypeAnalysis, CacheUtility::newFunc, GradientUtils::oldFunc, TypeResults::query(), GradientUtils::replaceAWithB(), setDiffe(), TypeTree::str(), and GradientUtils::strongZero.

Referenced by visitBinaryOperator().

createSelectInstAdjoint()

void AdjointGenerator::createSelectInstAdjoint ( llvm::SelectInst & SI )

inline

Definition at line 1498 of file AdjointGenerator.h.

References TypeResults::addingType(), addToDiffe(), CreateSelect(), diffe(), LoopContext::exitBlocks, GradientUtils::getContext(), GradientUtils::getNewFromOriginal(), GradientUtils::getOrInsertConditionalIndex(), getReverseBuilder(), GradientUtils::getShadowType(), LoopContext::header, LoopContext::incvar, GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), lookup(), GradientUtils::lookupM(), looseTypeAnalysis, CacheUtility::newFunc, GradientUtils::OrigLI, OUT_DIFF, and setDiffe().

Referenced by visitSelectInst().

diffe()

llvm::Value * AdjointGenerator::diffe ( llvm::Value * val, llvm::IRBuilder<> & Builder )

inline

Definition at line 2188 of file AdjointGenerator.h.

References ReverseModePrimal.

Referenced by createBinaryOperatorAdjoint(), createBinaryOperatorDual(), createSelectInstAdjoint(), handleAdjointForIntrinsic(), recursivelyHandleSubfunction(), visitBinaryOperator(), visitCastInst(), visitExtractElementInst(), visitExtractValueInst(), visitInsertElementInst(), visitInsertValueInst(), visitLoadLike(), and visitShuffleVectorInst().

DifferentiableMemCopyFloats()

void AdjointGenerator::DifferentiableMemCopyFloats ( llvm::CallInst & call, llvm::Value * origArg, llvm::Value * dsto, llvm::Value * srco, llvm::Value * len_arg, llvm::IRBuilder<> & Builder2, llvm::ArrayRef< llvm::OperandBundleDef > ReverseDefs )

inline

Definition at line 4801 of file AdjointGenerator.h.

References TypeTree::Data0(), TypeResults::dump(), EmitFailure(), getInt8PtrTy(), getOrInsertDifferentialFloatMemcpy(), getPointerType(), looseTypeAnalysis, CacheUtility::newFunc, TypeTree::Only(), TypeResults::query(), and TypeTree::ShiftIndices().

Referenced by handleMPI().

eraseIfUnused()

void AdjointGenerator::eraseIfUnused ( llvm::Instruction & I, bool erase = true, bool check = true )

inline

Definition at line 114 of file AdjointGenerator.h.

References GradientUtils::eraseWithPlaceholder(), GradientUtils::getNewFromOriginal(), and GradientUtils::knownRecomputeHeuristic.

Referenced by handleKnownCallDerivatives(), handleMPI(), recursivelyHandleSubfunction(), visitAllocaInst(), visitAtomicRMWInst(), visitBinaryOperator(), visitCallInst(), visitCastInst(), visitExtractElementInst(), visitExtractValueInst(), visitFCmpInst(), visitFenceInst(), visitGetElementPtrInst(), visitICmpInst(), visitInsertElementInst(), visitInsertValueInst(), visitInstruction(), visitIntrinsicInst(), visitLoadInst(), visitMemSetCommon(), visitMemTransferCommon(), visitOMPCall(), visitPHINode(), visitSelectInst(), visitShuffleVectorInst(), and visitStoreInst().

forwardModeInvertedPointerFallback()

void AdjointGenerator::forwardModeInvertedPointerFallback ( llvm::Instruction & I )

inline

Definition at line 313 of file AdjointGenerator.h.

References GradientUtils::erase(), getForwardBuilder(), GradientUtils::invertedPointers, GradientUtils::invertPointerM(), DifferentialUseAnalysis::is_value_needed_in_reverse(), GradientUtils::isConstantValue(), and GradientUtils::replaceAWithB().

Referenced by createBinaryOperatorDual(), visitAllocaInst(), visitCastInst(), visitExtractElementInst(), visitExtractValueInst(), visitGetElementPtrInst(), visitInsertElementInst(), visitInsertValueInst(), visitIntrinsicInst(), visitPHINode(), visitSelectInst(), and visitShuffleVectorInst().

getForwardBuilder()

void AdjointGenerator::getForwardBuilder ( llvm::IRBuilder<> & Builder2 )

inline

Definition at line 2184 of file AdjointGenerator.h.

Referenced by createBinaryOperatorDual(), forwardModeInvertedPointerFallback(), handleAdjointForIntrinsic(), handleKnownCallDerivatives(), handleMPI(), recursivelyHandleSubfunction(), visitAtomicRMWInst(), visitInstruction(), and visitMemSetCommon().

getReverseBuilder()

void AdjointGenerator::getReverseBuilder ( llvm::IRBuilder<> & Builder2, bool original = true )

inline

Definition at line 2180 of file AdjointGenerator.h.

Referenced by createBinaryOperatorAdjoint(), createSelectInstAdjoint(), handleAdjointForIntrinsic(), handleKnownCallDerivatives(), handleMPI(), recursivelyHandleSubfunction(), visitAtomicRMWInst(), visitBinaryOperator(), visitCallInst(), visitCastInst(), visitCommonStore(), visitExtractElementInst(), visitExtractValueInst(), visitFenceInst(), visitInsertElementInst(), visitInsertValueInst(), visitLoadLike(), visitMemSetCommon(), visitOMPCall(), and visitShuffleVectorInst().

handleAdjointForIntrinsic()

bool AdjointGenerator::handleAdjointForIntrinsic ( llvm::Intrinsic::ID ID, llvm::Instruction & I, llvm::SmallVectorImpl< llvm::Value * > & orig_ops )

inline

Definition at line 3876 of file AdjointGenerator.h.

References addToDiffe(), applyChainRule(), CreateSelect(), diffe(), EmitNoDerivativeError(), EmitWarning(), ForwardMode, ForwardModeError, ForwardModeSplit, getForwardBuilder(), getIntrinsicDeclaration(), GradientUtils::getNewFromOriginal(), getReverseBuilder(), GradientUtils::getShadowType(), GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), lookup(), looseTypeAnalysis, CacheUtility::newFunc, GradientUtils::oldFunc, parseTBAA(), ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, setDiffe(), visitCommonStore(), and visitLoadLike().

Referenced by handleKnownCallDerivatives(), visitCallInst(), and visitIntrinsicInst().

handleKnownCallDerivatives()

bool AdjointGenerator::handleKnownCallDerivatives	(	llvm::CallInst &	call,
		llvm::Function *	called,
		llvm::StringRef	funcName,
		bool	subsequent_calls_may_write,
		const std::vector< bool > &	overwritten_args,
		llvm::CallInst *const	newCall )

Definition at line 2212 of file CallDerivatives.cpp.

References GradientUtils::addReverseBlock(), applyChainRule(), GradientUtils::backwardsOnlyShadows, GradientUtils::cacheForReverse(), CreateAllocation(), CreateDealloc(), DUP_ARG, EmitNoDerivativeError(), EnzymeFreeInternalAllocations, EnzymeJuliaAddrLoad, EnzymeShadowAllocRewrite, GradientUtils::erase(), eraseIfUnused(), extractBLAS(), GradientUtils::extractMeta(), ForwardMode, ForwardModeError, ForwardModeSplit, freeKnownAllocation(), getFast(), getForwardBuilder(), GradientUtils::getInvertedBundles(), GradientUtils::getNewFromOriginal(), GradientUtils::getReturnDiffeType(), getReverseBuilder(), getUndefinedValueForType(), getUnqual(), GradientUtils::getWidth(), handleAdjointForIntrinsic(), handleMPI(), hasMetadata(), GradientUtils::invertedPointers, GradientUtils::invertPointerM(), DifferentialUseAnalysis::is_value_needed_in_reverse(), isAllocationFunction(), GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), isMemFreeLibMFunction(), GradientUtils::knownRecomputeHeuristic, GradientUtils::legalRecompute(), lookup(), GradientUtils::lookupM(), MPIInactiveCommAllocators, CacheUtility::newFunc, None, GradientUtils::oldFunc, Primal, GradientUtils::rematerializableAllocations, GradientUtils::rematerializedPrimalOrShadowAllocations, GradientUtils::replaceAWithB(), GradientUtils::reverseBlocks, GradientUtils::reverseBlockToPrimal, ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, Self, Shadow, shadowHandlers, shouldFree(), startsWith(), CacheUtility::TLI, GradientUtils::unnecessaryIntermediates, and zeroKnownAllocation().

Referenced by visitCallInst().

handleMPI()

void AdjointGenerator::handleMPI	(	llvm::CallInst &	call,
		llvm::Function *	called,
		llvm::StringRef	funcName )

Definition at line 36 of file CallDerivatives.cpp.

References GradientUtils::addReverseBlock(), GradientUtils::cacheForReverse(), Constant, CreateAllocation(), CreateDealloc(), CreateSelect(), DifferentiableMemCopyFloats(), EmitNoDerivativeError(), eraseIfUnused(), TypeResults::firstPointer(), ForwardMode, ForwardModeError, getFast(), getForwardBuilder(), getInt8PtrTy(), getIntrinsicDeclaration(), GradientUtils::getInvertedBundles(), getMPIHelper(), getMPIMemberPtr(), GradientUtils::getNewFromOriginal(), getOrInsertDifferentialMPI_Wait(), getOrInsertDifferentialWaitallSave(), getOrInsertOpFloatSum(), getRenamedPerCallingConv(), getReverseBuilder(), getUnqual(), GradientUtils::getWidth(), CacheUtility::inversionAllocs, GradientUtils::invertPointerM(), IRECV, GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), ISEND, isSum(), lookup(), MPI_COMM_RANK(), MPI_COMM_SIZE(), MPI_TYPE_SIZE(), MPIInactiveCommAllocators, CacheUtility::newFunc, None, GradientUtils::oldFunc, Primal, GradientUtils::reverseBlocks, GradientUtils::reverseBlockToPrimal, ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, Shadow, shouldFree(), Tape, and GradientUtils::TapesToPreventRecomputation.

Referenced by handleKnownCallDerivatives().

lookup()

llvm::Value * AdjointGenerator::lookup ( llvm::Value * val, llvm::IRBuilder<> & Builder )

inline

Definition at line 2237 of file AdjointGenerator.h.

References GradientUtils::lookupM().

Referenced by createBinaryOperatorAdjoint(), createSelectInstAdjoint(), handleAdjointForIntrinsic(), handleKnownCallDerivatives(), handleMPI(), recursivelyHandleSubfunction(), visitAtomicRMWInst(), visitBinaryOperator(), visitCommonStore(), visitInsertElementInst(), visitLoadLike(), and visitOMPCall().

MPI_COMM_RANK()

llvm::Value * AdjointGenerator::MPI_COMM_RANK ( llvm::Value * comm, llvm::IRBuilder<> & B, llvm::Type * rankTy, llvm::Function * caller )

inline

Definition at line 199 of file AdjointGenerator.h.

References addFunctionNoCapture(), getRenamedPerCallingConv(), getUnqual(), and CacheUtility::inversionAllocs.

Referenced by handleMPI().

MPI_COMM_SIZE()

llvm::Value * AdjointGenerator::MPI_COMM_SIZE ( llvm::Value * comm, llvm::IRBuilder<> & B, llvm::Type * rankTy, llvm::Function * caller )

inline

Definition at line 233 of file AdjointGenerator.h.

References addFunctionNoCapture(), getRenamedPerCallingConv(), getUnqual(), and CacheUtility::inversionAllocs.

Referenced by handleMPI().

MPI_TYPE_SIZE()

llvm::Value * AdjointGenerator::MPI_TYPE_SIZE ( llvm::Value * DT, llvm::IRBuilder<> & B, llvm::Type * intType, llvm::Function * caller )

inline

Definition at line 135 of file AdjointGenerator.h.

References addFunctionNoCapture(), Constant, getInt8PtrTy(), getRenamedPerCallingConv(), getUnqual(), and CacheUtility::inversionAllocs.

Referenced by handleMPI().

recursivelyHandleSubfunction()

void AdjointGenerator::recursivelyHandleSubfunction ( llvm::CallInst & call, llvm::Function * called, bool subsequent_calls_may_write, const std::vector< bool > & overwritten_args, bool shadowReturnUsed, DIFFE_TYPE subretType, bool subretused )

inline

We only need the shadow pointer for non-forward Mode if it is used in a non return setting

topLevel

Definition at line 4917 of file AdjointGenerator.h.

References TypeResults::addingType(), addToDiffe(), TypeResults::analyzer, TypeResults::anyPointer(), GradientUtils::AtomicAdd, Both, GradientUtils::cacheForReverse(), CONSTANT, AugmentedReturn::constant_args, convertSRetTypeToString(), EnzymeLogic::CreateAugmentedPrimal(), CreateDealloc(), EnzymeLogic::CreateForwardDiff(), EnzymeLogic::CreatePrimalAndGradient(), CustomErrorHandler, diffe(), DifferentialReturn, dumpMap(), DUP_ARG, DUP_NONEED, EmitFailure(), EmitNoDerivativeError(), GradientUtils::erase(), eraseIfUnused(), ErrorIfRuntimeInactive(), GradientUtils::fictiousPHIs, AugmentedReturn::fn, ForwardMode, ForwardModeError, ForwardModeSplit, getBaseObject(), TypeResults::getCallInfo(), getDefaultFunctionTypeForAugmentation(), getDefaultFunctionTypeForGradient(), GradientUtils::getDiffeType(), getFast(), getForwardBuilder(), getFuncNameFromCall(), getFunctionTypeForClone(), getInt8PtrTy(), GradientUtils::getInvertedBundles(), GradientUtils::getNewFromOriginal(), getPointerType(), getReverseBuilder(), GradientUtils::getShadowType(), getUndefinedValueForType(), getUnqual(), GradientUtils::getWidth(), IndexMappingError, insert_or_assign2(), InternalError, TypeAnalyzer::interprocedural, GradientUtils::invertedPointers, GradientUtils::invertPointerM(), DifferentialUseAnalysis::is_value_needed_in_reverse(), GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), isMemFreeLibMFunction(), isNoCapture(), GradientUtils::isOriginal(), isReadOnly(), isWriteOnly(), GradientUtils::knownRecomputeHeuristic, legalCombinedForwardReverse(), GradientUtils::Logic, lookup(), GradientUtils::lookupM(), CacheUtility::newFunc, GradientUtils::newToOriginalFn, None, GradientUtils::oldFunc, GradientUtils::originalToNewFn, OUT_DIFF, Pointer, Primal, PrimalParamAttrsToPreserve, TypeResults::query(), GradientUtils::replaceAndRemoveUnwrapCacheFor(), GradientUtils::replaceAWithB(), Return, AugmentedReturn::returns, ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, GradientUtils::runtimeActivity, Self, setDiffe(), Shadow, ShadowParamAttrsToPreserve, shouldAugmentCall(), shouldDisableNoWrite(), shouldFree(), str(), GradientUtils::strongZero, AugmentedReturn::subaugmentations, Tape, GradientUtils::TapesToPreventRecomputation, AugmentedReturn::tapeType, to_string(), GradientUtils::unnecessaryIntermediates, and whatType().

Referenced by visitCallInst().

setDiffe()

void AdjointGenerator::setDiffe ( llvm::Value * val, llvm::Value * dif, llvm::IRBuilder<> & Builder )

inline

Definition at line 2193 of file AdjointGenerator.h.

References ReverseModePrimal.

Referenced by createBinaryOperatorAdjoint(), createBinaryOperatorDual(), createSelectInstAdjoint(), handleAdjointForIntrinsic(), recursivelyHandleSubfunction(), visitAtomicRMWInst(), visitCastInst(), visitExtractElementInst(), visitExtractValueInst(), visitInsertElementInst(), visitInsertValueInst(), visitInstruction(), and visitShuffleVectorInst().

shouldFree()

bool AdjointGenerator::shouldFree ( )

inline

Definition at line 2223 of file AdjointGenerator.h.

References ForwardModeSplit, ReverseModeCombined, and ReverseModeGradient.

Referenced by handleKnownCallDerivatives(), handleMPI(), recursivelyHandleSubfunction(), and visitOMPCall().

visitAllocaInst()

void AdjointGenerator::visitAllocaInst ( llvm::AllocaInst & I )

inline

Definition at line 347 of file AdjointGenerator.h.

References eraseIfUnused(), ForwardMode, ForwardModeError, forwardModeInvertedPointerFallback(), and ForwardModeSplit.

visitAtomicRMWInst()

void AdjointGenerator::visitAtomicRMWInst ( llvm::AtomicRMWInst & I )

inline

Definition at line 740 of file AdjointGenerator.h.

References addToDiffe(), applyChainRule(), EmitNoDerivativeError(), eraseIfUnused(), TypeResults::firstPointer(), ForwardMode, ForwardModeError, ForwardModeSplit, getForwardBuilder(), GradientUtils::getNewFromOriginal(), getReverseBuilder(), GradientUtils::getShadowType(), GradientUtils::invertPointerM(), GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), lookup(), looseTypeAnalysis, CacheUtility::newFunc, ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, and setDiffe().

visitBinaryOperator()

void AdjointGenerator::visitBinaryOperator ( llvm::BinaryOperator & BO )

inline

Definition at line 2241 of file AdjointGenerator.h.

References addToDiffe(), applyChainRule(), createBinaryOperatorAdjoint(), createBinaryOperatorDual(), CreateSelect(), diffe(), eraseIfUnused(), LoopContext::exitBlocks, ForwardMode, ForwardModeError, ForwardModeSplit, GradientUtils::getContext(), GradientUtils::getNewFromOriginal(), GradientUtils::getOriginalFromNew(), GradientUtils::getOrInsertTotalMultiplicativeProduct(), getReverseBuilder(), LoopContext::header, GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), lookup(), GradientUtils::OrigLI, OUT_DIFF, LoopContext::preheader, ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, and GradientUtils::strongZero.

visitCallInst()

void AdjointGenerator::visitCallInst ( llvm::CallInst & call )

inline

Definition at line 6248 of file AdjointGenerator.h.

References GradientUtils::cacheForReverse(), DifferentialUseAnalysis::callShouldNotUseDerivative(), EnzymeLogic::CreateNoFree(), customCallHandlers, customFwdCallHandlers, EmitWarning(), EnzymeGlobalActivity, EnzymeJuliaAddrLoad, GradientUtils::erase(), eraseIfUnused(), ForwardMode, ForwardModeError, ForwardModeSplit, getBaseObject(), getFast(), getFuncNameFromCall(), getFunctionFromCall(), GradientUtils::getNewFromOriginal(), GradientUtils::getReturnDiffeType(), getReverseBuilder(), GradientUtils::getShadowType(), handleAdjointForIntrinsic(), handleKnownCallDerivatives(), IndexMappingError, insert_or_assign2(), GradientUtils::invertedPointers, DifferentialUseAnalysis::is_value_needed_in_reverse(), isAllocationCall(), GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), isSpecialPtr(), GradientUtils::knownRecomputeHeuristic, GradientUtils::legalRecompute(), GradientUtils::Logic, GradientUtils::lookupM(), Primal, TypeResults::query(), recursivelyHandleSubfunction(), GradientUtils::replaceAWithB(), ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, Self, Shadow, startsWith(), AugmentedReturn::subaugmentations, Tape, AugmentedReturn::tapeIndices, CacheUtility::TLI, GradientUtils::unnecessaryIntermediates, visitIntrinsicInst(), and visitOMPCall().

visitCastInst()

void AdjointGenerator::visitCastInst ( llvm::CastInst & I )

inline

Definition at line 1362 of file AdjointGenerator.h.

References TypeResults::addingType(), addToDiffe(), applyChainRule(), diffe(), EmitNoDerivativeError(), EmitNoTypeError(), EmitWarning(), eraseIfUnused(), ForwardMode, ForwardModeError, forwardModeInvertedPointerFallback(), ForwardModeSplit, GradientUtils::getNewFromOriginal(), getReverseBuilder(), GradientUtils::getShadowType(), Integer, GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), looseTypeAnalysis, CacheUtility::newFunc, TypeResults::query(), ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, setDiffe(), and str().

visitCommonStore()

void AdjointGenerator::visitCommonStore ( llvm::Instruction & I, llvm::Value * orig_ptr, llvm::Value * orig_val, llvm::MaybeAlign prevalign, bool isVolatile, llvm::AtomicOrdering ordering, llvm::SyncScope::ID syncScope, llvm::Value * mask )

inline

Storing a floating point value

Only need to update the reverse function

Storing an integer or pointer

Only need to update the forward function

Definition at line 922 of file AdjointGenerator.h.

References GradientUtils::addReverseBlock(), Anything, applyChainRule(), GradientUtils::backwardsOnlyShadows, Constant, CustomErrorHandler, defaultTypeTreeForLLVM(), EmitNoTypeError(), EmitWarning(), GradientUtils::extractMeta(), ForwardMode, ForwardModeError, ForwardModeSplit, getBaseObject(), GradientUtils::getDerivativeAliasScope(), getFast(), getIntrinsicDeclaration(), GradientUtils::getNewFromOriginal(), getReverseBuilder(), GradientUtils::getShadowType(), GradientUtils::getWidth(), Integer, GradientUtils::invertPointerM(), isAllocationCall(), GradientUtils::isConstantValue(), TypeTree::Lookup(), lookup(), looseTypeAnalysis, MixedActivityError, CacheUtility::newFunc, GradientUtils::oldFunc, parseTBAA(), Pointer, TypeResults::query(), ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, GradientUtils::runtimeActivity, GradientUtils::setPtrDiffe(), str(), and CacheUtility::TLI.

Referenced by handleAdjointForIntrinsic(), and visitStoreInst().

visitExtractElementInst()

void AdjointGenerator::visitExtractElementInst ( llvm::ExtractElementInst & EEI )

inline

Definition at line 1605 of file AdjointGenerator.h.

References TypeResults::addingType(), diffe(), eraseIfUnused(), GradientUtils::extractMeta(), ForwardMode, ForwardModeError, forwardModeInvertedPointerFallback(), ForwardModeSplit, GradientUtils::getNewFromOriginal(), getReverseBuilder(), GradientUtils::getShadowType(), GradientUtils::getWidth(), GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), CacheUtility::newFunc, ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, and setDiffe().

visitExtractValueInst()

void AdjointGenerator::visitExtractValueInst ( llvm::ExtractValueInst & EVI )

inline

Definition at line 1826 of file AdjointGenerator.h.

References Anything, diffe(), EmitNoTypeError(), eraseIfUnused(), ForwardMode, ForwardModeError, forwardModeInvertedPointerFallback(), ForwardModeSplit, getReverseBuilder(), GradientUtils::getShadowType(), Integer, GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), looseTypeAnalysis, CacheUtility::newFunc, GradientUtils::oldFunc, TypeResults::query(), ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, setDiffe(), and str().

visitFCmpInst()

void AdjointGenerator::visitFCmpInst ( llvm::FCmpInst & I )

inline

Definition at line 363 of file AdjointGenerator.h.

References eraseIfUnused().

visitFenceInst()

void AdjointGenerator::visitFenceInst ( llvm::FenceInst & FI )

inline

Definition at line 3778 of file AdjointGenerator.h.

References EnzymeJuliaAddrLoad, eraseIfUnused(), getReverseBuilder(), ReverseModeCombined, and ReverseModeGradient.

visitGetElementPtrInst()

void AdjointGenerator::visitGetElementPtrInst ( llvm::GetElementPtrInst & gep )

inline

Definition at line 1330 of file AdjointGenerator.h.

References eraseIfUnused(), ForwardMode, ForwardModeError, forwardModeInvertedPointerFallback(), and ForwardModeSplit.

visitICmpInst()

void AdjointGenerator::visitICmpInst ( llvm::ICmpInst & I )

inline

Definition at line 361 of file AdjointGenerator.h.

References eraseIfUnused().

visitInsertElementInst()

void AdjointGenerator::visitInsertElementInst ( llvm::InsertElementInst & IEI )

inline

Definition at line 1662 of file AdjointGenerator.h.

References TypeResults::addingType(), addToDiffe(), diffe(), eraseIfUnused(), GradientUtils::extractMeta(), ForwardMode, ForwardModeError, forwardModeInvertedPointerFallback(), ForwardModeSplit, GradientUtils::getNewFromOriginal(), getReverseBuilder(), GradientUtils::getShadowType(), GradientUtils::getWidth(), GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), lookup(), CacheUtility::newFunc, ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, and setDiffe().

visitInsertValueInst()

void AdjointGenerator::visitInsertValueInst ( llvm::InsertValueInst & IVI )

inline

Definition at line 1935 of file AdjointGenerator.h.

References Anything, applyChainRule(), diffe(), EmitNoTypeError(), eraseIfUnused(), ForwardMode, ForwardModeError, forwardModeInvertedPointerFallback(), ForwardModeSplit, getReverseBuilder(), GradientUtils::getShadowType(), TypeResults::intType(), GradientUtils::isConstantValue(), looseTypeAnalysis, CacheUtility::newFunc, TypeResults::query(), ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, setDiffe(), and str().

visitInstruction()

void AdjointGenerator::visitInstruction ( llvm::Instruction & inst )

inline

Definition at line 267 of file AdjointGenerator.h.

References EmitNoDerivativeError(), eraseIfUnused(), ForwardMode, ForwardModeError, ForwardModeSplit, getForwardBuilder(), GradientUtils::getNewFromOriginal(), GradientUtils::getShadowType(), GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), ReverseModePrimal, setDiffe(), and to_string().

visitIntrinsicInst()

void AdjointGenerator::visitIntrinsicInst ( llvm::IntrinsicInst & II )

inline

Definition at line 3820 of file AdjointGenerator.h.

References GradientUtils::cacheForReverse(), eraseIfUnused(), ForwardMode, ForwardModeError, forwardModeInvertedPointerFallback(), ForwardModeSplit, getFast(), GradientUtils::getNewFromOriginal(), handleAdjointForIntrinsic(), isIntelSubscriptIntrinsic(), GradientUtils::knownRecomputeHeuristic, Self, and visitMemSetCommon().

Referenced by visitCallInst().

visitLoadInst()

void AdjointGenerator::visitLoadInst ( llvm::LoadInst & LI )

inline

Definition at line 713 of file AdjointGenerator.h.

References eraseIfUnused(), CacheUtility::newFunc, parseTBAA(), and visitLoadLike().

visitLoadLike()

void AdjointGenerator::visitLoadLike ( llvm::Instruction & I, llvm::MaybeAlign alignment, bool constantval, llvm::Value * mask = nullptr, llvm::Value * orig_maskInit = nullptr )

inline

Store inverted pointer loads that need to be cached for use in reverse pass

Store loads that need to be cached for use in reverse pass

Definition at line 365 of file AdjointGenerator.h.

References addToDiffe(), applyChainRule(), GradientUtils::cacheForReverse(), GradientUtils::can_modref_map, CreateSelect(), defaultTypeTreeForLLVM(), diffe(), EmitNoTypeError(), EmitWarning(), GradientUtils::erase(), ForwardMode, ForwardModeError, ForwardModeSplit, GradientUtils::getDerivativeAliasScope(), GradientUtils::getNewFromOriginal(), getReverseBuilder(), GradientUtils::getShadowType(), GradientUtils::getWidth(), hasMetadata(), GradientUtils::invertedPointers, GradientUtils::invertPointerM(), DifferentialUseAnalysis::is_value_needed_in_reverse(), GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), GradientUtils::knownRecomputeHeuristic, lookup(), looseTypeAnalysis, CacheUtility::newFunc, nonmarkedglobals_inactiveloads, Primal, TypeResults::query(), ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, GradientUtils::runtimeActivity, Self, Shadow, str(), and GradientUtils::unnecessaryIntermediates.

Referenced by handleAdjointForIntrinsic(), and visitLoadInst().

visitMemSetCommon()

void AdjointGenerator::visitMemSetCommon ( llvm::CallInst & MS )

inline

Definition at line 2935 of file AdjointGenerator.h.

References applyChainRule(), GradientUtils::backwardsOnlyShadows, TypeTree::Data0(), TypeResults::dump(), EmitNoDerivativeError(), EmitNoTypeError(), EmitWarning(), eraseIfUnused(), ForwardMode, ForwardModeError, ForwardModeSplit, getBaseObject(), getForwardBuilder(), getFuncNameFromCall(), GradientUtils::getInvertedBundles(), GradientUtils::getNewFromOriginal(), getReverseBuilder(), hasMetadata(), Integer, GradientUtils::invertPointerM(), isAllocationCall(), GradientUtils::isConstantValue(), GradientUtils::lookupM(), looseTypeAnalysis, MD_ToCopy, CacheUtility::newFunc, TypeTree::Only(), parseTrueType(), Pointer, Primal, TypeResults::query(), GradientUtils::rematerializableAllocations, ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, Shadow, TypeTree::ShiftIndices(), startsWith(), str(), and CacheUtility::TLI.

Referenced by visitIntrinsicInst(), and visitMemSetInst().

visitMemSetInst()

void AdjointGenerator::visitMemSetInst ( llvm::MemSetInst & MS )

inline

Definition at line 2933 of file AdjointGenerator.h.

References visitMemSetCommon().

visitMemTransferCommon()

void AdjointGenerator::visitMemTransferCommon ( llvm::Intrinsic::ID ID, llvm::MaybeAlign srcAlign, llvm::MaybeAlign dstAlign, llvm::CallInst & MTI, llvm::Value * orig_dst, llvm::Value * orig_src, llvm::Value * new_size, llvm::Value * isVolatile )

inline

Definition at line 3419 of file AdjointGenerator.h.

References Anything, applyChainRule(), GradientUtils::backwardsOnlyShadows, TypeTree::Data0(), TypeResults::dump(), EmitNoTypeError(), EmitWarning(), eraseIfUnused(), ForwardMode, ForwardModeError, getInt8PtrTy(), GradientUtils::getNewFromOriginal(), hasMetadata(), Integer, GradientUtils::invertPointerM(), GradientUtils::isConstantValue(), looseTypeAnalysis, CacheUtility::newFunc, TypeTree::Only(), TypeTree::parse(), parseTrueType(), Pointer, TypeTree::PurgeAnything(), TypeResults::query(), GradientUtils::runtimeActivity, TypeTree::ShiftIndices(), str(), SubTransferHelper(), and Unknown.

Referenced by visitMemTransferInst().

visitMemTransferInst()

void AdjointGenerator::visitMemTransferInst ( llvm::MemTransferInst & MTI )

inline

Definition at line 3408 of file AdjointGenerator.h.

References GradientUtils::getNewFromOriginal(), and visitMemTransferCommon().

visitOMPCall()

void AdjointGenerator::visitOMPCall ( llvm::CallInst & call )

inline

Definition at line 4225 of file AdjointGenerator.h.

References TypeResults::addingType(), TypeResults::analyzer, FnTypeInfo::Arguments, GradientUtils::cacheForReverse(), CONSTANT, AugmentedReturn::constant_args, EnzymeLogic::CreateAugmentedPrimal(), CreateDealloc(), EnzymeLogic::CreatePrimalAndGradient(), DifferentialReturn, DUP_ARG, DUP_NONEED, GradientUtils::erase(), eraseIfUnused(), AugmentedReturn::fn, GradientUtils::getDiffeType(), getFast(), getFuncNameFromCall(), GradientUtils::getNewFromOriginal(), getPointerType(), getReverseBuilder(), getUnqual(), GradientUtils::getWidth(), TypeTree::insert(), insert_or_assign2(), Integer, TypeAnalyzer::interprocedural, IntToFloatTy(), CacheUtility::inversionAllocs, GradientUtils::invertPointerM(), isMemFreeLibMFunction(), TypeResults::knownIntegralValues(), FnTypeInfo::KnownValues, LegalFullUnwrap, GradientUtils::Logic, lookup(), CacheUtility::newFunc, GradientUtils::oldFunc, OUT_DIFF, Pointer, TypeResults::query(), Return, FnTypeInfo::Return, AugmentedReturn::returns, ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, GradientUtils::runtimeActivity, shouldFree(), GradientUtils::strongZero, AugmentedReturn::subaugmentations, Tape, AugmentedReturn::tapeIndices, AugmentedReturn::tapeIndiciesToFree, AugmentedReturn::tapeType, GradientUtils::unwrapM(), GradientUtils::unwrappedLoads, and whatType().

Referenced by visitCallInst().

visitPHINode()

void AdjointGenerator::visitPHINode ( llvm::PHINode & phi )

inline

Definition at line 1344 of file AdjointGenerator.h.

References eraseIfUnused(), ForwardMode, ForwardModeError, forwardModeInvertedPointerFallback(), ForwardModeSplit, ReverseModeCombined, ReverseModeGradient, and ReverseModePrimal.

visitSelectInst()

void AdjointGenerator::visitSelectInst ( llvm::SelectInst & SI )

inline

Definition at line 1474 of file AdjointGenerator.h.

References createSelectInstAdjoint(), eraseIfUnused(), ForwardMode, ForwardModeError, forwardModeInvertedPointerFallback(), ForwardModeSplit, GradientUtils::isConstantInstruction(), ReverseModeCombined, ReverseModeGradient, and ReverseModePrimal.

visitShuffleVectorInst()

void AdjointGenerator::visitShuffleVectorInst ( llvm::ShuffleVectorInst & SVI )

inline

Definition at line 1755 of file AdjointGenerator.h.

References TypeResults::addingType(), diffe(), eraseIfUnused(), GradientUtils::extractMeta(), ForwardMode, ForwardModeError, forwardModeInvertedPointerFallback(), ForwardModeSplit, getReverseBuilder(), GradientUtils::getShadowType(), GradientUtils::getWidth(), GradientUtils::isConstantInstruction(), GradientUtils::isConstantValue(), CacheUtility::newFunc, ReverseModeCombined, ReverseModeGradient, ReverseModePrimal, and setDiffe().

visitStoreInst()

void AdjointGenerator::visitStoreInst ( llvm::StoreInst & SI )

inline

Definition at line 878 of file AdjointGenerator.h.

References eraseIfUnused(), GradientUtils::rematerializableAllocations, ReverseModeGradient, and visitCommonStore().

---

The documentation for this class was generated from the following files:

• AdjointGenerator.h
• CallDerivatives.cpp