Utils.h File Reference

#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/ValueMap.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
#include <map>
#include <set>
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "TypeAnalysis/ConcreteType.h"
#include "llvm/IR/CFG.h"
#include <deque>
#include <functional>
#include "llvm/Analysis/LoopInfo.h"

Include dependency graph for Utils.h:

Go to the source code of this file.

Classes
class	EnzymeWarning
class	EnzymeFailure
struct	BlasInfo
class	AssertingReplacingVH
struct	CountTrackedPointers

Namespaces
namespace	llvm

Enumerations
enum class	ErrorType {   NoDerivative = 0 , NoShadow = 1 , IllegalTypeAnalysis = 2 , NoType = 3 ,   IllegalFirstPointer = 4 , InternalError = 5 , TypeDepthExceeded = 6 , MixedActivityError = 7 ,   IllegalReplaceFicticiousPHIs = 8 , GetIndexError = 9 , NoTruncate = 10 , GCRewrite = 11 ,   NaNError = 12 }
enum class	ReturnType {   ArgsWithReturn , ArgsWithTwoReturns , Args , TapeAndReturn ,   TapeAndTwoReturns , Tape , TwoReturns , Return ,   Void }
	Potential return type of generated functions. More...
enum class	DIFFE_TYPE { OUT_DIFF = 0 , DUP_ARG = 1 , CONSTANT = 2 , DUP_NONEED = 3 }
	Potential differentiable argument classifications. More...
enum class	BATCH_TYPE { SCALAR = 0 , VECTOR = 1 }
enum class	DerivativeMode {   ForwardMode = 0 , ReverseModePrimal = 1 , ReverseModeGradient = 2 , ReverseModeCombined = 3 ,   ForwardModeSplit = 4 , ForwardModeError = 5 }
enum class	ProbProgMode { Likelihood = 0 , Trace = 1 , Condition = 2 }
enum class	ValueType { None = 0 , Primal = 1 , Shadow = 2 , Both = Primal | Shadow }
	Classification of value as an original program variable, a derivative variable, neither, or both. More...
enum class	MPI_CallType { ISEND = 1 , IRECV = 2 }
enum class	MPI_Elem {   Buf = 0 , Count = 1 , DataType = 2 , Src = 3 ,   Tag = 4 , Comm = 5 , Call = 6 , Old = 7 }
enum	AddressSpace {   Generic = 0 , Tracked = 10 , Derived = 11 , CalleeRooted = 12 ,   Loaded = 13 , FirstSpecial = Tracked , LastSpecial = Loaded }
enum class	SRetRootMovement {   SRetPointerToRootPointer = 0 , SRetValueToRootPointer = 1 , RootPointerToSRetValue = 2 , RootPointerToSRetPointer = 3 ,   NullifySRetValue = 4 }

Functions
llvm::SmallVector< llvm::Instruction *, 2 >	PostCacheStore (llvm::StoreInst *SI, llvm::IRBuilder<> &B)
llvm::Value *	CreateAllocation (llvm::IRBuilder<> &B, llvm::Type *T, llvm::Value *Count, const llvm::Twine &Name="", llvm::CallInst **caller=nullptr, llvm::Instruction **ZeroMem=nullptr, bool isDefault=false)
llvm::CallInst *	CreateDealloc (llvm::IRBuilder<> &B, llvm::Value *ToFree)
void	ZeroMemory (llvm::IRBuilder<> &Builder, llvm::Type *T, llvm::Value *obj, bool isTape)
llvm::Value *	CreateReAllocation (llvm::IRBuilder<> &B, llvm::Value *prev, llvm::Type *T, llvm::Value *OuterCount, llvm::Value *InnerCount, const llvm::Twine &Name="", llvm::CallInst **caller=nullptr, bool ZeroMem=false)
llvm::PointerType *	getDefaultAnonymousTapeType (llvm::LLVMContext &C)
template<typename... Args>
void	EmitWarning (llvm::StringRef RemarkName, const llvm::DiagnosticLocation &Loc, const llvm::BasicBlock *BB, const Args &...args)
template<typename... Args>
void	EmitWarning (llvm::StringRef RemarkName, const llvm::Instruction &I, const Args &...args)
template<typename... Args>
void	EmitWarningAlways (llvm::StringRef RemarkName, const llvm::Function &F, const Args &...args)
template<typename... Args>
void	EmitWarning (llvm::StringRef RemarkName, const llvm::Function &F, const Args &...args)
llvm::Function *	getFirstFunctionDefinition (llvm::Module &M)
template<typename... Args>
void	EmitFailure (llvm::StringRef RemarkName, const llvm::DiagnosticLocation &Loc, const llvm::Instruction *CodeRegion, Args &...args)
template<typename... Args>
void	EmitFailure (llvm::StringRef RemarkName, const llvm::DiagnosticLocation &Loc, const llvm::Function *CodeRegion, Args &...args)
template<typename... Args>
void	EmitFailure (llvm::StringRef RemarkName, llvm::Module &M, Args &...args)
static llvm::Function *	isCalledFunction (llvm::Value *val)
llvm::Value *	EmitNoDerivativeError (const std::string &message, llvm::Instruction &inst, GradientUtils *gutils, llvm::IRBuilder<> &B, llvm::Value *condition=nullptr)
bool	EmitNoDerivativeError (const std::string &message, llvm::Value *todiff, RequestContext &ctx)
void	EmitNoTypeError (const std::string &, llvm::Instruction &inst, GradientUtils *gutils, llvm::IRBuilder<> &B)
llvm::FastMathFlags	getFast ()
	Get LLVM fast math flags.
template<typename T >
static T	max (T a, T b)
	Pick the maximum value.
template<typename T >
static T	min (T a, T b)
	Pick the maximum value.
template<typename T >
static std::string	to_string (const std::set< T > &us)
	Output a set as a string.
template<typename T , typename N >
static void	dumpMap (const llvm::ValueMap< T, N > &o, llvm::function_ref< bool(const llvm::Value *)> shouldPrint=[](T) { return true;})
	Print a map, optionally with a shouldPrint function to decide to print a given value.
template<typename T >
static void	dumpSet (const llvm::SmallPtrSetImpl< T * > &o)
	Print a set.
template<typename T >
static void	dumpSet (const llvm::SetVector< T * > &o)
static llvm::Instruction *	getNextNonDebugInstructionOrNull (llvm::Instruction *Z)
	Get the next non-debug instruction, if one exists.
static llvm::Instruction *	getNextNonDebugInstruction (llvm::Instruction *Z)
	Get the next non-debug instruction, erring if none exists.
static llvm::MDNode *	hasMetadata (const llvm::GlobalObject *O, llvm::StringRef kind)
	Check if a global has metadata.
static llvm::MDNode *	hasMetadata (const llvm::Instruction *O, llvm::StringRef kind)
	Check if an instruction has metadata.
static llvm::MDNode *	hasMetadata (const llvm::Instruction *O, unsigned kind)
static std::string	to_string (ValueType mode)
static llvm::raw_ostream &	operator<< (llvm::raw_ostream &os, ValueType mode)
static std::string	to_string (DerivativeMode mode)
static llvm::raw_ostream &	operator<< (llvm::raw_ostream &os, DerivativeMode mode)
static std::string	to_string (DIFFE_TYPE t)
	Convert DIFFE_TYPE to a string.
static llvm::raw_ostream &	operator<< (llvm::raw_ostream &os, DIFFE_TYPE mode)
static std::string	to_string (ReturnType t)
	Convert ReturnType to a string.
static llvm::raw_ostream &	operator<< (llvm::raw_ostream &os, ReturnType mode)
static DIFFE_TYPE	whatType (llvm::Type *arg, DerivativeMode mode, bool integersAreConstant, std::set< llvm::Type * > &seen)
	Attempt to automatically detect the differentiable classification based off of a given type.
llvm::Value *	get1ULP (llvm::IRBuilder<> &builder, llvm::Value *res)
static DIFFE_TYPE	whatType (llvm::Type *arg, DerivativeMode mode)
static bool	isReturned (llvm::Instruction *inst)
	Check whether this instruction is returned.
static llvm::Type *	FloatToIntTy (llvm::Type *T)
	Convert a floating point type to an integer type of the same size.
static llvm::Type *	IntToFloatTy (llvm::Type *T)
	Convert a integer type to a floating point type of the same size.
static bool	isDebugFunction (llvm::Function *called)
static bool	startsWith (llvm::StringRef string, llvm::StringRef prefix)
static bool	endsWith (llvm::StringRef string, llvm::StringRef suffix)
static bool	isCertainPrint (const llvm::StringRef name)
llvm::Optional< BlasInfo >	extractBLAS (llvm::StringRef in)
std::vector< std::tuple< llvm::Type *, size_t, size_t > >	parseTrueType (const llvm::MDNode *, DerivativeMode, bool const_src)
llvm::Function *	getOrInsertDifferentialFloatMemcpy (llvm::Module &M, llvm::Type *T, unsigned dstalign, unsigned srcalign, unsigned dstaddr, unsigned srcaddr, unsigned bitwidth)
	Create function for type that performs the derivative memcpy on floating point memory.
void	callMemcpyStridedBlas (llvm::IRBuilder<> &B, llvm::Module &M, BlasInfo blas, llvm::ArrayRef< llvm::Value * > args, llvm::Type *cublas_retty, llvm::ArrayRef< llvm::OperandBundleDef > bundles)
	Create function for type that performs memcpy with a stride using blas copy.
void	callMemcpyStridedLapack (llvm::IRBuilder<> &B, llvm::Module &M, BlasInfo blas, llvm::ArrayRef< llvm::Value * > args, llvm::ArrayRef< llvm::OperandBundleDef > bundles)
	Create function for type that performs memcpy using lapack copy.
void	callSPMVDiagUpdate (llvm::IRBuilder<> &B, llvm::Module &M, BlasInfo blas, llvm::IntegerType *IT, llvm::Type *BlasCT, llvm::Type *BlasFPT, llvm::Type *BlasPT, llvm::Type *BlasIT, llvm::Type *fpTy, llvm::ArrayRef< llvm::Value * > args, const llvm::ArrayRef< llvm::OperandBundleDef > bundles, bool byRef, bool julia_decl)
llvm::CallInst *	getorInsertInnerProd (llvm::IRBuilder<> &B, llvm::Module &M, BlasInfo blas, llvm::IntegerType *IT, llvm::Type *BlasPT, llvm::Type *BlasIT, llvm::Type *fpTy, llvm::ArrayRef< llvm::Value * > args, const llvm::ArrayRef< llvm::OperandBundleDef > bundles, bool byRef, bool cublas, bool julia_decl)
llvm::Function *	getOrInsertMemcpyStrided (llvm::Module &M, llvm::Type *elementType, llvm::PointerType *T, llvm::Type *IT, unsigned dstalign, unsigned srcalign)
	Create function for type that performs memcpy with a stride.
llvm::Function *	getOrInsertMemcpyMat (llvm::Module &M, llvm::Type *elementType, llvm::PointerType *PT, llvm::IntegerType *IT, unsigned dstalign, unsigned srcalign)
	Turned out to be a faster alternatives to lapacks lacpy function.
llvm::Function *	getOrInsertDifferentialFloatMemcpyMat (llvm::Module &M, llvm::Type *elementType, llvm::PointerType *PT, llvm::IntegerType *IT, llvm::IntegerType *CT, unsigned dstalign, unsigned srcalign, bool zeroSrc)
llvm::Function *	getOrInsertDifferentialFloatMemmove (llvm::Module &M, llvm::Type *T, unsigned dstalign, unsigned srcalign, unsigned dstaddr, unsigned srcaddr, unsigned bitwidth)
	Create function for type that performs the derivative memmove on floating point memory.
llvm::Function *	getOrInsertCheckedFree (llvm::Module &M, llvm::CallInst *call, llvm::Type *Type, unsigned width)
llvm::Function *	getOrInsertDifferentialMPI_Wait (llvm::Module &M, llvm::ArrayRef< llvm::Type * > T, llvm::Type *reqType, llvm::StringRef caller)
	Create function for type that performs the derivative MPI_Wait.
llvm::Value *	nextPowerOfTwo (llvm::IRBuilder<> &B, llvm::Value *V)
	Create function to computer nearest power of two.
template<typename K , typename V >
static std::map< K, V >::iterator	insert_or_assign (std::map< K, V > &map, K &key, V &&val)
	Insert into a map.
template<typename K , typename V >
static std::map< K, V >::iterator	insert_or_assign2 (std::map< K, V > &map, K key, V val)
	Insert into a map.
template<typename K , typename V >
static V *	findInMap (std::map< K, V > &map, K key)
static void	allFollowersOf (llvm::Instruction *inst, llvm::function_ref< bool(llvm::Instruction *)> f)
	Call the function f for all instructions that happen after inst If the function returns true, the iteration will early exit.
static void	allPredecessorsOf (llvm::Instruction *inst, llvm::function_ref< bool(llvm::Instruction *)> f)
	Call the function f for all instructions that happen before inst If the function returns true, the iteration will early exit.
static void	allDomPredecessorsOf (llvm::Instruction *inst, llvm::DominatorTree &DT, llvm::function_ref< bool(llvm::Instruction *)> f)
	Call the function f for all instructions that happen before inst If the function returns true, the iteration will early exit.
static void	allUnsyncdPredecessorsOf (llvm::Instruction *inst, llvm::function_ref< bool(llvm::Instruction *)> f, llvm::function_ref< void()> preEntry)
	Call the function f for all instructions that happen before inst If the function returns true, the iteration will early exit.
static llvm::Loop *	getAncestor (llvm::Loop *R1, llvm::Loop *R2)
void	mayExecuteAfter (llvm::SmallVectorImpl< llvm::Instruction * > &results, llvm::Instruction *inst, const llvm::SmallPtrSetImpl< llvm::Instruction * > &stores, const llvm::Loop *region)
bool	writesToMemoryReadBy (const TypeResults *TR, llvm::AAResults &AA, llvm::TargetLibraryInfo &TLI, llvm::Instruction *maybeReader, llvm::Instruction *maybeWriter)
	Return whether maybeReader can read from memory written to by maybeWriter.
bool	overwritesToMemoryReadBy (const TypeResults *TR, llvm::AAResults &AA, llvm::TargetLibraryInfo &TLI, llvm::ScalarEvolution &SE, llvm::LoopInfo &LI, llvm::DominatorTree &DT, llvm::Instruction *maybeReader, llvm::Instruction *maybeWriter, llvm::Loop *scope=nullptr)
static void	allInstructionsBetween (llvm::LoopInfo &LI, llvm::Instruction *inst1, llvm::Instruction *inst2, llvm::function_ref< bool(llvm::Instruction *)> f)
	Call the function f for all instructions that happen between inst1 and inst2 If the function returns true, the iteration will early exit.
static llvm::PointerType *	getPointerType (llvm::Type *T, unsigned AddressSpace=0)
static llvm::PointerType *	getInt8PtrTy (llvm::LLVMContext &Context, unsigned AddressSpace=0)
static llvm::PointerType *	getUnqual (llvm::Type *T)
static llvm::StructType *	getMPIHelper (llvm::LLVMContext &Context)
template<MPI_Elem E, bool Pointer = true>
static llvm::Value *	getMPIMemberPtr (llvm::IRBuilder<> &B, llvm::Value *V, llvm::Type *T)
llvm::Value *	getOrInsertOpFloatSum (llvm::Module &M, llvm::Type *OpPtr, llvm::Type *OpType, ConcreteType CT, llvm::Type *intType, llvm::IRBuilder<> &B2)
template<typename T >
static llvm::Function *	getFunctionFromCall (T *op)
static llvm::StringRef	getFuncName (llvm::Function *called)
static llvm::StringRef	getFuncNameFromCall (const llvm::CallBase *op)
static bool	hasNoCache (llvm::Value *op)
static llvm::Optional< size_t >	getAllocationIndexFromCall (const llvm::CallBase *op)
template<typename T >
static llvm::Function *	getDeallocatorFnFromCall (T *op)
template<typename T >
static std::vector< ssize_t >	getDeallocationIndicesFromCall (T *op)
llvm::Function *	getOrInsertDifferentialWaitallSave (llvm::Module &M, llvm::ArrayRef< llvm::Type * > T, llvm::PointerType *reqType)
void	ErrorIfRuntimeInactive (llvm::IRBuilder<> &B, llvm::Value *primal, llvm::Value *shadow, const char *Message, llvm::DebugLoc &&loc, llvm::Instruction *orig)
llvm::Function *	GetFunctionFromValue (llvm::Value *fn)
llvm::Value *	simplifyLoad (llvm::Value *LI, size_t valSz=0, size_t preOffset=0)
static bool	shouldDisableNoWrite (const llvm::CallInst *CI)
static bool	isIntelSubscriptIntrinsic (const llvm::IntrinsicInst &II)
static bool	isIntelSubscriptIntrinsic (const llvm::Value *val)
static bool	isPointerArithmeticInst (const llvm::Value *V, bool includephi=true, bool includebin=true)
static llvm::Value *	getBaseObject (llvm::Value *V, bool offsetAllowed=true)
static const llvm::Value *	getBaseObject (const llvm::Value *V)
static llvm::SetVector< llvm::Value * >	getBaseObjects (llvm::Value *V, bool offsetAllowed=true)
static bool	isReadOnly (const llvm::Function *F, ssize_t arg=-1)
static bool	isReadOnly (const llvm::CallBase *call, ssize_t arg=-1)
static bool	isLocalReadOnlyOrThrow (const llvm::Function *F)
static bool	isLocalReadOnlyOrThrow (const llvm::CallBase *call)
static bool	isReadOnlyOrThrow (const llvm::Function *F)
static bool	isReadOnlyOrThrow (const llvm::CallBase *call)
static bool	isWriteOnly (const llvm::Function *F, ssize_t arg=-1)
static bool	isWriteOnly (const llvm::CallBase *call, ssize_t arg=-1)
static bool	isReadNone (const llvm::CallBase *call, ssize_t arg=-1)
static bool	isReadNone (const llvm::Function *F, ssize_t arg=-1)
static bool	isNoCapture (const llvm::CallBase *call, size_t idx)
static bool	isNoAlias (const llvm::CallBase *call)
static bool	isNoAlias (const llvm::Value *val)
static bool	isNoEscapingAllocation (const llvm::Function *F)
static bool	isNoEscapingAllocation (const llvm::CallBase *call)
bool	attributeKnownFunctions (llvm::Function &F)
llvm::Constant *	getUndefinedValueForType (llvm::Module &M, llvm::Type *T, bool forceZero=false)
llvm::Value *	SanitizeDerivatives (llvm::Value *val, llvm::Value *toset, llvm::IRBuilder<> &BuilderM, llvm::Value *mask=nullptr)
static llvm::Value *	CreateSelect (llvm::IRBuilder<> &Builder2, llvm::Value *cmp, llvm::Value *tval, llvm::Value *fval, const llvm::Twine &Name="")
static llvm::Value *	checkedMul (bool strongZero, llvm::IRBuilder<> &Builder2, llvm::Value *idiff, llvm::Value *pres, const llvm::Twine &Name="")
static llvm::Value *	checkedDiv (bool strongZero, llvm::IRBuilder<> &Builder2, llvm::Value *idiff, llvm::Value *pres, const llvm::Twine &Name="")
static bool	containsOnlyAtMostTopBit (const llvm::Value *V, llvm::Type *FT, const llvm::DataLayout &dl, llvm::Type **vFT=nullptr)
void	addValueToCache (llvm::Value *arg, bool cache_arg, llvm::Type *ty, llvm::SmallVectorImpl< llvm::Value * > &cacheValues, llvm::IRBuilder<> &BuilderZ, const llvm::Twine &name="")
llvm::Value *	load_if_ref (llvm::IRBuilder<> &B, llvm::Type *intType, llvm::Value *V, bool byRef)
void	copy_lower_to_upper (llvm::IRBuilder<> &B, llvm::Type *fpType, BlasInfo blas, bool byRef, llvm::Value *layout, llvm::Value *uplo, llvm::Value *A, llvm::Value *lda, llvm::Value *N)
llvm::Value *	to_blas_callconv (llvm::IRBuilder<> &B, llvm::Value *V, bool byRef, bool cublas, llvm::IntegerType *julia_decl, llvm::IRBuilder<> &entryBuilder, llvm::Twine const &="")
llvm::Value *	to_blas_fp_callconv (llvm::IRBuilder<> &B, llvm::Value *V, bool byRef, llvm::Type *julia_decl, llvm::IRBuilder<> &entryBuilder, llvm::Twine const &="")
llvm::Value *	get_cached_mat_width (llvm::IRBuilder<> &B, llvm::ArrayRef< llvm::Value * > trans, llvm::Value *arg_ld, llvm::Value *dim_1, llvm::Value *dim_2, bool cacheMat, bool byRef, bool cublas)
template<typename T >
static void	append (llvm::SmallVectorImpl< T > &vec)
template<typename T , typename... T2>
static void	append (llvm::SmallVectorImpl< T > &vec, llvm::ArrayRef< T > vals, T2 &&...ts)
template<typename... T>
static llvm::SmallVector< llvm::Value *, 1 >	concat_values (T &&...t)
llvm::Value *	is_normal (llvm::IRBuilder<> &B, llvm::Value *trans, bool byRef, bool cublas)
llvm::Value *	is_left (llvm::IRBuilder<> &B, llvm::Value *side, bool byRef, bool cublas)
llvm::Value *	is_lower (llvm::IRBuilder<> &B, llvm::Value *uplo, bool byRef, bool cublas)
llvm::Value *	is_nonunit (llvm::IRBuilder<> &B, llvm::Value *uplo, bool byRef, bool cublas)
llvm::Value *	lookup_with_layout (llvm::IRBuilder<> &B, llvm::Type *fpType, llvm::Value *layout, llvm::Value *base, llvm::Value *lda, llvm::Value *row, llvm::Value *col)
llvm::Value *	transpose (std::string floatType, llvm::IRBuilder<> &B, llvm::Value *V, bool cublas)
llvm::Value *	transpose (std::string floatType, llvm::IRBuilder<> &B, llvm::Value *V, bool byRef, bool cublas, llvm::IntegerType *IT, llvm::IRBuilder<> &entryBuilder, const llvm::Twine &name)
llvm::SmallVector< llvm::Value *, 1 >	get_blas_row (llvm::IRBuilder<> &B, llvm::ArrayRef< llvm::Value * > trans, llvm::ArrayRef< llvm::Value * > row, llvm::ArrayRef< llvm::Value * > col, bool byRef, bool cublas)
llvm::SmallVector< llvm::Value *, 1 >	get_blas_row (llvm::IRBuilder<> &B, llvm::ArrayRef< llvm::Value * > trans, bool byRef, bool cublas)
static llvm::Function *	getIntrinsicDeclaration (llvm::Module *M, llvm::Intrinsic::ID id, llvm::ArrayRef< llvm::Type * > Tys={})
static llvm::Instruction *	getFirstNonPHIOrDbg (llvm::BasicBlock *B)
static llvm::Instruction *	getFirstNonPHIOrDbgOrLifetime (llvm::BasicBlock *B)
static void	addCallSiteNoCapture (llvm::CallBase *call, size_t idx)
static void	addFunctionNoCapture (llvm::Function *call, size_t idx)
static llvm::AttributeList	addFunctionNoCapture (llvm::LLVMContext &ctx, llvm::AttributeList list, size_t idx)
static llvm::Type *	getSubType (llvm::Type *T)
template<typename Arg1 , typename... Args>
static llvm::Type *	getSubType (llvm::Type *T, Arg1 i, Args... args)
static bool	isSpecialPtr (llvm::Type *Ty)
bool	collectOffset (llvm::GEPOperator *gep, const llvm::DataLayout &DL, unsigned BitWidth, llvm::MapVector< llvm::Value *, llvm::APInt > &VariableOffsets, llvm::APInt &ConstantOffset)
llvm::CallInst *	createIntrinsicCall (llvm::IRBuilderBase &B, llvm::Intrinsic::ID ID, llvm::Type *RetTy, llvm::ArrayRef< llvm::Value * > Args, llvm::Instruction *FMFSource=nullptr, const llvm::Twine &Name="")
bool	isNVLoad (const llvm::Value *V)
bool	notCapturedBefore (llvm::Value *V, llvm::Instruction *inst, size_t checkLoadCaptured)
	Check if value if b captured after definition before executing inst.
bool	notCaptured (llvm::Value *V)
	Check if value if b captured.
llvm::Optional< bool >	arePointersGuaranteedNoAlias (llvm::TargetLibraryInfo &TLI, llvm::AAResults &AA, llvm::LoopInfo &LI, llvm::Value *op0, llvm::Value *op1, bool offsetAllowed=false)
static std::tuple< llvm::StringRef, llvm::StringRef, llvm::StringRef >	tripleSplitDollar (llvm::StringRef caller)
static std::string	getRenamedPerCallingConv (llvm::StringRef caller, llvm::StringRef callee)
static std::string	convertSRetTypeToString (llvm::Type *T)
static llvm::Type *	convertSRetTypeFromString (llvm::StringRef str, llvm::LLVMContext *C=nullptr)
static size_t	convertRRootCountFromString (llvm::StringRef str)
static bool	hasSRetRRootsOrUnionSRet (llvm::CallBase *CB)
llvm::Value *	moveSRetToFromRoots (llvm::IRBuilder<> &B, llvm::Type *jltype, llvm::Value *sret, llvm::Type *root_ty, llvm::Value *rootRet, size_t rootOffset, SRetRootMovement direction)
void	copyNonJLValueInto (llvm::IRBuilder<> &B, llvm::Type *curType, llvm::Type *dstType, llvm::Value *dst, llvm::ArrayRef< unsigned > dstPrefix, llvm::Type *srcType, llvm::Value *src, llvm::ArrayRef< unsigned > srcPrefix, bool shouldZero)
static bool	anyJuliaObjects (llvm::Type *T)
llvm::SmallVector< llvm::Value *, 1 >	getJuliaObjects (llvm::Value *v, llvm::IRBuilder<> &B)
llvm::SmallVector< std::tuple< llvm::Instruction *, llvm::Value *, size_t >, 1 >	findAllUsersOf (llvm::Value *AI)
static bool	hasTerminator (llvm::BasicBlock *BB)

Variables
llvm::cl::opt< bool >	EnzymePrintPerf
	Print additional debug info relevant to performance.
llvm::cl::opt< bool >	EnzymeNonPower2Cache
llvm::cl::opt< bool >	EnzymeBlasCopy
llvm::cl::opt< bool >	EnzymeLapackCopy
llvm::cl::opt< bool >	EnzymeJuliaAddrLoad
LLVMValueRef(*	CustomErrorHandler )(const char *, LLVMValueRef, ErrorType, const void *, LLVMValueRef, LLVMBuilderRef)
llvm::StringMap< std::function< llvm::Value *(llvm::IRBuilder<> &, llvm::CallInst *, llvm::ArrayRef< llvm::Value * >, GradientUtils *)> >	shadowHandlers
static llvm::Attribute::AttrKind	PrimalParamAttrsToPreserve []
static llvm::Attribute::AttrKind	ShadowParamAttrsToPreserve []

Enumeration Type Documentation

AddressSpace

enum AddressSpace

Enumerator
Generic
Tracked
Derived
CalleeRooted
Loaded
FirstSpecial
LastSpecial

Definition at line 2339 of file Utils.h.

BATCH_TYPE

enum class BATCH_TYPE

strong

Enumerator
SCALAR
VECTOR

Definition at line 385 of file Utils.h.

DerivativeMode

enum class DerivativeMode

strong

Enumerator
ForwardMode
ReverseModePrimal
ReverseModeGradient
ReverseModeCombined
ForwardModeSplit
ForwardModeError

Definition at line 390 of file Utils.h.

DIFFE_TYPE

enum class DIFFE_TYPE

strong

Potential differentiable argument classifications.

Enumerator
OUT_DIFF
DUP_ARG
CONSTANT
DUP_NONEED

Definition at line 374 of file Utils.h.

ErrorType

enum class ErrorType

strong

Enumerator
NoDerivative
NoShadow
IllegalTypeAnalysis
NoType
IllegalFirstPointer
InternalError
TypeDepthExceeded
MixedActivityError
IllegalReplaceFicticiousPHIs
GetIndexError
NoTruncate
GCRewrite
NaNError

Definition at line 77 of file Utils.h.

MPI_CallType

enum class MPI_CallType

strong

Enumerator
ISEND
IRECV

Definition at line 1149 of file Utils.h.

MPI_Elem

enum class MPI_Elem

strong

Enumerator
Buf
Count
DataType
Src
Tag
Comm
Call
Old

Definition at line 1154 of file Utils.h.

ProbProgMode

enum class ProbProgMode

strong

Enumerator
Likelihood
Trace
Condition

Definition at line 399 of file Utils.h.

ReturnType

enum class ReturnType

strong

Potential return type of generated functions.

Enumerator
ArgsWithReturn	Return is a struct of all args and the original return.
ArgsWithTwoReturns	Return is a struct of all args and two of the original return.
Args	Return is a struct of all args.
TapeAndReturn	Return is a tape type and the original return.
TapeAndTwoReturns	Return is a tape type and the two of the original return.
Tape	Return is a tape type.
TwoReturns
Return
Void

Definition at line 355 of file Utils.h.

SRetRootMovement

enum class SRetRootMovement

strong

Enumerator
SRetPointerToRootPointer
SRetValueToRootPointer
RootPointerToSRetValue
RootPointerToSRetPointer
NullifySRetValue

Definition at line 2505 of file Utils.h.

ValueType

enum class ValueType

strong

Classification of value as an original program variable, a derivative variable, neither, or both.

This type is used both in differential use analysis and to describe argument bundles.

Enumerator
None
Primal
Shadow
Both

Definition at line 409 of file Utils.h.

Function Documentation

addCallSiteNoCapture()

static void addCallSiteNoCapture ( llvm::CallBase * call, size_t idx )

inlinestatic

Definition at line 2289 of file Utils.h.

Referenced by TraceUtils::GetChoice(), TraceUtils::GetTrace(), TraceUtils::HasCall(), TraceUtils::HasChoice(), TraceUtils::InsertArgument(), TraceUtils::InsertArgumentGradient(), TraceUtils::InsertCall(), TraceUtils::InsertChoice(), TraceUtils::InsertChoiceGradient(), and SimplifyMPIQueries().

addFunctionNoCapture() [1/2]

static void addFunctionNoCapture ( llvm::Function * call, size_t idx )

inlinestatic

Definition at line 2299 of file Utils.h.

Referenced by attributeKnownFunctions(), callSPMVDiagUpdate(), copy_lower_to_upper(), DetectPointerArgOfFn(), ErrorIfRuntimeInactive(), getOrInsertCheckedFree(), getOrInsertDifferentialFloatMemcpy(), getorInsertInnerProd(), getOrInsertMemcpyMat(), getOrInsertMemcpyStrided(), getOrInsertOpFloatSum(), AdjointGenerator::MPI_COMM_RANK(), AdjointGenerator::MPI_COMM_SIZE(), AdjointGenerator::MPI_TYPE_SIZE(), and OldAllocationSize().

addFunctionNoCapture() [2/2]

static llvm::AttributeList addFunctionNoCapture ( llvm::LLVMContext & ctx, llvm::AttributeList list, size_t idx )

inlinestaticnodiscard

Definition at line 2310 of file Utils.h.

addValueToCache()

void addValueToCache	(	llvm::Value *	arg,
		bool	cache_arg,
		llvm::Type *	ty,
		llvm::SmallVectorImpl< llvm::Value * > &	cacheValues,
		llvm::IRBuilder<> &	BuilderZ,
		const llvm::Twine &	name = "" )

allDomPredecessorsOf()

static void allDomPredecessorsOf ( llvm::Instruction * inst, llvm::DominatorTree & DT, llvm::function_ref< bool(llvm::Instruction *)> f )

inlinestatic

Call the function f for all instructions that happen before inst If the function returns true, the iteration will early exit.

Definition at line 941 of file Utils.h.

allFollowersOf()

static void allFollowersOf ( llvm::Instruction * inst, llvm::function_ref< bool(llvm::Instruction *)> f )

inlinestatic

Call the function f for all instructions that happen after inst If the function returns true, the iteration will early exit.

Definition at line 869 of file Utils.h.

allInstructionsBetween()

static void allInstructionsBetween ( llvm::LoopInfo & LI, llvm::Instruction * inst1, llvm::Instruction * inst2, llvm::function_ref< bool(llvm::Instruction *)> f )

inlinestatic

Call the function f for all instructions that happen between inst1 and inst2 If the function returns true, the iteration will early exit.

Definition at line 1099 of file Utils.h.

References allPredecessorsOf().

Referenced by arePointersGuaranteedNoAlias().

allPredecessorsOf()

static void allPredecessorsOf ( llvm::Instruction * inst, llvm::function_ref< bool(llvm::Instruction *)> f )

inlinestatic

Call the function f for all instructions that happen before inst If the function returns true, the iteration will early exit.

Definition at line 904 of file Utils.h.

Referenced by allInstructionsBetween().

allUnsyncdPredecessorsOf()

static void allUnsyncdPredecessorsOf ( llvm::Instruction * inst, llvm::function_ref< bool(llvm::Instruction *)> f, llvm::function_ref< void()> preEntry )

inlinestatic

Call the function f for all instructions that happen before inst If the function returns true, the iteration will early exit.

Definition at line 980 of file Utils.h.

anyJuliaObjects()

static bool anyJuliaObjects ( llvm::Type * T )

static

Definition at line 2524 of file Utils.h.

References anyJuliaObjects(), and isSpecialPtr().

Referenced by anyJuliaObjects(), and getJuliaObjects().

append() [1/2]

template<typename T >

static void append ( llvm::SmallVectorImpl< T > & vec )

inlinestatic

Definition at line 2153 of file Utils.h.

Referenced by append(), and concat_values().

append() [2/2]

template<typename T , typename... T2>

static void append ( llvm::SmallVectorImpl< T > & vec, llvm::ArrayRef< T > vals, T2 &&... ts )

inlinestatic

Definition at line 2155 of file Utils.h.

References append().

arePointersGuaranteedNoAlias()

llvm::Optional< bool > arePointersGuaranteedNoAlias	(	llvm::TargetLibraryInfo &	TLI,
		llvm::AAResults &	AA,
		llvm::LoopInfo &	LI,
		llvm::Value *	op0,
		llvm::Value *	op1,
		bool	offsetAllowed = false )

attributeKnownFunctions()

bool attributeKnownFunctions

(

llvm::Function &

F

)

Definition at line 114 of file Utils.cpp.

References addFunctionNoCapture(), Constant, and getFuncName().

Referenced by callMemcpyStridedBlas(), callMemcpyStridedLapack(), copy_lower_to_upper(), EnzymeAttributeKnownFunctions(), getorInsertInnerProd(), and preserveNVVM().

callMemcpyStridedBlas()

void callMemcpyStridedBlas	(	llvm::IRBuilder<> &	B,
		llvm::Module &	M,
		BlasInfo	blas,
		llvm::ArrayRef< llvm::Value * >	args,
		llvm::Type *	cublas_retty,
		llvm::ArrayRef< llvm::OperandBundleDef >	bundles )

Create function for type that performs memcpy with a stride using blas copy.

Definition at line 1316 of file Utils.cpp.

References attributeKnownFunctions(), BlasInfo::floatType, GetFunctionFromValue(), BlasInfo::prefix, and BlasInfo::suffix.

callMemcpyStridedLapack()

void callMemcpyStridedLapack	(	llvm::IRBuilder<> &	B,
		llvm::Module &	M,
		BlasInfo	blas,
		llvm::ArrayRef< llvm::Value * >	args,
		llvm::ArrayRef< llvm::OperandBundleDef >	bundles )

Create function for type that performs memcpy using lapack copy.

Definition at line 1336 of file Utils.cpp.

References attributeKnownFunctions(), BlasInfo::floatType, GetFunctionFromValue(), BlasInfo::prefix, and BlasInfo::suffix.

callSPMVDiagUpdate()

void callSPMVDiagUpdate	(	llvm::IRBuilder<> &	B,
		llvm::Module &	M,
		BlasInfo	blas,
		llvm::IntegerType *	IT,
		llvm::Type *	BlasCT,
		llvm::Type *	BlasFPT,
		llvm::Type *	BlasPT,
		llvm::Type *	BlasIT,
		llvm::Type *	fpTy,
		llvm::ArrayRef< llvm::Value * >	args,
		const llvm::ArrayRef< llvm::OperandBundleDef >	bundles,
		bool	byRef,
		bool	julia_decl )

checkedDiv()

static llvm::Value * checkedDiv ( bool strongZero, llvm::IRBuilder<> & Builder2, llvm::Value * idiff, llvm::Value * pres, const llvm::Twine & Name = "" )

inlinestatic

Definition at line 2032 of file Utils.h.

checkedMul()

static llvm::Value * checkedMul ( bool strongZero, llvm::IRBuilder<> & Builder2, llvm::Value * idiff, llvm::Value * pres, const llvm::Twine & Name = "" )

inlinestatic

Definition at line 2018 of file Utils.h.

Referenced by AdjointGenerator::createBinaryOperatorAdjoint(), and AdjointGenerator::createBinaryOperatorDual().

collectOffset()

bool collectOffset	(	llvm::GEPOperator *	gep,
		const llvm::DataLayout &	DL,
		unsigned	BitWidth,
		llvm::MapVector< llvm::Value *, llvm::APInt > &	VariableOffsets,
		llvm::APInt &	ConstantOffset )

concat_values()

template<typename... T>

static llvm::SmallVector< llvm::Value *, 1 > concat_values ( T &&... t )

inlinestatic

Definition at line 2161 of file Utils.h.

References append().

containsOnlyAtMostTopBit()

static bool containsOnlyAtMostTopBit ( const llvm::Value * V, llvm::Type * FT, const llvm::DataLayout & dl, llvm::Type ** vFT = nullptr )

inlinestatic

Definition at line 2047 of file Utils.h.

References containsOnlyAtMostTopBit().

Referenced by containsOnlyAtMostTopBit(), AdjointGenerator::createBinaryOperatorAdjoint(), AdjointGenerator::createBinaryOperatorDual(), and TypeAnalyzer::visitBinaryOperation().

convertRRootCountFromString()

static size_t convertRRootCountFromString ( llvm::StringRef str )

inlinestatic

Definition at line 2481 of file Utils.h.

References str().

Referenced by EnzymeFixupJuliaCallingConvention(), and needsReReturning().

convertSRetTypeFromString()

static llvm::Type * convertSRetTypeFromString ( llvm::StringRef str, llvm::LLVMContext * C = nullptr )

inlinestatic

Definition at line 2433 of file Utils.h.

References str(), and Tracked.

Referenced by EnzymeFixupJuliaCallingConvention(), and needsReRooting().

convertSRetTypeToString()

static std::string convertSRetTypeToString ( llvm::Type * T )

inlinestatic

Definition at line 2428 of file Utils.h.

Referenced by PreProcessCache::CloneFunctionWithReturns(), EnzymeFixupJuliaCallingConvention(), and AdjointGenerator::recursivelyHandleSubfunction().

copy_lower_to_upper()

void copy_lower_to_upper	(	llvm::IRBuilder<> &	B,
		llvm::Type *	fpType,
		BlasInfo	blas,
		bool	byRef,
		llvm::Value *	layout,
		llvm::Value *	uplo,
		llvm::Value *	A,
		llvm::Value *	lda,
		llvm::Value *	N )

Definition at line 1184 of file Utils.cpp.

References addFunctionNoCapture(), attributeKnownFunctions(), CreateSelect(), BlasInfo::floatType, getFunctionFromCall(), GetFunctionFromValue(), lookup_with_layout(), BlasInfo::prefix, BlasInfo::suffix, and to_blas_callconv().

copyNonJLValueInto()

void copyNonJLValueInto	(	llvm::IRBuilder<> &	B,
		llvm::Type *	curType,
		llvm::Type *	dstType,
		llvm::Value *	dst,
		llvm::ArrayRef< unsigned >	dstPrefix,
		llvm::Type *	srcType,
		llvm::Value *	src,
		llvm::ArrayRef< unsigned >	srcPrefix,
		bool	shouldZero )

Definition at line 4837 of file Utils.cpp.

References constantInBoundsGEPHelper(), CountTrackedPointers::count, and getUndefinedValueForType().

Referenced by EnzymeFixupJuliaCallingConvention().

CreateAllocation()

llvm::Value * CreateAllocation	(	llvm::IRBuilder<> &	B,
		llvm::Type *	T,
		llvm::Value *	Count,
		const llvm::Twine &	Name = "",
		llvm::CallInst **	caller = nullptr,
		llvm::Instruction **	ZeroMem = nullptr,
		bool	isDefault = false )

CreateDealloc()

llvm::CallInst * CreateDealloc	(	llvm::IRBuilder<> &	B,
		llvm::Value *	ToFree )

Definition at line 742 of file Utils.cpp.

References CustomDeallocator, and getInt8PtrTy().

Referenced by DiffeGradientUtils::freeCache(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), and AdjointGenerator::visitOMPCall().

createIntrinsicCall()

llvm::CallInst * createIntrinsicCall	(	llvm::IRBuilderBase &	B,
		llvm::Intrinsic::ID	ID,
		llvm::Type *	RetTy,
		llvm::ArrayRef< llvm::Value * >	Args,
		llvm::Instruction *	FMFSource = nullptr,
		const llvm::Twine &	Name = "" )

Definition at line 4233 of file Utils.cpp.

References Args.

CreateReAllocation()

llvm::Value * CreateReAllocation	(	llvm::IRBuilder<> &	B,
		llvm::Value *	prev,
		llvm::Type *	T,
		llvm::Value *	OuterCount,
		llvm::Value *	InnerCount,
		const llvm::Twine &	Name = "",
		llvm::CallInst **	caller = nullptr,
		bool	ZeroMem = false )

Definition at line 590 of file Utils.cpp.

References getOrInsertExponentialAllocator().

Referenced by CacheUtility::createCacheForScope().

CreateSelect()

static llvm::Value * CreateSelect ( llvm::IRBuilder<> & Builder2, llvm::Value * cmp, llvm::Value * tval, llvm::Value * fval, const llvm::Twine & Name = "" )

inlinestatic

Definition at line 2005 of file Utils.h.

Referenced by copy_lower_to_upper(), AdjointGenerator::createBinaryOperatorAdjoint(), AdjointGenerator::createBinaryOperatorDual(), AdjointGenerator::createSelectInstAdjoint(), get_cached_mat_width(), AdjointGenerator::handleAdjointForIntrinsic(), AdjointGenerator::handleMPI(), lookup_with_layout(), AdjointGenerator::visitBinaryOperator(), and AdjointGenerator::visitLoadLike().

dumpMap()

template<typename T , typename N >

static void dumpMap ( const llvm::ValueMap< T, N > & o, llvm::function_ref< bool(const llvm::Value *)> shouldPrint = [](T) { return true; } )

inlinestatic

Print a map, optionally with a shouldPrint function to decide to print a given value.

Definition at line 286 of file Utils.h.

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

dumpSet() [1/2]

template<typename T >

static void dumpSet ( const llvm::SetVector< T * > & o )

inlinestatic

Definition at line 309 of file Utils.h.

dumpSet() [2/2]

template<typename T >

static void dumpSet ( const llvm::SmallPtrSetImpl< T * > & o )

inlinestatic

Print a set.

Definition at line 301 of file Utils.h.

EmitFailure() [1/3]

template<typename... Args>

void EmitFailure	(	llvm::StringRef	RemarkName,
		const llvm::DiagnosticLocation &	Loc,
		const llvm::Function *	CodeRegion,
		Args &...	args )

Definition at line 214 of file Utils.h.

References str().

EmitFailure() [2/3]

template<typename... Args>

void EmitFailure	(	llvm::StringRef	RemarkName,
		const llvm::DiagnosticLocation &	Loc,
		const llvm::Instruction *	CodeRegion,
		Args &...	args )

Definition at line 203 of file Utils.h.

References str().

Referenced by Constraints::allSolutions(), AdjointGenerator::DifferentiableMemCopyFloats(), EmitFailure(), EmitNoDerivativeError(), EmitNoDerivativeError(), EmitNoTypeError(), EnzymeFixupJuliaCallingConvention(), CacheUtility::erase(), TypeResults::firstPointer(), fixSparseIndices(), getSparseConditions(), OldAllocationSize(), AdjointGenerator::recursivelyHandleSubfunction(), RecursivelyReplaceAddressSpace(), replaceToDense(), ActivityAnalysisPrinterNewPM::run(), TypeAnalysisPrinterNewPM::run(), transpose(), TypeAnalyzer::visitBinaryOperation(), InstructionBatcher::visitBranchInst(), InstructionBatcher::visitInstruction(), TypeAnalyzer::visitIntrinsicInst(), TypeAnalyzer::visitMemTransferCommon(), TypeAnalyzer::visitPHINode(), and InstructionBatcher::visitSwitchInst().

EmitFailure() [3/3]

template<typename... Args>

void EmitFailure	(	llvm::StringRef	RemarkName,
		llvm::Module &	M,
		Args &...	args )

Definition at line 225 of file Utils.h.

References EmitFailure(), getFirstFunctionDefinition(), and str().

EmitNoDerivativeError() [1/2]

llvm::Value * EmitNoDerivativeError	(	const std::string &	message,
		llvm::Instruction &	inst,
		GradientUtils *	gutils,
		llvm::IRBuilder<> &	B,
		llvm::Value *	condition = nullptr )

Definition at line 4295 of file Utils.cpp.

References contains(), CustomErrorHandler, TypeResults::dump(), emit_backtrace(), EmitFailure(), EnzymeRuntimeError(), getString(), NoDerivative, str(), and GradientUtils::TR.

Referenced by AdjointGenerator::createBinaryOperatorAdjoint(), AdjointGenerator::createBinaryOperatorDual(), AdjointGenerator::handleAdjointForIntrinsic(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitAtomicRMWInst(), AdjointGenerator::visitCastInst(), AdjointGenerator::visitInstruction(), and AdjointGenerator::visitMemSetCommon().

EmitNoDerivativeError() [2/2]

bool EmitNoDerivativeError	(	const std::string &	message,
		llvm::Value *	todiff,
		RequestContext &	ctx )

EmitNoTypeError()

void EmitNoTypeError	(	const std::string &	message,
		llvm::Instruction &	inst,
		GradientUtils *	gutils,
		llvm::IRBuilder<> &	B )

Definition at line 4377 of file Utils.cpp.

References TypeResults::analyzer, CustomErrorHandler, TypeResults::dump(), emit_backtrace(), EmitFailure(), EnzymeRuntimeError(), getString(), NoType, str(), and GradientUtils::TR.

Referenced by AdjointGenerator::visitCastInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitExtractValueInst(), AdjointGenerator::visitInsertValueInst(), AdjointGenerator::visitLoadLike(), AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitMemTransferCommon().

EmitWarning() [1/3]

template<typename... Args>

void EmitWarning	(	llvm::StringRef	RemarkName,
		const llvm::DiagnosticLocation &	Loc,
		const llvm::BasicBlock *	BB,
		const Args &...	args )

Definition at line 133 of file Utils.h.

References EnzymePrintPerf, GradientUtils::getContext(), and str().

Referenced by DetectReadonlyOrThrowFn(), EmitWarning(), CacheUtility::getContext(), getSparseConditions(), CacheUtility::getSubLimits(), AdjointGenerator::handleAdjointForIntrinsic(), TypeTree::Only(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitCastInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitLoadLike(), AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitMemTransferCommon().

EmitWarning() [2/3]

template<typename... Args>

void EmitWarning	(	llvm::StringRef	RemarkName,
		const llvm::Function &	F,
		const Args &...	args )

Definition at line 177 of file Utils.h.

References EnzymePrintPerf, and str().

EmitWarning() [3/3]

template<typename... Args>

void EmitWarning	(	llvm::StringRef	RemarkName,
		const llvm::Instruction &	I,
		const Args &...	args )

Definition at line 152 of file Utils.h.

References EmitWarning().

EmitWarningAlways()

template<typename... Args>

void EmitWarningAlways	(	llvm::StringRef	RemarkName,
		const llvm::Function &	F,
		const Args &...	args )

Definition at line 166 of file Utils.h.

References str().

endsWith()

static bool endsWith ( llvm::StringRef string, llvm::StringRef suffix )

inlinestatic

Definition at line 721 of file Utils.h.

Referenced by calculateUnusedValues(), EnzymePlugin::EnzymePlugin(), and isMemFreeLibMFunction().

ErrorIfRuntimeInactive()

void ErrorIfRuntimeInactive	(	llvm::IRBuilder<> &	B,
		llvm::Value *	primal,
		llvm::Value *	shadow,
		const char *	Message,
		llvm::DebugLoc &&	loc,
		llvm::Instruction *	orig )

Definition at line 902 of file Utils.cpp.

References addFunctionNoCapture(), CustomRuntimeInactiveError, emit_backtrace(), getInt8PtrTy(), getString(), and str().

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

extractBLAS()

llvm::Optional< BlasInfo > extractBLAS

(

llvm::StringRef

in

)

Definition at line 3563 of file Utils.cpp.

References str().

Referenced by AdjointGenerator::handleKnownCallDerivatives(), and TypeAnalyzer::visitCallBase().

findAllUsersOf()

llvm::SmallVector< std::tuple< llvm::Instruction *, llvm::Value *, size_t >, 1 > findAllUsersOf

(

llvm::Value *

AI

)

findInMap()

template<typename K , typename V >

static V * findInMap ( std::map< K, V > & map, K key )

inlinestatic

Definition at line 855 of file Utils.h.

Referenced by CacheUtility::erase(), and CacheUtility::getContext().

FloatToIntTy()

static llvm::Type * FloatToIntTy ( llvm::Type * T )

inlinestatic

Convert a floating point type to an integer type of the same size.

Definition at line 641 of file Utils.h.

References FloatToIntTy().

Referenced by FloatToIntTy().

get1ULP()

llvm::Value * get1ULP	(	llvm::IRBuilder<> &	builder,
		llvm::Value *	res )

Definition at line 4272 of file Utils.cpp.

get_blas_row() [1/2]

llvm::SmallVector< llvm::Value *, 1 > get_blas_row	(	llvm::IRBuilder<> &	B,
		llvm::ArrayRef< llvm::Value * >	trans,
		bool	byRef,
		bool	cublas )

get_blas_row() [2/2]

llvm::SmallVector< llvm::Value *, 1 > get_blas_row	(	llvm::IRBuilder<> &	B,
		llvm::ArrayRef< llvm::Value * >	trans,
		llvm::ArrayRef< llvm::Value * >	row,
		llvm::ArrayRef< llvm::Value * >	col,
		bool	byRef,
		bool	cublas )

get_cached_mat_width()

llvm::Value * get_cached_mat_width	(	llvm::IRBuilder<> &	B,
		llvm::ArrayRef< llvm::Value * >	trans,
		llvm::Value *	arg_ld,
		llvm::Value *	dim_1,
		llvm::Value *	dim_2,
		bool	cacheMat,
		bool	byRef,
		bool	cublas )

Definition at line 4018 of file Utils.cpp.

References CreateSelect(), and is_normal().

getAllocationIndexFromCall()

static llvm::Optional< size_t > getAllocationIndexFromCall ( const llvm::CallBase * op )

inlinestatic

Definition at line 1318 of file Utils.h.

References getFunctionFromCall().

Referenced by TypeAnalyzer::visitCallBase(), and zeroKnownAllocation().

getAncestor()

static llvm::Loop * getAncestor ( llvm::Loop * R1, llvm::Loop * R2 )

inlinestatic

Definition at line 1053 of file Utils.h.

Referenced by overwritesToMemoryReadByLoop().

getBaseObject() [1/2]

static const llvm::Value * getBaseObject ( const llvm::Value * V )

inlinestatic

Definition at line 1638 of file Utils.h.

References getBaseObject().

getBaseObject() [2/2]

static llvm::Value * getBaseObject ( llvm::Value * V, bool offsetAllowed = true )

inlinestatic

Definition at line 1507 of file Utils.h.

References Call, getFuncNameFromCall(), getFunctionFromCall(), and isIntelSubscriptIntrinsic().

Referenced by PreProcessCache::AlwaysInline(), arePointersGuaranteedNoAlias(), DetectReadonlyOrThrowFn(), EnzymeFixupJuliaCallingConvention(), getBaseObject(), getBaseObjects(), moveSRetToFromRoots(), needsReRooting(), AdjointGenerator::recursivelyHandleSubfunction(), RecursivelyReplaceAddressSpace(), replaceToDense(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitCommonStore(), and AdjointGenerator::visitMemSetCommon().

getBaseObjects()

static llvm::SetVector< llvm::Value * > getBaseObjects ( llvm::Value * V, bool offsetAllowed = true )

inlinestatic

Definition at line 1643 of file Utils.h.

References getBaseObject().

getDeallocationIndicesFromCall()

template<typename T >

static std::vector< ssize_t > getDeallocationIndicesFromCall ( T * op )

inlinestatic

Definition at line 1381 of file Utils.h.

References getFunctionFromCall().

getDeallocatorFnFromCall()

template<typename T >

static llvm::Function * getDeallocatorFnFromCall ( T * op )

inlinestatic

Definition at line 1360 of file Utils.h.

References getFunctionFromCall(), and hasMetadata().

getDefaultAnonymousTapeType()

llvm::PointerType * getDefaultAnonymousTapeType

(

llvm::LLVMContext &

C

)

Definition at line 437 of file Utils.cpp.

References EnzymeDefaultTapeType, and getInt8PtrTy().

Referenced by getFunctionTypeForClone(), and traceType().

getFast()

llvm::FastMathFlags getFast

(

)

Get LLVM fast math flags.

Definition at line 3731 of file Utils.cpp.

References EnzymeFastMath().

Referenced by callSPMVDiagUpdate(), CacheUtility::createCacheForScope(), fixSparse_inner(), fixSparseIndices(), DiffeGradientUtils::freeCache(), DiffeGradientUtils::getDifferential(), getOrInsertDifferentialFloatMemcpy(), getorInsertInnerProd(), getOrInsertMemcpyStrided(), getOrInsertOpFloatSum(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), CacheUtility::storeInstructionInCache(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitIntrinsicInst(), and AdjointGenerator::visitOMPCall().

getFirstFunctionDefinition()

llvm::Function * getFirstFunctionDefinition

(

llvm::Module &

M

)

Referenced by EmitFailure().

getFirstNonPHIOrDbg()

static llvm::Instruction * getFirstNonPHIOrDbg ( llvm::BasicBlock * B )

inlinestatic

Definition at line 2272 of file Utils.h.

Referenced by DynamicTraceInterface::DynamicTraceInterface(), and FindCanonicalIV().

getFirstNonPHIOrDbgOrLifetime()

static llvm::Instruction * getFirstNonPHIOrDbgOrLifetime ( llvm::BasicBlock * B )

inlinestatic

Definition at line 2281 of file Utils.h.

Referenced by TraceUtils::GetChoice(), and TraceGenerator::visitFunction().

getFuncName()

static llvm::StringRef getFuncName ( llvm::Function * called )

inlinestatic

Definition at line 1260 of file Utils.h.

Referenced by attributeKnownFunctions(), getFuncNameFromCall(), and SimplifyMPIQueries().

getFuncNameFromCall()

static llvm::StringRef getFuncNameFromCall ( const llvm::CallBase * op )

inlinestatic

Definition at line 1269 of file Utils.h.

References getFuncName(), and getFunctionFromCall().

Referenced by DetectPointerArgOfFn(), DetectReadonlyOrThrowFn(), getBaseObject(), getOrInsertCheckedFree(), hasNoCache(), DifferentialUseAnalysis::is_value_needed_in_reverse(), isAllocationCall(), isDeallocationCall(), isIntelSubscriptIntrinsic(), isPointerArithmeticInst(), LowerSparsification(), AdjointGenerator::recursivelyHandleSubfunction(), replaceToDense(), TypeAnalyzer::run(), shouldDisableNoWrite(), TypeAnalyzer::visitCallBase(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitMemSetCommon(), AdjointGenerator::visitOMPCall(), and writesToMemoryReadBy().

getFunctionFromCall()

template<typename T >

static llvm::Function * getFunctionFromCall ( T * op )

inlinestatic

Definition at line 1236 of file Utils.h.

Referenced by getAllocationIndexFromCall(), getBaseObject(), getDeallocationIndicesFromCall(), getDeallocatorFnFromCall(), getFuncNameFromCall(), hasNoCache(), isLocalReadOnlyOrThrow(), isNoAlias(), isNoCapture(), isNoEscapingAllocation(), isReadOnly(), isReadOnlyOrThrow(), isWriteOnly(), and shouldDisableNoWrite().

GetFunctionFromValue()

llvm::Function * GetFunctionFromValue

(

llvm::Value *

fn

)

getInt8PtrTy()

static llvm::PointerType * getInt8PtrTy ( llvm::LLVMContext & Context, unsigned AddressSpace = 0 )

inlinestatic

Definition at line 1174 of file Utils.h.

References getPointerType().

Referenced by addressType(), CreateDealloc(), AdjointGenerator::DifferentiableMemCopyFloats(), ErrorIfRuntimeInactive(), TraceUtils::GetChoice(), getDefaultAnonymousTapeType(), getMPIHelper(), getOrInsertDifferentialMPI_Wait(), getOrInsertOpFloatSum(), AdjointGenerator::handleMPI(), TraceUtils::InsertFunction(), AdjointGenerator::MPI_TYPE_SIZE(), PreProcessCache::preprocessForClone(), AdjointGenerator::recursivelyHandleSubfunction(), PreProcessCache::ReplaceReallocs(), SanitizeDerivatives(), TraceInterface::stringType(), to_blas_callconv(), AdjointGenerator::visitMemTransferCommon(), and zeroKnownAllocation().

getIntrinsicDeclaration()

static llvm::Function * getIntrinsicDeclaration ( llvm::Module * M, llvm::Intrinsic::ID id, llvm::ArrayRef< llvm::Type * > Tys = {} )

inlinestatic

Definition at line 2263 of file Utils.h.

Referenced by CreateAllocation(), getOrInsertExponentialAllocator(), AdjointGenerator::handleAdjointForIntrinsic(), AdjointGenerator::handleMPI(), PreProcessCache::preprocessForClone(), RecursivelyReplaceAddressSpace(), PreProcessCache::ReplaceReallocs(), and AdjointGenerator::visitCommonStore().

getJuliaObjects()

llvm::SmallVector< llvm::Value *, 1 > getJuliaObjects	(	llvm::Value *	v,
		llvm::IRBuilder<> &	B )

Definition at line 4913 of file Utils.cpp.

References anyJuliaObjects(), and isSpecialPtr().

Referenced by RecursivelyReplaceAddressSpace().

getMPIHelper()

static llvm::StructType * getMPIHelper ( llvm::LLVMContext & Context )

inlinestatic

Definition at line 1183 of file Utils.h.

References getInt8PtrTy().

Referenced by AdjointGenerator::handleMPI().

getMPIMemberPtr()

template<MPI_Elem E, bool Pointer = true>

static llvm::Value * getMPIMemberPtr ( llvm::IRBuilder<> & B, llvm::Value * V, llvm::Type * T )

inlinestatic

Definition at line 1200 of file Utils.h.

References Pointer.

Referenced by AdjointGenerator::handleMPI().

getNextNonDebugInstruction()

static llvm::Instruction * getNextNonDebugInstruction ( llvm::Instruction * Z )

inlinestatic

Get the next non-debug instruction, erring if none exists.

Definition at line 327 of file Utils.h.

References getNextNonDebugInstructionOrNull().

Referenced by CacheUtility::storeInstructionInCache().

getNextNonDebugInstructionOrNull()

static llvm::Instruction * getNextNonDebugInstructionOrNull ( llvm::Instruction * Z )

inlinestatic

Get the next non-debug instruction, if one exists.

Definition at line 318 of file Utils.h.

Referenced by getNextNonDebugInstruction().

getOrInsertCheckedFree()

llvm::Function * getOrInsertCheckedFree	(	llvm::Module &	M,
		llvm::CallInst *	call,
		llvm::Type *	Type,
		unsigned	width )

getOrInsertDifferentialFloatMemcpy()

llvm::Function * getOrInsertDifferentialFloatMemcpy	(	llvm::Module &	M,
		llvm::Type *	T,
		unsigned	dstalign,
		unsigned	srcalign,
		unsigned	dstaddr,
		unsigned	srcaddr,
		unsigned	bitwidth )

Create function for type that performs the derivative memcpy on floating point memory.

getOrInsertDifferentialFloatMemcpyMat()

llvm::Function * getOrInsertDifferentialFloatMemcpyMat	(	llvm::Module &	M,
		llvm::Type *	elementType,
		llvm::PointerType *	PT,
		llvm::IntegerType *	IT,
		llvm::IntegerType *	CT,
		unsigned	dstalign,
		unsigned	srcalign,
		bool	zeroSrc )

getOrInsertDifferentialFloatMemmove()

llvm::Function * getOrInsertDifferentialFloatMemmove	(	llvm::Module &	M,
		llvm::Type *	T,
		unsigned	dstalign,
		unsigned	srcalign,
		unsigned	dstaddr,
		unsigned	srcaddr,
		unsigned	bitwidth )

Create function for type that performs the derivative memmove on floating point memory.

getOrInsertDifferentialMPI_Wait()

llvm::Function * getOrInsertDifferentialMPI_Wait	(	llvm::Module &	M,
		llvm::ArrayRef< llvm::Type * >	T,
		llvm::Type *	reqType,
		llvm::StringRef	caller )

Create function for type that performs the derivative MPI_Wait.

getOrInsertDifferentialWaitallSave()

llvm::Function * getOrInsertDifferentialWaitallSave	(	llvm::Module &	M,
		llvm::ArrayRef< llvm::Type * >	T,
		llvm::PointerType *	reqType )

getorInsertInnerProd()

llvm::CallInst * getorInsertInnerProd	(	llvm::IRBuilder<> &	B,
		llvm::Module &	M,
		BlasInfo	blas,
		llvm::IntegerType *	IT,
		llvm::Type *	BlasPT,
		llvm::Type *	BlasIT,
		llvm::Type *	fpTy,
		llvm::ArrayRef< llvm::Value * >	args,
		const llvm::ArrayRef< llvm::OperandBundleDef >	bundles,
		bool	byRef,
		bool	cublas,
		bool	julia_decl )

getOrInsertMemcpyMat()

llvm::Function * getOrInsertMemcpyMat	(	llvm::Module &	M,
		llvm::Type *	elementType,
		llvm::PointerType *	PT,
		llvm::IntegerType *	IT,
		unsigned	dstalign,
		unsigned	srcalign )

Turned out to be a faster alternatives to lapacks lacpy function.

getOrInsertMemcpyStrided()

llvm::Function * getOrInsertMemcpyStrided	(	llvm::Module &	M,
		llvm::Type *	elementType,
		llvm::PointerType *	T,
		llvm::Type *	IT,
		unsigned	dstalign,
		unsigned	srcalign )

Create function for type that performs memcpy with a stride.

getOrInsertOpFloatSum()

llvm::Value * getOrInsertOpFloatSum	(	llvm::Module &	M,
		llvm::Type *	OpPtr,
		llvm::Type *	OpType,
		ConcreteType	CT,
		llvm::Type *	intType,
		llvm::IRBuilder<> &	B2 )

getPointerType()

static llvm::PointerType * getPointerType ( llvm::Type * T, unsigned AddressSpace = 0 )

inlinestatic

Definition at line 1165 of file Utils.h.

Referenced by AdjointGenerator::DifferentiableMemCopyFloats(), getInt8PtrTy(), getUnqual(), AdjointGenerator::recursivelyHandleSubfunction(), and AdjointGenerator::visitOMPCall().

getRenamedPerCallingConv()

static std::string getRenamedPerCallingConv ( llvm::StringRef caller, llvm::StringRef callee )

inlinestatic

Definition at line 2413 of file Utils.h.

References getRenamedPerCallingConv(), startsWith(), and tripleSplitDollar().

Referenced by getOrInsertDifferentialMPI_Wait(), getRenamedPerCallingConv(), AdjointGenerator::handleMPI(), AdjointGenerator::MPI_COMM_RANK(), AdjointGenerator::MPI_COMM_SIZE(), and AdjointGenerator::MPI_TYPE_SIZE().

getSubType() [1/2]

static llvm::Type * getSubType ( llvm::Type * T )

inlinestatic

Definition at line 2323 of file Utils.h.

Referenced by getSubType().

getSubType() [2/2]

template<typename Arg1 , typename... Args>

static llvm::Type * getSubType ( llvm::Type * T, Arg1 i, Args... args )

inlinestatic

Definition at line 2326 of file Utils.h.

References getSubType().

getUndefinedValueForType()

llvm::Constant * getUndefinedValueForType	(	llvm::Module &	M,
		llvm::Type *	T,
		bool	forceZero = false )

Definition at line 3623 of file Utils.cpp.

References EnzymeUndefinedValueForType, and EnzymeZeroCache.

Referenced by copyNonJLValueInto(), CacheUtility::createCacheForScope(), AdjointGenerator::handleKnownCallDerivatives(), moveSRetToFromRoots(), and AdjointGenerator::recursivelyHandleSubfunction().

getUnqual()

static llvm::PointerType * getUnqual ( llvm::Type * T )

inlinestatic

Definition at line 1179 of file Utils.h.

References getPointerType().

Referenced by defaultTypeTreeForLLVM(), EnzymeFixupJuliaCallingConvention(), TraceUtils::FromClone(), getConstantAnalysis(), getOrInsertDifferentialWaitallSave(), getOrInsertOpFloatSum(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), load_if_ref(), lookup_with_layout(), AdjointGenerator::MPI_COMM_RANK(), AdjointGenerator::MPI_COMM_SIZE(), AdjointGenerator::MPI_TYPE_SIZE(), AdjointGenerator::recursivelyHandleSubfunction(), replaceToDense(), simplifyLoad(), SimplifyMPIQueries(), TypeAnalyzer::visitCallBase(), TypeAnalyzer::visitExtractValueInst(), TypeAnalyzer::visitInsertValueInst(), AdjointGenerator::visitOMPCall(), and TypeAnalyzer::visitShuffleVectorInst().

hasMetadata() [1/3]

static llvm::MDNode * hasMetadata ( const llvm::GlobalObject * O, llvm::StringRef kind )

inlinestatic

Check if a global has metadata.

Definition at line 339 of file Utils.h.

Referenced by PreProcessCache::AlwaysInline(), arePointersGuaranteedNoAlias(), CoaleseTrivialMallocs(), DetectReadonlyOrThrowFn(), EnzymeHasFromStack(), fixSparse_inner(), getDeallocatorFnFromCall(), AdjointGenerator::handleKnownCallDerivatives(), hasNoCache(), PreProcessCache::LowerAllocAddr(), shouldDisableNoWrite(), TypeAnalyzer::visitCallBase(), AdjointGenerator::visitLoadLike(), AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitMemTransferCommon().

hasMetadata() [2/3]

static llvm::MDNode * hasMetadata ( const llvm::Instruction * O, llvm::StringRef kind )

inlinestatic

Check if an instruction has metadata.

Definition at line 345 of file Utils.h.

hasMetadata() [3/3]

static llvm::MDNode * hasMetadata ( const llvm::Instruction * O, unsigned kind )

inlinestatic

Definition at line 349 of file Utils.h.

hasNoCache()

static bool hasNoCache ( llvm::Value * op )

inlinestatic

Definition at line 1283 of file Utils.h.

References EnzymeJuliaAddrLoad, EnzymeNonPower2Cache, getFuncNameFromCall(), getFunctionFromCall(), and hasMetadata().

hasSRetRRootsOrUnionSRet()

static bool hasSRetRRootsOrUnionSRet ( llvm::CallBase * CB )

inlinestatic

Definition at line 2489 of file Utils.h.

hasTerminator()

static bool hasTerminator ( llvm::BasicBlock * BB )

static

Definition at line 2552 of file Utils.h.

insert_or_assign()

template<typename K , typename V >

static std::map< K, V >::iterator insert_or_assign ( std::map< K, V > & map, K & key, V && val )

inlinestatic

Insert into a map.

Definition at line 835 of file Utils.h.

insert_or_assign2()

template<typename K , typename V >

static std::map< K, V >::iterator insert_or_assign2 ( std::map< K, V > & map, K key, V val )

inlinestatic

Insert into a map.

Definition at line 846 of file Utils.h.

Referenced by AdjointGenerator::recursivelyHandleSubfunction(), CacheUtility::replaceAWithB(), AdjointGenerator::visitCallInst(), and AdjointGenerator::visitOMPCall().

IntToFloatTy()

static llvm::Type * IntToFloatTy ( llvm::Type * T )

inlinestatic

Convert a integer type to a floating point type of the same size.

Definition at line 665 of file Utils.h.

References IntToFloatTy().

Referenced by IntToFloatTy(), and AdjointGenerator::visitOMPCall().

is_left()

llvm::Value * is_left	(	llvm::IRBuilder<> &	B,
		llvm::Value *	side,
		bool	byRef,
		bool	cublas )

is_lower()

llvm::Value * is_lower	(	llvm::IRBuilder<> &	B,
		llvm::Value *	uplo,
		bool	byRef,
		bool	cublas )

is_nonunit()

llvm::Value * is_nonunit	(	llvm::IRBuilder<> &	B,
		llvm::Value *	uplo,
		bool	byRef,
		bool	cublas )

is_normal()

llvm::Value * is_normal	(	llvm::IRBuilder<> &	B,
		llvm::Value *	trans,
		bool	byRef,
		bool	cublas )

isCalledFunction()

static llvm::Function * isCalledFunction ( llvm::Value * val )

inlinestatic

Definition at line 239 of file Utils.h.

isCertainPrint()

static bool isCertainPrint ( const llvm::StringRef name )

inlinestatic

Definition at line 729 of file Utils.h.

References startsWith().

Referenced by writesToMemoryReadBy().

isDebugFunction()

static bool isDebugFunction ( llvm::Function * called )

inlinestatic

Definition at line 690 of file Utils.h.

Referenced by DetectReadonlyOrThrowFn(), and writesToMemoryReadBy().

isIntelSubscriptIntrinsic() [1/2]

static bool isIntelSubscriptIntrinsic ( const llvm::IntrinsicInst & II )

inlinestatic

Definition at line 1445 of file Utils.h.

References getFuncNameFromCall(), and startsWith().

Referenced by analyzeIntelSubscriptIntrinsic(), getBaseObject(), DifferentialUseAnalysis::is_value_needed_in_reverse(), isIntelSubscriptIntrinsic(), isPointerArithmeticInst(), PreProcessCache::preprocessForClone(), RecursivelyReplaceAddressSpace(), and AdjointGenerator::visitIntrinsicInst().

isIntelSubscriptIntrinsic() [2/2]

static bool isIntelSubscriptIntrinsic ( const llvm::Value * val )

inlinestatic

Definition at line 1449 of file Utils.h.

References isIntelSubscriptIntrinsic().

isLocalReadOnlyOrThrow() [1/2]

static bool isLocalReadOnlyOrThrow ( const llvm::CallBase * call )

inlinestatic

Definition at line 1730 of file Utils.h.

References getFunctionFromCall(), isLocalReadOnlyOrThrow(), and isReadOnly().

isLocalReadOnlyOrThrow() [2/2]

static bool isLocalReadOnlyOrThrow ( const llvm::Function * F )

inlinestatic

Definition at line 1719 of file Utils.h.

References isReadOnly().

Referenced by DetectReadonlyOrThrowFn(), and isLocalReadOnlyOrThrow().

isNoAlias() [1/2]

static bool isNoAlias ( const llvm::CallBase * call )

inlinestatic

Definition at line 1858 of file Utils.h.

References getFunctionFromCall().

Referenced by arePointersGuaranteedNoAlias(), and isNoAlias().

isNoAlias() [2/2]

static bool isNoAlias ( const llvm::Value * val )

inlinestatic

Definition at line 1869 of file Utils.h.

References isNoAlias().

isNoCapture()

static bool isNoCapture ( const llvm::CallBase * call, size_t idx )

inlinestatic

Definition at line 1840 of file Utils.h.

References getFunctionFromCall().

Referenced by DetectPointerArgOfFn(), notCapturedBefore(), AdjointGenerator::recursivelyHandleSubfunction(), and mlir::enzyme::PointsToPointerAnalysis::visitCallControlFlowTransfer().

isNoEscapingAllocation() [1/2]

static bool isNoEscapingAllocation ( const llvm::CallBase * call )

inlinestatic

Definition at line 1981 of file Utils.h.

References getFunctionFromCall(), and isNoEscapingAllocation().

isNoEscapingAllocation() [2/2]

static bool isNoEscapingAllocation ( const llvm::Function * F )

inlinestatic

Definition at line 1878 of file Utils.h.

Referenced by isNoEscapingAllocation().

isNVLoad()

bool isNVLoad

(

const llvm::Value *

V

)

Definition at line 4483 of file Utils.cpp.

isPointerArithmeticInst()

static bool isPointerArithmeticInst ( const llvm::Value * V, bool includephi = true, bool includebin = true )

inlinestatic

Definition at line 1456 of file Utils.h.

References Call, getFuncNameFromCall(), and isIntelSubscriptIntrinsic().

Referenced by DetectPointerArgOfFn(), and notCapturedBefore().

isReadNone() [1/2]

static bool isReadNone ( const llvm::CallBase * call, ssize_t arg = -1 )

inlinestatic

Definition at line 1832 of file Utils.h.

References isReadOnly(), and isWriteOnly().

isReadNone() [2/2]

static bool isReadNone ( const llvm::Function * F, ssize_t arg = -1 )

inlinestatic

Definition at line 1836 of file Utils.h.

References isReadOnly(), and isWriteOnly().

isReadOnly() [1/2]

static bool isReadOnly ( const llvm::CallBase * call, ssize_t arg = -1 )

inlinestatic

Definition at line 1691 of file Utils.h.

References getFunctionFromCall(), and isReadOnly().

isReadOnly() [2/2]

static bool isReadOnly ( const llvm::Function * F, ssize_t arg = -1 )

inlinestatic

Definition at line 1673 of file Utils.h.

Referenced by isLocalReadOnlyOrThrow(), isLocalReadOnlyOrThrow(), isReadNone(), isReadNone(), isReadOnly(), isReadOnlyOrThrow(), and isReadOnlyOrThrow().

isReadOnlyOrThrow() [1/2]

static bool isReadOnlyOrThrow ( const llvm::CallBase * call )

inlinestatic

Definition at line 1769 of file Utils.h.

References getFunctionFromCall(), isReadOnly(), and isReadOnlyOrThrow().

isReadOnlyOrThrow() [2/2]

static bool isReadOnlyOrThrow ( const llvm::Function * F )

inlinestatic

Definition at line 1759 of file Utils.h.

References isReadOnly().

Referenced by DetectReadonlyOrThrowFn(), and isReadOnlyOrThrow().

isReturned()

static bool isReturned ( llvm::Instruction * inst )

inlinestatic

Check whether this instruction is returned.

Definition at line 631 of file Utils.h.

isSpecialPtr()

static bool isSpecialPtr ( llvm::Type * Ty )

inlinestatic

Definition at line 2354 of file Utils.h.

References FirstSpecial, and LastSpecial.

Referenced by anyJuliaObjects(), CountTrackedPointers::CountTrackedPointers(), getJuliaObjects(), moveSRetToFromRoots(), needsReRooting(), and AdjointGenerator::visitCallInst().

isWriteOnly() [1/2]

static bool isWriteOnly ( const llvm::CallBase * call, ssize_t arg = -1 )

inlinestatic

Definition at line 1804 of file Utils.h.

References getFunctionFromCall(), and isWriteOnly().

isWriteOnly() [2/2]

static bool isWriteOnly ( const llvm::Function * F, ssize_t arg = -1 )

inlinestatic

Definition at line 1788 of file Utils.h.

Referenced by DetectPointerArgOfFn(), EnzymeFixupJuliaCallingConvention(), isReadNone(), isReadNone(), isWriteOnly(), and AdjointGenerator::recursivelyHandleSubfunction().

load_if_ref()

llvm::Value * load_if_ref	(	llvm::IRBuilder<> &	B,
		llvm::Type *	intType,
		llvm::Value *	V,
		bool	byRef )

Definition at line 4075 of file Utils.cpp.

References getUnqual().

Referenced by callSPMVDiagUpdate(), and getorInsertInnerProd().

lookup_with_layout()

llvm::Value * lookup_with_layout	(	llvm::IRBuilder<> &	B,
		llvm::Type *	fpType,
		llvm::Value *	layout,
		llvm::Value *	base,
		llvm::Value *	lda,
		llvm::Value *	row,
		llvm::Value *	col )

max()

template<typename T >

static T max ( T a, T b )

inlinestatic

Pick the maximum value.

Definition at line 262 of file Utils.h.

Referenced by TypeAnalyzer::considerTBAA(), TypeAnalyzer::visitCallBase(), and TypeAnalyzer::visitMemTransferCommon().

mayExecuteAfter()

void mayExecuteAfter	(	llvm::SmallVectorImpl< llvm::Instruction * > &	results,
		llvm::Instruction *	inst,
		const llvm::SmallPtrSetImpl< llvm::Instruction * > &	stores,
		const llvm::Loop *	region )

min()

template<typename T >

static T min ( T a, T b )

inlinestatic

Pick the maximum value.

Definition at line 268 of file Utils.h.

moveSRetToFromRoots()

llvm::Value * moveSRetToFromRoots	(	llvm::IRBuilder<> &	B,
		llvm::Type *	jltype,
		llvm::Value *	sret,
		llvm::Type *	root_ty,
		llvm::Value *	rootRet,
		size_t	rootOffset,
		SRetRootMovement	direction )

Definition at line 4714 of file Utils.cpp.

References constantInBoundsGEPHelper(), CountTrackedPointers::count, GradientUtils::extractMeta(), getBaseObject(), getUndefinedValueForType(), isSpecialPtr(), NullifySRetValue, RootPointerToSRetPointer, RootPointerToSRetValue, SRetPointerToRootPointer, SRetValueToRootPointer, and Tracked.

Referenced by EnzymeFixupJuliaCallingConvention().

nextPowerOfTwo()

llvm::Value * nextPowerOfTwo	(	llvm::IRBuilder<> &	B,
		llvm::Value *	V )

Create function to computer nearest power of two.

Definition at line 2192 of file Utils.cpp.

Referenced by PreProcessCache::ReplaceReallocs().

notCaptured()

bool notCaptured

(

llvm::Value *

V

)

Check if value if b captured.

Definition at line 4608 of file Utils.cpp.

References notCapturedBefore().

Referenced by DetectReadonlyOrThrowFn().

notCapturedBefore()

bool notCapturedBefore	(	llvm::Value *	V,
		llvm::Instruction *	inst,
		size_t	checkLoadCaptured )

Check if value if b captured after definition before executing inst.

If checkLoadCaptured != 0, also consider catpures of any loads of the value as a capture (for the number of loads set).

operator<<() [1/4]

static llvm::raw_ostream & operator<< ( llvm::raw_ostream & os, DerivativeMode mode )

inlinestatic

Definition at line 458 of file Utils.h.

References to_string().

operator<<() [2/4]

static llvm::raw_ostream & operator<< ( llvm::raw_ostream & os, DIFFE_TYPE mode )

inlinestatic

Definition at line 480 of file Utils.h.

References to_string().

operator<<() [3/4]

static llvm::raw_ostream & operator<< ( llvm::raw_ostream & os, ReturnType mode )

inlinestatic

Definition at line 510 of file Utils.h.

References to_string().

operator<<() [4/4]

static llvm::raw_ostream & operator<< ( llvm::raw_ostream & os, ValueType mode )

inlinestatic

Definition at line 435 of file Utils.h.

References to_string().

overwritesToMemoryReadBy()

bool overwritesToMemoryReadBy	(	const TypeResults *	TR,
		llvm::AAResults &	AA,
		llvm::TargetLibraryInfo &	TLI,
		llvm::ScalarEvolution &	SE,
		llvm::LoopInfo &	LI,
		llvm::DominatorTree &	DT,
		llvm::Instruction *	maybeReader,
		llvm::Instruction *	maybeWriter,
		llvm::Loop *	scope = nullptr )

Definition at line 2765 of file Utils.cpp.

References arePointersGuaranteedNoAlias(), overwritesToMemoryReadByLoop(), and writesToMemoryReadBy().

parseTrueType()

std::vector< std::tuple< llvm::Type *, size_t, size_t > > parseTrueType	(	const llvm::MDNode *	md,
		DerivativeMode	Mode,
		bool	const_src )

Definition at line 4411 of file Utils.cpp.

References Anything, ForwardMode, and ForwardModeError.

Referenced by AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitMemTransferCommon().

PostCacheStore()

llvm::SmallVector< llvm::Instruction *, 2 > PostCacheStore	(	llvm::StoreInst *	SI,
		llvm::IRBuilder<> &	B )

Definition at line 423 of file Utils.cpp.

References EnzymePostCacheStore.

Referenced by CacheUtility::createCacheForScope(), EnzymeFixupJuliaCallingConvention(), and RecursivelyReplaceAddressSpace().

SanitizeDerivatives()

llvm::Value * SanitizeDerivatives	(	llvm::Value *	val,
		llvm::Value *	toset,
		llvm::IRBuilder<> &	BuilderM,
		llvm::Value *	mask = nullptr )

Definition at line 3634 of file Utils.cpp.

References CustomErrorHandler, emit_backtrace(), EnzymeCheckDerivativeNaN(), EnzymeSanitizeDerivatives, getInt8PtrTy(), getString(), NaNError, and str().

Referenced by DiffeGradientUtils::setDiffe().

shouldDisableNoWrite()

static bool shouldDisableNoWrite ( const llvm::CallInst * CI )

inlinestatic

Definition at line 1423 of file Utils.h.

References getFuncNameFromCall(), getFunctionFromCall(), and hasMetadata().

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

simplifyLoad()

llvm::Value * simplifyLoad	(	llvm::Value *	LI,
		size_t	valSz = 0,
		size_t	preOffset = 0 )

startsWith()

static bool startsWith ( llvm::StringRef string, llvm::StringRef prefix )

inlinestatic

Definition at line 713 of file Utils.h.

Referenced by fixSparseIndices(), ForceRecursiveInlining(), getRenamedPerCallingConv(), handleCustomDerivative(), AdjointGenerator::handleKnownCallDerivatives(), isCertainPrint(), mlir::enzyme::ActivityAnalyzer::isConstantValue(), isIntelSubscriptIntrinsic(), isItaniumEncoding(), isMemFreeLibMFunction(), isProduct(), isSum(), PreProcessCache::preprocessForClone(), preserveNVVM(), tripleSplitDollar(), TypeAnalyzer::visitCallBase(), AdjointGenerator::visitCallInst(), and AdjointGenerator::visitMemSetCommon().

to_blas_callconv()

llvm::Value * to_blas_callconv	(	llvm::IRBuilder<> &	B,
		llvm::Value *	V,
		bool	byRef,
		bool	cublas,
		llvm::IntegerType *	julia_decl,
		llvm::IRBuilder<> &	entryBuilder,
		llvm::Twine const &	= "" )

to_blas_fp_callconv()

llvm::Value * to_blas_fp_callconv	(	llvm::IRBuilder<> &	B,
		llvm::Value *	V,
		bool	byRef,
		llvm::Type *	julia_decl,
		llvm::IRBuilder<> &	entryBuilder,
		llvm::Twine const &	= "" )

to_string() [1/5]

template<typename T >

static std::string to_string ( const std::set< T > & us )

inlinestatic

Output a set as a string.

Definition at line 276 of file Utils.h.

Referenced by operator<<(), operator<<(), operator<<(), and operator<<().

to_string() [2/5]

static std::string to_string ( DerivativeMode mode )

inlinestatic

Definition at line 440 of file Utils.h.

References ForwardMode, ForwardModeError, ForwardModeSplit, ReverseModeCombined, ReverseModeGradient, and ReverseModePrimal.

to_string() [3/5]

static std::string to_string ( DIFFE_TYPE t )

inlinestatic

Convert DIFFE_TYPE to a string.

Definition at line 464 of file Utils.h.

References CONSTANT, DUP_ARG, DUP_NONEED, and OUT_DIFF.

to_string() [4/5]

static std::string to_string ( ReturnType t )

inlinestatic

Convert ReturnType to a string.

Definition at line 486 of file Utils.h.

References Args, ArgsWithReturn, ArgsWithTwoReturns, Return, Tape, TapeAndReturn, TapeAndTwoReturns, TwoReturns, and Void.

to_string() [5/5]

static std::string to_string ( ValueType mode )

inlinestatic

Definition at line 421 of file Utils.h.

References Both, None, Primal, and Shadow.

transpose() [1/2]

llvm::Value * transpose	(	std::string	floatType,
		llvm::IRBuilder<> &	B,
		llvm::Value *	V,
		bool	byRef,
		bool	cublas,
		llvm::IntegerType *	IT,
		llvm::IRBuilder<> &	entryBuilder,
		const llvm::Twine &	name )

Definition at line 4034 of file Utils.cpp.

References to_blas_callconv(), and transpose().

transpose() [2/2]

llvm::Value * transpose	(	std::string	floatType,
		llvm::IRBuilder<> &	B,
		llvm::Value *	V,
		bool	cublas )

tripleSplitDollar()

static std::tuple< llvm::StringRef, llvm::StringRef, llvm::StringRef > tripleSplitDollar ( llvm::StringRef caller )

inlinestatic

Definition at line 2404 of file Utils.h.

References startsWith().

Referenced by getOrInsertDifferentialMPI_Wait(), and getRenamedPerCallingConv().

whatType() [1/2]

static DIFFE_TYPE whatType ( llvm::Type * arg, DerivativeMode mode )

inlinestatic

Definition at line 625 of file Utils.h.

References whatType().

whatType() [2/2]

static DIFFE_TYPE whatType ( llvm::Type * arg, DerivativeMode mode, bool integersAreConstant, std::set< llvm::Type * > & seen )

inlinestatic

Attempt to automatically detect the differentiable classification based off of a given type.

Definition at line 519 of file Utils.h.

References CONSTANT, DUP_ARG, DUP_NONEED, ForwardMode, ForwardModeError, ForwardModeSplit, OUT_DIFF, and whatType().

Referenced by AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitOMPCall(), whatType(), and whatType().

writesToMemoryReadBy()

bool writesToMemoryReadBy	(	const TypeResults *	TR,
		llvm::AAResults &	AA,
		llvm::TargetLibraryInfo &	TLI,
		llvm::Instruction *	maybeReader,
		llvm::Instruction *	maybeWriter )

Return whether maybeReader can read from memory written to by maybeWriter.

Definition at line 2880 of file Utils.cpp.

References Anything, Constant, getFuncNameFromCall(), isAllocationFunction(), isCertainPrint(), isDeallocationFunction(), isDebugFunction(), ConcreteType::isKnown(), isMemFreeLibMFunction(), TypeTree::Lookup(), parseTBAA(), Pointer, TypeResults::query(), and Unknown.

Referenced by arePointersGuaranteedNoAlias(), and overwritesToMemoryReadBy().

ZeroMemory()

void ZeroMemory	(	llvm::IRBuilder<> &	Builder,
		llvm::Type *	T,
		llvm::Value *	obj,
		bool	isTape )

Definition at line 414 of file Utils.cpp.

References CustomZero.

Referenced by DiffeGradientUtils::getDifferential().

Variable Documentation

CustomErrorHandler

LLVMValueRef(* CustomErrorHandler) (const char *, LLVMValueRef, ErrorType, const void *, LLVMValueRef, LLVMBuilderRef) ( const char * , LLVMValueRef , ErrorType , const void * , LLVMValueRef , LLVMBuilderRef  )

extern

Definition at line 62 of file Utils.cpp.

Referenced by analyzeIntelSubscriptIntrinsic(), EmitNoDerivativeError(), EmitNoDerivativeError(), EmitNoTypeError(), EnzymeFixupJuliaCallingConvention(), CacheUtility::erase(), TypeResults::firstPointer(), TypeTree::insert(), needsReRooting(), TypeTree::Only(), AdjointGenerator::recursivelyHandleSubfunction(), RecursivelyReplaceAddressSpace(), SanitizeDerivatives(), transpose(), TypeAnalyzer::visitBinaryOperation(), AdjointGenerator::visitCommonStore(), TypeAnalyzer::visitGEPOperator(), TypeAnalyzer::visitIntrinsicInst(), TypeAnalyzer::visitMemTransferCommon(), and TypeAnalyzer::visitPHINode().

EnzymeBlasCopy

llvm::cl::opt<bool> EnzymeBlasCopy

extern

EnzymeJuliaAddrLoad

llvm::cl::opt<bool> EnzymeJuliaAddrLoad

extern

Referenced by DetectReadonlyOrThrowFn(), AdjointGenerator::handleKnownCallDerivatives(), hasNoCache(), RecursivelyReplaceAddressSpace(), AdjointGenerator::visitCallInst(), and AdjointGenerator::visitFenceInst().

EnzymeLapackCopy

llvm::cl::opt<bool> EnzymeLapackCopy

extern

EnzymeNonPower2Cache

llvm::cl::opt<bool> EnzymeNonPower2Cache

extern

Referenced by hasNoCache().

EnzymePrintPerf

llvm::cl::opt<bool> EnzymePrintPerf

extern

Print additional debug info relevant to performance.

Referenced by DetectReadonlyOrThrowFn(), EmitWarning(), and EmitWarning().

PrimalParamAttrsToPreserve

llvm::Attribute::AttrKind PrimalParamAttrsToPreserve[]

inlinestatic

Initial value:

= {
  llvm::Attribute::AttrKind::ReadOnly,
  llvm::Attribute::AttrKind::WriteOnly,
  llvm::Attribute::AttrKind::ZExt,
  llvm::Attribute::AttrKind::SExt,
  llvm::Attribute::AttrKind::InReg,
  llvm::Attribute::AttrKind::ByVal,
 
 
 
  llvm::Attribute::AttrKind::Preallocated,
  llvm::Attribute::AttrKind::InAlloca,
 
 
 
 
 
 
  llvm::Attribute::AttrKind::NoFree,
  llvm::Attribute::AttrKind::Alignment,
  llvm::Attribute::AttrKind::StackAlignment,
 
 
 
  llvm::Attribute::AttrKind::NoCapture,
 
  llvm::Attribute::AttrKind::ReadNone
}

Definition at line 2208 of file Utils.h.

Referenced by PreProcessCache::CloneFunctionWithReturns(), and AdjointGenerator::recursivelyHandleSubfunction().

shadowHandlers

llvm::StringMap<std::function<llvm::Value *( llvm::IRBuilder<> &, llvm::CallInst *, llvm::ArrayRef<llvm::Value *>, GradientUtils *)> > shadowHandlers

extern

Definition at line 70 of file GradientUtils.cpp.

Referenced by EnzymeRegisterAllocationHandler(), and AdjointGenerator::handleKnownCallDerivatives().

ShadowParamAttrsToPreserve

llvm::Attribute::AttrKind ShadowParamAttrsToPreserve[]

inlinestatic

Initial value:

= {
  llvm::Attribute::AttrKind::ZExt,
  llvm::Attribute::AttrKind::SExt,
 
 
 
  llvm::Attribute::AttrKind::NoFree,
  llvm::Attribute::AttrKind::Alignment,
  llvm::Attribute::AttrKind::StackAlignment,
 
 
 
  llvm::Attribute::AttrKind::NoCapture,
 
  llvm::Attribute::AttrKind::ReadNone,
}

Definition at line 2240 of file Utils.h.

Referenced by PreProcessCache::CloneFunctionWithReturns(), and AdjointGenerator::recursivelyHandleSubfunction().