Utils.h File Reference

#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/ValueMap.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/IR/Instructions.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
#include <map>
#include <set>
#include "llvm/ADT/Triple.h"
#include "llvm/Support/AMDGPUMetadata.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "TypeAnalysis/ConcreteType.h"
#include "EnzymeCallMarkers.h"
#include "llvm/IR/CFG.h"
#include <deque>
#include <functional>
#include "llvm/Analysis/LoopInfo.h"

Include dependency graph for Utils.h:

Go to the source code of this file.

Classes
class	EnzymeWarning
class	EnzymeFailure
struct	BlasInfo
class	AssertingReplacingVH
struct	CountTrackedPointers

Namespaces
namespace	llvm

Macros
#define	amd_target   amdgcn

Enumerations
enum class	ErrorType {   NoDerivative = 0 , NoShadow = 1 , IllegalTypeAnalysis = 2 , NoType = 3 ,   IllegalFirstPointer = 4 , InternalError = 5 , TypeDepthExceeded = 6 , MixedActivityError = 7 ,   IllegalReplaceFicticiousPHIs = 8 , GetIndexError = 9 , NoTruncate = 10 , GCRewrite = 11 ,   NaNError = 12 , ShowInternalError = 12 , NoAccumulate = 13 }
enum class	ReturnType {   ArgsWithReturn , ArgsWithTwoReturns , Args , TapeAndReturn ,   TapeAndTwoReturns , Tape , TwoReturns , Return ,   Void }
	Potential return type of generated functions. More...
enum class	BATCH_TYPE { SCALAR = 0 , VECTOR = 1 }
enum class	DerivativeMode {   ForwardMode = 0 , ReverseModePrimal = 1 , ReverseModeGradient = 2 , ReverseModeCombined = 3 ,   ForwardModeSplit = 4 , ForwardModeError = 5 }
enum class	ProbProgMode { Likelihood = 0 , Trace = 1 , Condition = 2 }
enum class	ValueType { None = 0 , Primal = 1 , Shadow = 2 , Both = Primal | Shadow }
	Classification of value as an original program variable, a derivative variable, neither, or both. More...
enum class	MPI_CallType { ISEND = 1 , IRECV = 2 }
enum class	MPI_Elem {   Buf = 0 , Count = 1 , DataType = 2 , Src = 3 ,   Tag = 4 , Comm = 5 , Call = 6 , Old = 7 }
enum	AddressSpace {   Generic = 0 , Tracked = 10 , Derived = 11 , CalleeRooted = 12 ,   Loaded = 13 , FirstSpecial = Tracked , LastSpecial = Loaded }
enum class	SRetRootMovement {   SRetPointerToRootPointer = 0 , SRetValueToRootPointer = 1 , RootPointerToSRetValue = 2 , RootPointerToSRetPointer = 3 ,   NullifySRetValue = 4 }

Functions
static bool	isAnyBranch (const llvm::Value *V)
static bool	isConditionalBranch (const llvm::Value *V)
static bool	isUnconditionalBranch (const llvm::Value *V)
static void	resetClonedGUID (llvm::Function *F)
static llvm::Value *	getBranchCondition (llvm::Value *V)
	The condition of a branch isConditionalBranch says yes to.
static void	setBranchCondition (llvm::Value *V, llvm::Value *Cond)
template<typename PosT >
static llvm::Instruction *	createUnconditionalBranch (llvm::BasicBlock *Target, PosT InsertBefore)
static bool	isGPUArch (const llvm::Triple &TT)
static unsigned	getGPUSharedAddrSpace (const llvm::Triple &TT)
llvm::SmallVector< llvm::Instruction *, 2 >	PostCacheStore (llvm::StoreInst *SI, llvm::IRBuilder<> &B)
llvm::Value *	CreateAllocation (llvm::IRBuilder<> &B, llvm::Type *T, llvm::Value *Count, const llvm::Twine &Name="", llvm::CallInst **caller=nullptr, llvm::Instruction **ZeroMem=nullptr, bool isDefault=false)
llvm::CallInst *	CreateDealloc (llvm::IRBuilder<> &B, llvm::Value *ToFree)
void	ZeroMemory (llvm::IRBuilder<> &Builder, llvm::Type *T, llvm::Value *obj, bool isTape)
llvm::Value *	CreateReAllocation (llvm::IRBuilder<> &B, llvm::Value *prev, llvm::Type *T, llvm::Value *OuterCount, llvm::Value *InnerCount, const llvm::Twine &Name="", llvm::CallInst **caller=nullptr, bool ZeroMem=false)
llvm::PointerType *	getDefaultAnonymousTapeType (llvm::LLVMContext &C)
static bool	EmitWarningEnabled (const llvm::LLVMContext &Ctx)
template<typename... Args>
void	EmitWarning (llvm::StringRef RemarkName, const llvm::DiagnosticLocation &Loc, const llvm::BasicBlock *BB, const Args &...args)
template<typename... Args>
void	EmitWarning (llvm::StringRef RemarkName, const llvm::Instruction &I, const Args &...args)
template<typename... Args>
void	EmitWarningAlways (llvm::StringRef RemarkName, const llvm::DiagnosticLocation &Loc, const llvm::Function &F, const Args &...args)
	Emit a diagnostic which is always shown, unlike EmitWarning which requires the user to opt in with -Rpass=enzyme.
template<typename... Args>
void	EmitWarningAlways (llvm::StringRef RemarkName, const llvm::Instruction &I, const Args &...args)
template<typename... Args>
void	EmitWarningAlways (llvm::StringRef RemarkName, const llvm::Function &F, const Args &...args)
template<typename... Args>
void	EmitWarning (llvm::StringRef RemarkName, const llvm::Function &F, const Args &...args)
llvm::Function *	getFirstFunctionDefinition (llvm::Module &M)
template<typename... Args>
void	EmitFailure (llvm::StringRef RemarkName, const llvm::DiagnosticLocation &Loc, const llvm::Instruction *CodeRegion, Args &...args)
template<typename... Args>
void	EmitFailure (llvm::StringRef RemarkName, const llvm::DiagnosticLocation &Loc, const llvm::Function *CodeRegion, Args &...args)
template<typename... Args>
void	EmitFailure (llvm::StringRef RemarkName, llvm::Module &M, Args &...args)
static llvm::Function *	isCalledFunction (llvm::Value *val)
llvm::Value *	EmitNoDerivativeError (const std::string &message, llvm::Instruction &inst, GradientUtils *gutils, llvm::IRBuilder<> &B, llvm::Value *condition=nullptr)
bool	EmitNoDerivativeError (const std::string &message, llvm::Value *todiff, RequestContext &ctx)
void	EmitNoTypeError (const std::string &, llvm::Instruction &inst, GradientUtils *gutils, llvm::IRBuilder<> &B)
llvm::FastMathFlags	getFast ()
	Get LLVM fast math flags.
template<typename T >
static T	max (T a, T b)
	Pick the maximum value.
template<typename T >
static T	min (T a, T b)
	Pick the maximum value.
template<typename T >
static std::string	to_string (const std::set< T > &us)
	Output a set as a string.
template<typename T , typename N >
static void	dumpMap (const llvm::ValueMap< T, N > &o, llvm::function_ref< bool(const llvm::Value *)> shouldPrint=[](T) { return true;})
	Print a map, optionally with a shouldPrint function to decide to print a given value.
template<typename T >
static void	dumpSet (const llvm::SmallPtrSetImpl< T * > &o)
	Print a set.
template<typename T >
static void	dumpSet (const llvm::SetVector< T * > &o)
static llvm::Instruction *	getNextNonDebugInstructionOrNull (llvm::Instruction *Z)
	Get the next non-debug instruction, if one exists.
static llvm::Instruction *	getNextNonDebugInstruction (llvm::Instruction *Z)
	Get the next non-debug instruction, erring if none exists.
static llvm::MDNode *	hasMetadata (const llvm::GlobalObject *O, llvm::StringRef kind)
	Check if a global has metadata.
static llvm::MDNode *	hasMetadata (const llvm::Instruction *O, llvm::StringRef kind)
	Check if an instruction has metadata.
static llvm::MDNode *	hasMetadata (const llvm::Instruction *O, unsigned kind)
static std::string	to_string (ValueType mode)
static llvm::raw_ostream &	operator<< (llvm::raw_ostream &os, ValueType mode)
static std::string	to_string (DerivativeMode mode)
static llvm::raw_ostream &	operator<< (llvm::raw_ostream &os, DerivativeMode mode)
static std::string	to_string (DIFFE_TYPE t)
	Convert DIFFE_TYPE to a string.
static llvm::raw_ostream &	operator<< (llvm::raw_ostream &os, DIFFE_TYPE mode)
static std::string	to_string (ReturnType t)
	Convert ReturnType to a string.
static llvm::raw_ostream &	operator<< (llvm::raw_ostream &os, ReturnType mode)
static DIFFE_TYPE	whatType (llvm::Type *arg, DerivativeMode mode, bool integersAreConstant, std::set< llvm::Type * > &seen)
	Attempt to automatically detect the differentiable classification based off of a given type.
llvm::Value *	get1ULP (llvm::IRBuilder<> &builder, llvm::Value *res)
static DIFFE_TYPE	whatType (llvm::Type *arg, DerivativeMode mode)
static bool	isReturned (llvm::Instruction *inst)
	Check whether this instruction is returned.
static llvm::Type *	FloatToIntTy (llvm::Type *T)
	Convert a floating point type to an integer type of the same size.
static llvm::Type *	IntToFloatTy (llvm::Type *T)
	Convert a integer type to a floating point type of the same size.
static bool	isDebugFunction (llvm::Function *called)
static bool	startsWith (llvm::StringRef string, llvm::StringRef prefix)
static bool	endsWith (llvm::StringRef string, llvm::StringRef suffix)
static bool	isCertainPrint (const llvm::StringRef name)
llvm::Optional< BlasInfo >	extractBLAS (llvm::StringRef in)
std::vector< std::tuple< llvm::Type *, size_t, size_t > >	parseTrueType (const llvm::MDNode *, DerivativeMode, bool const_src)
llvm::MDNode *	sliceTrueType (const llvm::MDNode *md, size_t start, size_t length)
	Restrict an enzyme_truetype annotation describing a memory region to the byte range [start, start + length), rebasing its offsets to that range.
bool	isAtomic (llvm::Value *origptr, bool AtomicAdd, llvm::Function *newFunc)
llvm::Function *	getOrInsertDifferentialFloatMemcpy (llvm::Module &M, llvm::Type *T, unsigned dstalign, unsigned srcalign, unsigned dstaddr, unsigned srcaddr, unsigned bitwidth, bool runtimeActivity, bool atomic)
	Create function for type that performs the derivative memcpy on floating point memory.
void	callMemcpyStridedBlas (llvm::IRBuilder<> &B, llvm::Module &M, BlasInfo blas, llvm::ArrayRef< llvm::Value * > args, llvm::Type *cublas_retty, llvm::ArrayRef< llvm::OperandBundleDef > bundles)
	Create function for type that performs memcpy with a stride using blas copy.
void	callMemcpyStridedLapack (llvm::IRBuilder<> &B, llvm::Module &M, BlasInfo blas, llvm::ArrayRef< llvm::Value * > args, llvm::ArrayRef< llvm::OperandBundleDef > bundles)
	Create function for type that performs memcpy using lapack copy.
void	callSPMVDiagUpdate (llvm::IRBuilder<> &B, llvm::Module &M, BlasInfo blas, llvm::IntegerType *IT, llvm::Type *BlasCT, llvm::Type *BlasFPT, llvm::Type *BlasPT, llvm::Type *BlasIT, llvm::Type *fpTy, llvm::ArrayRef< llvm::Value * > args, const llvm::ArrayRef< llvm::OperandBundleDef > bundles, bool byRef, bool julia_decl)
llvm::CallInst *	getorInsertInnerProd (llvm::IRBuilder<> &B, llvm::Module &M, BlasInfo blas, llvm::IntegerType *IT, llvm::Type *BlasPT, llvm::Type *BlasIT, llvm::Type *fpTy, llvm::ArrayRef< llvm::Value * > args, const llvm::ArrayRef< llvm::OperandBundleDef > bundles, bool byRef, bool cublas, bool julia_decl)
llvm::Function *	getOrInsertMemcpyStrided (llvm::Module &M, llvm::Type *elementType, llvm::PointerType *T, llvm::Type *IT, unsigned dstalign, unsigned srcalign)
	Create function for type that performs memcpy with a stride.
llvm::Function *	getOrInsertMemcpyMat (llvm::Module &M, llvm::Type *elementType, llvm::PointerType *PT, llvm::IntegerType *IT, unsigned dstalign, unsigned srcalign)
	Turned out to be a faster alternatives to lapacks lacpy function.
llvm::Function *	getOrInsertDifferentialFloatMemcpyMat (llvm::Module &M, llvm::Type *elementType, llvm::PointerType *PT, llvm::IntegerType *IT, llvm::IntegerType *CT, unsigned dstalign, unsigned srcalign, bool zeroSrc)
llvm::Function *	getOrInsertDifferentialFloatMemmove (llvm::Module &M, llvm::Type *T, unsigned dstalign, unsigned srcalign, unsigned dstaddr, unsigned srcaddr, unsigned bitwidth, bool runtimeActivity, bool atomic)
	Create function for type that performs the derivative memmove on floating point memory.
llvm::Function *	getOrInsertCheckedFree (llvm::Module &M, llvm::CallInst *call, llvm::Type *Type, unsigned width)
llvm::Function *	getOrInsertDifferentialMPI_Wait (llvm::Module &M, llvm::ArrayRef< llvm::Type * > T, llvm::Type *reqType, llvm::StringRef caller)
	Create function for type that performs the derivative MPI_Wait.
llvm::Value *	nextPowerOfTwo (llvm::IRBuilder<> &B, llvm::Value *V)
	Create function to computer nearest power of two.
template<typename K , typename V >
static std::map< K, V >::iterator	insert_or_assign (std::map< K, V > &map, K &key, V &&val)
	Insert into a map.
template<typename K , typename V >
static std::map< K, V >::iterator	insert_or_assign2 (std::map< K, V > &map, K key, V val)
	Insert into a map.
template<typename K , typename V >
static V *	findInMap (std::map< K, V > &map, K key)
static void	allFollowersOf (llvm::Instruction *inst, llvm::function_ref< bool(llvm::Instruction *)> f)
	Call the function f for all instructions that happen after inst If the function returns true, the iteration will early exit.
static void	allPredecessorsOf (llvm::Instruction *inst, llvm::function_ref< bool(llvm::Instruction *)> f)
	Call the function f for all instructions that happen before inst If the function returns true, the iteration will early exit.
static void	allDomPredecessorsOf (llvm::Instruction *inst, llvm::DominatorTree &DT, llvm::function_ref< bool(llvm::Instruction *)> f)
	Call the function f for all instructions that happen before inst If the function returns true, the iteration will early exit.
static void	allUnsyncdPredecessorsOf (llvm::Instruction *inst, llvm::function_ref< bool(llvm::Instruction *)> f, llvm::function_ref< void()> preEntry)
	Call the function f for all instructions that happen before inst If the function returns true, the iteration will early exit.
static llvm::Loop *	getAncestor (llvm::Loop *R1, llvm::Loop *R2)
void	mayExecuteAfter (llvm::SmallVectorImpl< llvm::Instruction * > &results, llvm::Instruction *inst, const llvm::SetVector< llvm::Instruction * > &stores, const llvm::Loop *region)
bool	writesToMemoryReadBy (const TypeResults *TR, llvm::AAResults &AA, llvm::TargetLibraryInfo &TLI, llvm::Instruction *maybeReader, llvm::Instruction *maybeWriter)
	Return whether maybeReader can read from memory written to by maybeWriter.
bool	overwritesToMemoryReadBy (const TypeResults *TR, llvm::AAResults &AA, llvm::TargetLibraryInfo &TLI, llvm::ScalarEvolution &SE, llvm::LoopInfo &LI, llvm::DominatorTree &DT, llvm::Instruction *maybeReader, llvm::Instruction *maybeWriter, llvm::Loop *scope=nullptr)
static void	allInstructionsBetween (llvm::LoopInfo &LI, llvm::Instruction *inst1, llvm::Instruction *inst2, llvm::function_ref< bool(llvm::Instruction *)> f)
	Call the function f for all instructions that happen between inst1 and inst2 If the function returns true, the iteration will early exit.
static llvm::PointerType *	getPointerType (llvm::Type *T, unsigned AddressSpace=0)
	Pointer into the given address space.
static llvm::PointerType *	getInt8PtrTy (llvm::LLVMContext &Context, unsigned AddressSpace=0)
static llvm::PointerType *	getUnqual (llvm::Type *T)
static llvm::PointerType *	changePointerAddrSpace (llvm::PointerType *PT, unsigned AddressSpace)
	The same pointer type, in a different address space.
static uint64_t	getAggregateElementOffset (const llvm::DataLayout &DL, llvm::Type *T, llvm::ArrayRef< unsigned > Idxs)
	The byte offset of the aggregate element addressed by Idxs within T, that is, what gep T, ptr, 0, Idxs... would compute.
static uint64_t	getAggregateElementOffset (const llvm::DataLayout &DL, llvm::Type *T, uint64_t Idx)
	The byte offset of element Idx of the aggregate type T.
static llvm::StructType *	getMPIHelper (llvm::LLVMContext &Context)
template<MPI_Elem E, bool Pointer = true>
static llvm::Value *	getMPIMemberPtr (llvm::IRBuilder<> &B, llvm::Value *V, llvm::Type *T)
llvm::Value *	getOrInsertOpFloatSum (llvm::Module &M, llvm::Function *templateFn, llvm::Type *OpPtr, llvm::Type *OpType, ConcreteType CT, llvm::Type *intType, llvm::IRBuilder<> &B2)
	templateFn is the MPI entry point this reduction is being built for; the MPI_Op_create it needs is named to match.
template<typename T >
static llvm::Function *	getFunctionFromCall (T *op)
static llvm::StringRef	getFuncName (llvm::Function *called)
static llvm::StringRef	getFuncNameFromCall (const llvm::CallBase *op)
static bool	hasNoCache (llvm::Value *op)
static llvm::Optional< size_t >	getAllocationIndexFromCall (const llvm::CallBase *op)
template<typename T >
static llvm::Function *	getDeallocatorFnFromCall (T *op)
template<typename T >
static std::vector< ssize_t >	getDeallocationIndicesFromCall (T *op)
llvm::Function *	getOrInsertDifferentialWaitallSave (llvm::Module &M, llvm::ArrayRef< llvm::Type * > T, llvm::PointerType *reqType)
void	ErrorIfRuntimeInactive (llvm::IRBuilder<> &B, llvm::Value *primal, llvm::Value *shadow, const char *Message, llvm::DebugLoc &&loc, llvm::Instruction *orig)
llvm::Function *	GetFunctionFromValue (llvm::Value *fn)
llvm::Value *	simplifyLoad (llvm::Value *LI, size_t valSz=0, size_t preOffset=0)
template<typename T >
static bool	hasCustomRuleMetadata (const T *V)
static bool	shouldDisableNoWrite (const llvm::CallInst *CI)
static bool	isIntelSubscriptIntrinsic (const llvm::IntrinsicInst &II)
static bool	isIntelSubscriptIntrinsic (const llvm::Value *val)
static bool	isPointerArithmeticInst (const llvm::Value *V, bool includephi=true, bool includebin=true)
static llvm::Value *	getBaseObject (llvm::Value *V, bool offsetAllowed=true)
static const llvm::Value *	getBaseObject (const llvm::Value *V)
static llvm::SetVector< llvm::Value * >	getBaseObjects (llvm::Value *V, bool offsetAllowed=true)
static bool	isReadOnly (const llvm::Function *F, ssize_t arg=-1)
static bool	isReadOnly (const llvm::CallBase *call, ssize_t arg=-1)
static bool	isLocalReadOnlyOrThrow (const llvm::Function *F)
static bool	isLocalReadOnlyOrThrow (const llvm::CallBase *call)
static bool	isReadOnlyOrThrow (const llvm::Function *F)
static bool	isReadOnlyOrThrow (const llvm::CallBase *call)
static bool	isWriteOnly (const llvm::Function *F, ssize_t arg=-1)
static bool	isWriteOnly (const llvm::CallBase *call, ssize_t arg=-1)
static bool	isReadNone (const llvm::CallBase *call, ssize_t arg=-1)
static bool	isReadNone (const llvm::Function *F, ssize_t arg=-1)
static bool	isNoCapture (const llvm::CallBase *call, size_t idx)
static bool	isNoAlias (const llvm::CallBase *call)
static bool	isNoAlias (const llvm::Value *val)
bool	isFlangRuntimeNoFree (llvm::StringRef name)
	LLVM flang runtime functions that neither free memory of the program nor return or store an allocation into it: I/O (which frees only the I/O statement state that _FortranAioBegin* allocated and _FortranAioEndIoStatement releases), FLUSH, the scalar character inquiries and comparison, and MOD/MODULO.
static bool	isNoEscapingAllocation (const llvm::Function *F)
static bool	isNoEscapingAllocation (const llvm::CallBase *call)
bool	attributeKnownFunctions (llvm::Function &F)
llvm::Constant *	getUndefinedValueForType (llvm::Module &M, llvm::Type *T, bool forceZero=false)
llvm::Value *	SanitizeDerivatives (llvm::Value *val, llvm::Value *toset, llvm::IRBuilder<> &BuilderM, llvm::Value *mask=nullptr)
static llvm::Value *	CreateSelect (llvm::IRBuilder<> &Builder2, llvm::Value *cmp, llvm::Value *tval, llvm::Value *fval, const llvm::Twine &Name="")
static llvm::Value *	checkedMul (bool strongZero, llvm::IRBuilder<> &Builder2, llvm::Value *idiff, llvm::Value *pres, const llvm::Twine &Name="")
static llvm::Value *	checkedDiv (bool strongZero, llvm::IRBuilder<> &Builder2, llvm::Value *idiff, llvm::Value *pres, const llvm::Twine &Name="")
static bool	containsOnlyAtMostTopBit (const llvm::Value *V, llvm::Type *FT, const llvm::DataLayout &dl, llvm::Type **vFT=nullptr)
void	addValueToCache (llvm::Value *arg, bool cache_arg, llvm::Type *ty, llvm::SmallVectorImpl< llvm::Value * > &cacheValues, llvm::IRBuilder<> &BuilderZ, const llvm::Twine &name="")
llvm::Value *	load_if_ref (llvm::IRBuilder<> &B, llvm::Type *intType, llvm::Value *V, bool byRef)
void	copy_lower_to_upper (llvm::IRBuilder<> &B, llvm::Type *fpType, BlasInfo blas, bool byRef, llvm::Value *layout, llvm::Value *uplo, llvm::Value *A, llvm::Value *lda, llvm::Value *N)
llvm::Value *	to_blas_callconv (llvm::IRBuilder<> &B, llvm::Value *V, bool byRef, bool cublas, llvm::IntegerType *julia_decl, llvm::IRBuilder<> &entryBuilder, llvm::Twine const &="")
llvm::Value *	to_blas_fp_callconv (llvm::IRBuilder<> &B, llvm::Value *V, bool byRef, llvm::Type *julia_decl, llvm::IRBuilder<> &entryBuilder, llvm::Twine const &="")
llvm::Value *	get_cached_mat_width (llvm::IRBuilder<> &B, llvm::ArrayRef< llvm::Value * > trans, llvm::Value *arg_ld, llvm::Value *dim_1, llvm::Value *dim_2, bool cacheMat, bool byRef, bool cublas)
template<typename T >
static void	append (llvm::SmallVectorImpl< T > &vec)
template<typename T , typename... T2>
static void	append (llvm::SmallVectorImpl< T > &vec, llvm::ArrayRef< T > vals, T2 &&...ts)
template<typename... T>
static llvm::SmallVector< llvm::Value *, 1 >	concat_values (T &&...t)
llvm::Value *	is_normal (llvm::IRBuilder<> &B, llvm::Value *trans, bool byRef, bool cublas)
llvm::Value *	is_left (llvm::IRBuilder<> &B, llvm::Value *side, bool byRef, bool cublas)
llvm::Value *	is_lower (llvm::IRBuilder<> &B, llvm::Value *uplo, bool byRef, bool cublas)
llvm::Value *	is_nonunit (llvm::IRBuilder<> &B, llvm::Value *uplo, bool byRef, bool cublas)
llvm::Value *	lookup_with_layout (llvm::IRBuilder<> &B, llvm::Type *fpType, llvm::Value *layout, llvm::Value *base, llvm::Value *lda, llvm::Value *row, llvm::Value *col)
llvm::Value *	transpose (std::string floatType, llvm::IRBuilder<> &B, llvm::Value *V, bool cublas)
llvm::Value *	transpose (std::string floatType, llvm::IRBuilder<> &B, llvm::Value *V, bool byRef, bool cublas, llvm::IntegerType *IT, llvm::IRBuilder<> &entryBuilder, const llvm::Twine &name)
llvm::SmallVector< llvm::Value *, 1 >	get_blas_row (llvm::IRBuilder<> &B, llvm::ArrayRef< llvm::Value * > trans, llvm::ArrayRef< llvm::Value * > row, llvm::ArrayRef< llvm::Value * > col, bool byRef, bool cublas)
llvm::SmallVector< llvm::Value *, 1 >	get_blas_row (llvm::IRBuilder<> &B, llvm::ArrayRef< llvm::Value * > trans, bool byRef, bool cublas)
static llvm::Function *	getIntrinsicDeclaration (llvm::Module *M, llvm::Intrinsic::ID id, llvm::ArrayRef< llvm::Type * > Tys={})
static llvm::Instruction *	getFirstNonPHI (llvm::BasicBlock *B)
static void	insertBeforeInst (llvm::Instruction *I, llvm::Instruction *Before)
static void	moveBeforeInst (llvm::Instruction *I, llvm::Instruction *Before)
static llvm::Instruction *	getFirstNonPHIOrDbg (llvm::BasicBlock *B)
static llvm::Instruction *	getFirstNonPHIOrDbgOrLifetime (llvm::BasicBlock *B)
static void	addCallSiteNoCapture (llvm::CallBase *call, size_t idx)
static void	addFunctionNoCapture (llvm::Function *call, size_t idx)
static llvm::AttributeList	addFunctionNoCapture (llvm::LLVMContext &ctx, llvm::AttributeList list, size_t idx)
static llvm::Type *	getSubType (llvm::Type *T)
template<typename Arg1 , typename... Args>
static llvm::Type *	getSubType (llvm::Type *T, Arg1 i, Args... args)
static bool	isSpecialPtr (llvm::Type *Ty)
bool	collectOffset (llvm::GEPOperator *gep, const llvm::DataLayout &DL, unsigned BitWidth, llvm::MapVector< llvm::Value *, llvm::APInt > &VariableOffsets, llvm::APInt &ConstantOffset)
llvm::CallInst *	createIntrinsicCall (llvm::IRBuilderBase &B, llvm::Intrinsic::ID ID, llvm::Type *RetTy, llvm::ArrayRef< llvm::Value * > Args, llvm::Instruction *FMFSource=nullptr, const llvm::Twine &Name="")
bool	isNVLoad (const llvm::Value *V)
bool	notCapturedBefore (llvm::Value *V, llvm::Instruction *inst, size_t checkLoadCaptured, llvm::Instruction *startinst=nullptr, llvm::TargetLibraryInfo *TLI=nullptr)
	Check if value if b captured after definition before executing inst.
bool	notCaptured (llvm::Value *V, llvm::TargetLibraryInfo *TLI=nullptr)
	Check if value if b captured.
llvm::Optional< bool >	arePointersGuaranteedNoAlias (llvm::TargetLibraryInfo &TLI, llvm::AAResults &AA, llvm::DominatorTree &DT, llvm::LoopInfo &LI, llvm::Value *op0, llvm::Value *op1, bool offsetAllowed=false)
static llvm::StringRef	canonicalizeMPIName (llvm::StringRef Name)
	Canonicalize the name of an MPI routine across calling conventions: both the C convention ("MPI_Recv", "PMPI_Recv") and common Fortran ABI manglings ("mpi_recv_", "mpi_recv__", "pmpi_recv_", "MPI_RECV", ...) map to the canonical C name without profiling prefix (e.g.
static bool	isFortranMPICall (llvm::StringRef Name)
	True if Name uses a Fortran ABI mangling of an MPI routine (trailing underscore(s), e.g.
static std::tuple< llvm::StringRef, llvm::StringRef, llvm::StringRef >	tripleSplitDollar (llvm::StringRef caller)
static std::string	getRenamedPerCallingConv (llvm::StringRef caller, llvm::StringRef callee)
llvm::FunctionCallee	getOrInsertPerCallingConv (llvm::Module &M, llvm::Function *templateFn, llvm::StringRef callee, llvm::FunctionType *FT)
	Declare callee in M under whatever naming convention the frontend used to reach templateFn, recording the plain name in enzyme_math so that the declaration is still recognized afterwards.
static std::string	convertSRetTypeToString (llvm::Type *T)
static llvm::Type *	convertSRetTypeFromString (llvm::StringRef str, llvm::LLVMContext *C=nullptr)
static size_t	convertRRootCountFromString (llvm::StringRef str)
static bool	hasSRetRRootsOrUnionSRet (llvm::CallBase *CB)
llvm::Value *	moveSRetToFromRoots (llvm::IRBuilder<> &B, llvm::Type *jltype, llvm::Value *sret, llvm::Type *root_ty, llvm::Value *rootRet, size_t rootOffset, SRetRootMovement direction)
void	copyNonJLValueInto (llvm::IRBuilder<> &B, llvm::Type *curType, llvm::Type *dstType, llvm::Value *dst, llvm::ArrayRef< unsigned > dstPrefix, llvm::Type *srcType, llvm::Value *src, llvm::ArrayRef< unsigned > srcPrefix, bool shouldZero)
static bool	anyJuliaObjects (llvm::Type *T)
llvm::SmallVector< llvm::Value *, 1 >	getJuliaObjects (llvm::Value *v, llvm::IRBuilder<> &B)
llvm::SmallVector< std::tuple< llvm::Instruction *, llvm::Value *, size_t >, 1 >	findAllUsersOf (llvm::Value *AI)
static bool	hasTerminator (llvm::BasicBlock *BB)
const llvm::SCEV *	evaluateAtIterationWithoutExt (const llvm::SCEVAddRecExpr *AR, const llvm::SCEV *It, llvm::ScalarEvolution &SE)
	The value of the add recurrence AR at iteration It, like SCEVAddRecExpr::evaluateAtIteration, but without the widening that its BinomialCoefficient performs: LLVM computes I*(I-1)/2 in a type one bit wider than AR so that the halving is exact, and SCEVExpander then has to materialize a canonical induction variable of that wider type, a new loop-carried phi.

Variables
llvm::cl::opt< bool >	EnzymePrintPerf
	Print additional debug info relevant to performance.
llvm::cl::opt< bool >	EnzymeNonPower2Cache
llvm::cl::opt< bool >	EnzymeBlasCopy
llvm::cl::opt< bool >	EnzymeLapackCopy
llvm::cl::opt< bool >	EnzymeJuliaAddrLoad
LLVMValueRef(*	CustomErrorHandler )(const char *, LLVMValueRef, ErrorType, const void *, LLVMValueRef, LLVMBuilderRef)
llvm::StringMap< std::function< llvm::Value *(llvm::IRBuilder<> &, llvm::CallInst *, llvm::ArrayRef< llvm::Value * >, GradientUtils *)> >	shadowHandlers
static constexpr const char *	MixedActivityHint
	Remedy appended to mismatched-activity warnings, which are emitted when a value Enzyme proved inactive is used where an active value may be required.
static llvm::Attribute::AttrKind	PrimalParamAttrsToPreserve []
static llvm::Attribute::AttrKind	ShadowParamAttrsToPreserve []
static const std::pair< const char *, const char * >	KnownMPIFunctions []
	MPI routines recognized by Enzyme, keyed by their lowercase, underscore-free base name and mapped to the canonical C name (without the profiling "P" prefix).

Macro Definition Documentation

amd_target

#define amd_target   amdgcn

Definition at line 149 of file Utils.h.

Enumeration Type Documentation

AddressSpace

enum AddressSpace

Enumerator
Generic
Tracked
Derived
CalleeRooted
Loaded
FirstSpecial
LastSpecial

Definition at line 2649 of file Utils.h.

BATCH_TYPE

enum class BATCH_TYPE

strong

Enumerator
SCALAR
VECTOR

Definition at line 519 of file Utils.h.

DerivativeMode

enum class DerivativeMode

strong

Enumerator
ForwardMode
ReverseModePrimal
ReverseModeGradient
ReverseModeCombined
ForwardModeSplit
ForwardModeError

Definition at line 524 of file Utils.h.

ErrorType

enum class ErrorType

strong

Enumerator
NoDerivative
NoShadow
IllegalTypeAnalysis
NoType
IllegalFirstPointer
InternalError
TypeDepthExceeded
MixedActivityError
IllegalReplaceFicticiousPHIs
GetIndexError
NoTruncate
GCRewrite
NaNError
ShowInternalError
NoAccumulate

Definition at line 189 of file Utils.h.

MPI_CallType

enum class MPI_CallType

strong

Enumerator
ISEND
IRECV

Definition at line 1293 of file Utils.h.

MPI_Elem

enum class MPI_Elem

strong

Enumerator
Buf
Count
DataType
Src
Tag
Comm
Call
Old

Definition at line 1298 of file Utils.h.

ProbProgMode

enum class ProbProgMode

strong

Enumerator
Likelihood
Trace
Condition

Definition at line 533 of file Utils.h.

ReturnType

enum class ReturnType

strong

Potential return type of generated functions.

Enumerator
ArgsWithReturn	Return is a struct of all args and the original return.
ArgsWithTwoReturns	Return is a struct of all args and two of the original return.
Args	Return is a struct of all args.
TapeAndReturn	Return is a tape type and the original return.
TapeAndTwoReturns	Return is a tape type and the two of the original return.
Tape	Return is a tape type.
TwoReturns
Return
Void

Definition at line 499 of file Utils.h.

SRetRootMovement

enum class SRetRootMovement

strong

Enumerator
SRetPointerToRootPointer
SRetValueToRootPointer
RootPointerToSRetValue
RootPointerToSRetPointer
NullifySRetValue

Definition at line 2934 of file Utils.h.

ValueType

enum class ValueType

strong

Classification of value as an original program variable, a derivative variable, neither, or both.

This type is used both in differential use analysis and to describe argument bundles.

Enumerator
None
Primal
Shadow
Both

Definition at line 543 of file Utils.h.

Function Documentation

addCallSiteNoCapture()

static void addCallSiteNoCapture ( llvm::CallBase * call, size_t idx )

inlinestatic

Definition at line 2599 of file Utils.h.

Referenced by TraceUtils::GetChoice(), TraceUtils::GetTrace(), TraceUtils::HasCall(), TraceUtils::HasChoice(), TraceUtils::InsertArgument(), TraceUtils::InsertArgumentGradient(), TraceUtils::InsertCall(), TraceUtils::InsertChoice(), TraceUtils::InsertChoiceGradient(), and SimplifyMPIQueries().

addFunctionNoCapture() [1/2]

static void addFunctionNoCapture ( llvm::Function * call, size_t idx )

inlinestatic

Definition at line 2609 of file Utils.h.

Referenced by attributeKnownFunctions(), callSPMVDiagUpdate(), copy_lower_to_upper(), DetectPointerArgOfFn(), ErrorIfRuntimeInactive(), getOrInsertCheckedFree(), getOrInsertDifferentialFloatMemcpy(), getorInsertInnerProd(), getOrInsertMemcpyMat(), getOrInsertMemcpyStrided(), getOrInsertOpFloatSum(), AdjointGenerator::MPI_COMM_RANK(), AdjointGenerator::MPI_COMM_SIZE(), AdjointGenerator::MPI_TYPE_SIZE(), and OldAllocationSize().

addFunctionNoCapture() [2/2]

static llvm::AttributeList addFunctionNoCapture ( llvm::LLVMContext & ctx, llvm::AttributeList list, size_t idx )

inlinestaticnodiscard

Definition at line 2620 of file Utils.h.

addValueToCache()

void addValueToCache	(	llvm::Value *	arg,
		bool	cache_arg,
		llvm::Type *	ty,
		llvm::SmallVectorImpl< llvm::Value * > &	cacheValues,
		llvm::IRBuilder<> &	BuilderZ,
		const llvm::Twine &	name = "" )

allDomPredecessorsOf()

static void allDomPredecessorsOf ( llvm::Instruction * inst, llvm::DominatorTree & DT, llvm::function_ref< bool(llvm::Instruction *)> f )

inlinestatic

Call the function f for all instructions that happen before inst If the function returns true, the iteration will early exit.

Definition at line 1085 of file Utils.h.

allFollowersOf()

static void allFollowersOf ( llvm::Instruction * inst, llvm::function_ref< bool(llvm::Instruction *)> f )

inlinestatic

Call the function f for all instructions that happen after inst If the function returns true, the iteration will early exit.

Definition at line 1013 of file Utils.h.

allInstructionsBetween()

static void allInstructionsBetween ( llvm::LoopInfo & LI, llvm::Instruction * inst1, llvm::Instruction * inst2, llvm::function_ref< bool(llvm::Instruction *)> f )

inlinestatic

Call the function f for all instructions that happen between inst1 and inst2 If the function returns true, the iteration will early exit.

Definition at line 1243 of file Utils.h.

References allPredecessorsOf().

Referenced by arePointersGuaranteedNoAlias().

allPredecessorsOf()

static void allPredecessorsOf ( llvm::Instruction * inst, llvm::function_ref< bool(llvm::Instruction *)> f )

inlinestatic

Call the function f for all instructions that happen before inst If the function returns true, the iteration will early exit.

Definition at line 1048 of file Utils.h.

Referenced by allInstructionsBetween().

allUnsyncdPredecessorsOf()

static void allUnsyncdPredecessorsOf ( llvm::Instruction * inst, llvm::function_ref< bool(llvm::Instruction *)> f, llvm::function_ref< void()> preEntry )

inlinestatic

Call the function f for all instructions that happen before inst If the function returns true, the iteration will early exit.

Definition at line 1124 of file Utils.h.

anyJuliaObjects()

static bool anyJuliaObjects ( llvm::Type * T )

static

Definition at line 2953 of file Utils.h.

References anyJuliaObjects(), and isSpecialPtr().

Referenced by anyJuliaObjects(), and getJuliaObjects().

append() [1/2]

template<typename T >

static void append ( llvm::SmallVectorImpl< T > & vec )

inlinestatic

Definition at line 2431 of file Utils.h.

Referenced by append(), and concat_values().

append() [2/2]

template<typename T , typename... T2>

static void append ( llvm::SmallVectorImpl< T > & vec, llvm::ArrayRef< T > vals, T2 &&... ts )

inlinestatic

Definition at line 2433 of file Utils.h.

References append().

arePointersGuaranteedNoAlias()

llvm::Optional< bool > arePointersGuaranteedNoAlias	(	llvm::TargetLibraryInfo &	TLI,
		llvm::AAResults &	AA,
		llvm::DominatorTree &	DT,
		llvm::LoopInfo &	LI,
		llvm::Value *	op0,
		llvm::Value *	op1,
		bool	offsetAllowed = false )

attributeKnownFunctions()

bool attributeKnownFunctions

(

llvm::Function &

F

)

Definition at line 139 of file Utils.cpp.

References addFunctionNoCapture(), canonicalizeMPIName(), getFuncName(), and isFlangRuntimeNoFree().

Referenced by callMemcpyStridedBlas(), callMemcpyStridedLapack(), copy_lower_to_upper(), EnzymeAttributeKnownFunctions(), getorInsertInnerProd(), and preserveNVVM().

callMemcpyStridedBlas()

void callMemcpyStridedBlas	(	llvm::IRBuilder<> &	B,
		llvm::Module &	M,
		BlasInfo	blas,
		llvm::ArrayRef< llvm::Value * >	args,
		llvm::Type *	cublas_retty,
		llvm::ArrayRef< llvm::OperandBundleDef >	bundles )

Create function for type that performs memcpy with a stride using blas copy.

Definition at line 1526 of file Utils.cpp.

References attributeKnownFunctions(), BlasInfo::floatType, GetFunctionFromValue(), BlasInfo::prefix, and BlasInfo::suffix.

callMemcpyStridedLapack()

void callMemcpyStridedLapack	(	llvm::IRBuilder<> &	B,
		llvm::Module &	M,
		BlasInfo	blas,
		llvm::ArrayRef< llvm::Value * >	args,
		llvm::ArrayRef< llvm::OperandBundleDef >	bundles )

Create function for type that performs memcpy using lapack copy.

Definition at line 1546 of file Utils.cpp.

References attributeKnownFunctions(), BlasInfo::floatType, GetFunctionFromValue(), BlasInfo::prefix, and BlasInfo::suffix.

callSPMVDiagUpdate()

void callSPMVDiagUpdate	(	llvm::IRBuilder<> &	B,
		llvm::Module &	M,
		BlasInfo	blas,
		llvm::IntegerType *	IT,
		llvm::Type *	BlasCT,
		llvm::Type *	BlasFPT,
		llvm::Type *	BlasPT,
		llvm::Type *	BlasIT,
		llvm::Type *	fpTy,
		llvm::ArrayRef< llvm::Value * >	args,
		const llvm::ArrayRef< llvm::OperandBundleDef >	bundles,
		bool	byRef,
		bool	julia_decl )

canonicalizeMPIName()

static llvm::StringRef canonicalizeMPIName ( llvm::StringRef Name )

inlinestatic

Canonicalize the name of an MPI routine across calling conventions: both the C convention ("MPI_Recv", "PMPI_Recv") and common Fortran ABI manglings ("mpi_recv_", "mpi_recv__", "pmpi_recv_", "MPI_RECV", ...) map to the canonical C name without profiling prefix (e.g.

"MPI_Recv"). Returns an empty StringRef if the name is not a known MPI routine.

Definition at line 2780 of file Utils.h.

References KnownMPIFunctions.

Referenced by attributeKnownFunctions(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), isFortranMPICall(), and TypeAnalyzer::visitCallBase().

changePointerAddrSpace()

static llvm::PointerType * changePointerAddrSpace ( llvm::PointerType * PT, unsigned AddressSpace )

inlinestatic

The same pointer type, in a different address space.

On a typed-pointer build the pointee type is carried over; an opaque pointer has none to carry. Prefer this to rebuilding a pointer type by hand: it says that only the address space changes, rather than leaving it to the reader to check that dropping the pointee was deliberate.

Definition at line 1338 of file Utils.h.

References getPointerType().

Referenced by RecursivelyReplaceAddressSpace(), and replaceToDense().

checkedDiv()

static llvm::Value * checkedDiv ( bool strongZero, llvm::IRBuilder<> & Builder2, llvm::Value * idiff, llvm::Value * pres, const llvm::Twine & Name = "" )

inlinestatic

Definition at line 2310 of file Utils.h.

checkedMul()

static llvm::Value * checkedMul ( bool strongZero, llvm::IRBuilder<> & Builder2, llvm::Value * idiff, llvm::Value * pres, const llvm::Twine & Name = "" )

inlinestatic

Definition at line 2296 of file Utils.h.

Referenced by AdjointGenerator::createBinaryOperatorAdjoint(), and AdjointGenerator::createBinaryOperatorDual().

collectOffset()

bool collectOffset	(	llvm::GEPOperator *	gep,
		const llvm::DataLayout &	DL,
		unsigned	BitWidth,
		llvm::MapVector< llvm::Value *, llvm::APInt > &	VariableOffsets,
		llvm::APInt &	ConstantOffset )

concat_values()

template<typename... T>

static llvm::SmallVector< llvm::Value *, 1 > concat_values ( T &&... t )

inlinestatic

Definition at line 2439 of file Utils.h.

References append().

containsOnlyAtMostTopBit()

static bool containsOnlyAtMostTopBit ( const llvm::Value * V, llvm::Type * FT, const llvm::DataLayout & dl, llvm::Type ** vFT = nullptr )

inlinestatic

Definition at line 2325 of file Utils.h.

References containsOnlyAtMostTopBit().

Referenced by containsOnlyAtMostTopBit(), AdjointGenerator::createBinaryOperatorAdjoint(), AdjointGenerator::createBinaryOperatorDual(), and TypeAnalyzer::visitBinaryOperation().

convertRRootCountFromString()

static size_t convertRRootCountFromString ( llvm::StringRef str )

inlinestatic

Definition at line 2910 of file Utils.h.

References str().

Referenced by EnzymeFixupJuliaCallingConvention(), and needsReReturning().

convertSRetTypeFromString()

static llvm::Type * convertSRetTypeFromString ( llvm::StringRef str, llvm::LLVMContext * C = nullptr )

inlinestatic

Definition at line 2858 of file Utils.h.

References getPointerType(), str(), and Tracked.

Referenced by EnzymeFixupJuliaCallingConvention(), and needsReRooting().

convertSRetTypeToString()

static std::string convertSRetTypeToString ( llvm::Type * T )

inlinestatic

Definition at line 2853 of file Utils.h.

Referenced by PreProcessCache::CloneFunctionWithReturns(), EnzymeFixupJuliaCallingConvention(), and AdjointGenerator::recursivelyHandleSubfunction().

copy_lower_to_upper()

void copy_lower_to_upper	(	llvm::IRBuilder<> &	B,
		llvm::Type *	fpType,
		BlasInfo	blas,
		bool	byRef,
		llvm::Value *	layout,
		llvm::Value *	uplo,
		llvm::Value *	A,
		llvm::Value *	lda,
		llvm::Value *	N )

Definition at line 1382 of file Utils.cpp.

References addFunctionNoCapture(), attributeKnownFunctions(), CreateSelect(), BlasInfo::floatType, getFunctionFromCall(), GetFunctionFromValue(), lookup_with_layout(), BlasInfo::prefix, BlasInfo::suffix, and to_blas_callconv().

copyNonJLValueInto()

void copyNonJLValueInto	(	llvm::IRBuilder<> &	B,
		llvm::Type *	curType,
		llvm::Type *	dstType,
		llvm::Value *	dst,
		llvm::ArrayRef< unsigned >	dstPrefix,
		llvm::Type *	srcType,
		llvm::Value *	src,
		llvm::ArrayRef< unsigned >	srcPrefix,
		bool	shouldZero )

Definition at line 5271 of file Utils.cpp.

References constantInBoundsGEPHelper(), CountTrackedPointers::count, and getUndefinedValueForType().

Referenced by EnzymeFixupJuliaCallingConvention().

CreateAllocation()

llvm::Value * CreateAllocation	(	llvm::IRBuilder<> &	B,
		llvm::Type *	T,
		llvm::Value *	Count,
		const llvm::Twine &	Name = "",
		llvm::CallInst **	caller = nullptr,
		llvm::Instruction **	ZeroMem = nullptr,
		bool	isDefault = false )

CreateDealloc()

llvm::CallInst * CreateDealloc	(	llvm::IRBuilder<> &	B,
		llvm::Value *	ToFree )

Definition at line 871 of file Utils.cpp.

References CustomDeallocator, and getInt8PtrTy().

Referenced by DiffeGradientUtils::freeCache(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), and AdjointGenerator::visitOMPCall().

createIntrinsicCall()

llvm::CallInst * createIntrinsicCall	(	llvm::IRBuilderBase &	B,
		llvm::Intrinsic::ID	ID,
		llvm::Type *	RetTy,
		llvm::ArrayRef< llvm::Value * >	Args,
		llvm::Instruction *	FMFSource = nullptr,
		const llvm::Twine &	Name = "" )

Definition at line 4528 of file Utils.cpp.

References Args.

CreateReAllocation()

llvm::Value * CreateReAllocation	(	llvm::IRBuilder<> &	B,
		llvm::Value *	prev,
		llvm::Type *	T,
		llvm::Value *	OuterCount,
		llvm::Value *	InnerCount,
		const llvm::Twine &	Name = "",
		llvm::CallInst **	caller = nullptr,
		bool	ZeroMem = false )

Definition at line 685 of file Utils.cpp.

References getOrInsertExponentialAllocator().

Referenced by CacheUtility::createCacheForScope().

CreateSelect()

static llvm::Value * CreateSelect ( llvm::IRBuilder<> & Builder2, llvm::Value * cmp, llvm::Value * tval, llvm::Value * fval, const llvm::Twine & Name = "" )

inlinestatic

Definition at line 2283 of file Utils.h.

Referenced by copy_lower_to_upper(), AdjointGenerator::createBinaryOperatorAdjoint(), AdjointGenerator::createBinaryOperatorDual(), AdjointGenerator::createSelectInstAdjoint(), get_cached_mat_width(), AdjointGenerator::handleAdjointForIntrinsic(), AdjointGenerator::handleMPI(), lookup_with_layout(), AdjointGenerator::visitBinaryOperator(), and AdjointGenerator::visitLoadLike().

createUnconditionalBranch()

template<typename PosT >

static llvm::Instruction * createUnconditionalBranch ( llvm::BasicBlock * Target, PosT InsertBefore )

inlinestatic

Definition at line 127 of file Utils.h.

dumpMap()

template<typename T , typename N >

static void dumpMap ( const llvm::ValueMap< T, N > & o, llvm::function_ref< bool(const llvm::Value *)> shouldPrint = [](T) { return true; } )

inlinestatic

Print a map, optionally with a shouldPrint function to decide to print a given value.

Definition at line 430 of file Utils.h.

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

dumpSet() [1/2]

template<typename T >

static void dumpSet ( const llvm::SetVector< T * > & o )

inlinestatic

Definition at line 453 of file Utils.h.

dumpSet() [2/2]

template<typename T >

static void dumpSet ( const llvm::SmallPtrSetImpl< T * > & o )

inlinestatic

Print a set.

Definition at line 445 of file Utils.h.

EmitFailure() [1/3]

template<typename... Args>

void EmitFailure	(	llvm::StringRef	RemarkName,
		const llvm::DiagnosticLocation &	Loc,
		const llvm::Function *	CodeRegion,
		Args &...	args )

Definition at line 358 of file Utils.h.

References str().

EmitFailure() [2/3]

template<typename... Args>

void EmitFailure	(	llvm::StringRef	RemarkName,
		const llvm::DiagnosticLocation &	Loc,
		const llvm::Instruction *	CodeRegion,
		Args &...	args )

Definition at line 347 of file Utils.h.

References str().

Referenced by Constraints::allSolutions(), checkEnzymeTypeExtent(), AdjointGenerator::DifferentiableMemCopyFloats(), EmitFailure(), EmitNoDerivativeError(), EmitNoDerivativeError(), EmitNoTypeError(), EnzymeFixupJuliaCallingConvention(), CacheUtility::erase(), fixSparseIndices(), getSparseConditions(), OldAllocationSize(), AdjointGenerator::recursivelyHandleSubfunction(), RecursivelyReplaceAddressSpace(), replaceToDense(), ActivityAnalysisPrinterNewPM::run(), TypeAnalysisPrinterNewPM::run(), transpose(), TypeAnalyzer::visitBinaryOperation(), InstructionBatcher::visitBranchInst(), InstructionBatcher::visitInstruction(), TypeAnalyzer::visitIntrinsicInst(), TypeAnalyzer::visitMemTransferCommon(), TypeAnalyzer::visitPHINode(), and InstructionBatcher::visitSwitchInst().

EmitFailure() [3/3]

template<typename... Args>

void EmitFailure	(	llvm::StringRef	RemarkName,
		llvm::Module &	M,
		Args &...	args )

Definition at line 369 of file Utils.h.

References EmitFailure(), getFirstFunctionDefinition(), and str().

EmitNoDerivativeError() [1/2]

llvm::Value * EmitNoDerivativeError	(	const std::string &	message,
		llvm::Instruction &	inst,
		GradientUtils *	gutils,
		llvm::IRBuilder<> &	B,
		llvm::Value *	condition = nullptr )

Definition at line 4593 of file Utils.cpp.

References contains(), CustomErrorHandler, TypeResults::dump(), emit_backtrace(), EmitFailure(), EnzymeRuntimeError(), getString(), NoDerivative, str(), and GradientUtils::TR.

Referenced by AdjointGenerator::createBinaryOperatorAdjoint(), AdjointGenerator::createBinaryOperatorDual(), AdjointGenerator::handleAdjointForIntrinsic(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitAtomicRMWInst(), AdjointGenerator::visitCastInst(), AdjointGenerator::visitInstruction(), and AdjointGenerator::visitMemSetCommon().

EmitNoDerivativeError() [2/2]

bool EmitNoDerivativeError	(	const std::string &	message,
		llvm::Value *	todiff,
		RequestContext &	ctx )

EmitNoTypeError()

void EmitNoTypeError	(	const std::string &	message,
		llvm::Instruction &	inst,
		GradientUtils *	gutils,
		llvm::IRBuilder<> &	B )

Definition at line 4675 of file Utils.cpp.

References TypeResults::analyzer, CustomErrorHandler, TypeResults::dump(), emit_backtrace(), EmitFailure(), EnzymeRuntimeError(), getString(), NoType, str(), and GradientUtils::TR.

Referenced by TypeResults::firstPointer(), TypeResults::intType(), AdjointGenerator::visitCastInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitExtractValueInst(), AdjointGenerator::visitInsertValueInst(), AdjointGenerator::visitLoadLike(), AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitMemTransferCommon().

EmitWarning() [1/3]

template<typename... Args>

void EmitWarning	(	llvm::StringRef	RemarkName,
		const llvm::DiagnosticLocation &	Loc,
		const llvm::BasicBlock *	BB,
		const Args &...	args )

Definition at line 252 of file Utils.h.

References EnzymePrintPerf, and str().

Referenced by EmitWarning(), CacheUtility::getContext(), getSparseConditions(), CacheUtility::getSubLimits(), AdjointGenerator::handleAdjointForIntrinsic(), TypeTree::Only(), ReadOnlyOrThrowDetector::run(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitCastInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitLoadLike(), AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitMemTransferCommon().

EmitWarning() [2/3]

template<typename... Args>

void EmitWarning	(	llvm::StringRef	RemarkName,
		const llvm::Function &	F,
		const Args &...	args )

Definition at line 321 of file Utils.h.

References EnzymePrintPerf, and str().

EmitWarning() [3/3]

template<typename... Args>

void EmitWarning	(	llvm::StringRef	RemarkName,
		const llvm::Instruction &	I,
		const Args &...	args )

Definition at line 271 of file Utils.h.

References EmitWarning().

EmitWarningAlways() [1/3]

template<typename... Args>

void EmitWarningAlways	(	llvm::StringRef	RemarkName,
		const llvm::DiagnosticLocation &	Loc,
		const llvm::Function &	F,
		const Args &...	args )

Emit a diagnostic which is always shown, unlike EmitWarning which requires the user to opt in with -Rpass=enzyme.

Use this for messages which describe a potential correctness problem (e.g. mismatched activity) as opposed to a performance note.

Definition at line 289 of file Utils.h.

References str().

Referenced by EmitWarningAlways(), EmitWarningAlways(), AdjointGenerator::handleKnownCallDerivatives(), and AdjointGenerator::visitCommonStore().

EmitWarningAlways() [2/3]

template<typename... Args>

void EmitWarningAlways	(	llvm::StringRef	RemarkName,
		const llvm::Function &	F,
		const Args &...	args )

Definition at line 308 of file Utils.h.

References EmitWarningAlways().

EmitWarningAlways() [3/3]

template<typename... Args>

void EmitWarningAlways	(	llvm::StringRef	RemarkName,
		const llvm::Instruction &	I,
		const Args &...	args )

Definition at line 301 of file Utils.h.

References EmitWarningAlways().

EmitWarningEnabled()

static bool EmitWarningEnabled ( const llvm::LLVMContext & Ctx )

inlinestatic

Definition at line 246 of file Utils.h.

References EnzymePrintPerf.

Referenced by AdjointGenerator::visitCommonStore(), and AdjointGenerator::visitLoadLike().

endsWith()

static bool endsWith ( llvm::StringRef string, llvm::StringRef suffix )

inlinestatic

Definition at line 855 of file Utils.h.

Referenced by calculateUnusedValues(), EnzymePlugin::EnzymePlugin(), isFortranMPICall(), and isMemFreeLibMFunction().

ErrorIfRuntimeInactive()

void ErrorIfRuntimeInactive	(	llvm::IRBuilder<> &	B,
		llvm::Value *	primal,
		llvm::Value *	shadow,
		const char *	Message,
		llvm::DebugLoc &&	loc,
		llvm::Instruction *	orig )

Definition at line 1031 of file Utils.cpp.

References addFunctionNoCapture(), CustomRuntimeInactiveError, emit_backtrace(), getInt8PtrTy(), getString(), and str().

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

evaluateAtIterationWithoutExt()

const llvm::SCEV * evaluateAtIterationWithoutExt	(	const llvm::SCEVAddRecExpr *	AR,
		const llvm::SCEV *	It,
		llvm::ScalarEvolution &	SE )

The value of the add recurrence AR at iteration It, like SCEVAddRecExpr::evaluateAtIteration, but without the widening that its BinomialCoefficient performs: LLVM computes I*(I-1)/2 in a type one bit wider than AR so that the halving is exact, and SCEVExpander then has to materialize a canonical induction variable of that wider type, a new loop-carried phi.

Here the halving is done on whichever of I and I-1 is even, in the wider of AR's and It's own types, so that every product is of exact integers and no new type appears. Recurrences of degree at most three are handled this way; higher degrees fall back to LLVM's version.

extractBLAS()

llvm::Optional< BlasInfo > extractBLAS

(

llvm::StringRef

in

)

Definition at line 3871 of file Utils.cpp.

References matchBLASName().

Referenced by AdjointGenerator::handleKnownCallDerivatives(), and TypeAnalyzer::visitCallBase().

findAllUsersOf()

llvm::SmallVector< std::tuple< llvm::Instruction *, llvm::Value *, size_t >, 1 > findAllUsersOf

(

llvm::Value *

AI

)

findInMap()

template<typename K , typename V >

static V * findInMap ( std::map< K, V > & map, K key )

inlinestatic

Definition at line 999 of file Utils.h.

Referenced by CacheUtility::erase(), and CacheUtility::getContext().

FloatToIntTy()

static llvm::Type * FloatToIntTy ( llvm::Type * T )

inlinestatic

Convert a floating point type to an integer type of the same size.

Definition at line 775 of file Utils.h.

References FloatToIntTy().

Referenced by FloatToIntTy().

get1ULP()

llvm::Value * get1ULP	(	llvm::IRBuilder<> &	builder,
		llvm::Value *	res )

Definition at line 4570 of file Utils.cpp.

get_blas_row() [1/2]

llvm::SmallVector< llvm::Value *, 1 > get_blas_row	(	llvm::IRBuilder<> &	B,
		llvm::ArrayRef< llvm::Value * >	trans,
		bool	byRef,
		bool	cublas )

get_blas_row() [2/2]

llvm::SmallVector< llvm::Value *, 1 > get_blas_row	(	llvm::IRBuilder<> &	B,
		llvm::ArrayRef< llvm::Value * >	trans,
		llvm::ArrayRef< llvm::Value * >	row,
		llvm::ArrayRef< llvm::Value * >	col,
		bool	byRef,
		bool	cublas )

get_cached_mat_width()

llvm::Value * get_cached_mat_width	(	llvm::IRBuilder<> &	B,
		llvm::ArrayRef< llvm::Value * >	trans,
		llvm::Value *	arg_ld,
		llvm::Value *	dim_1,
		llvm::Value *	dim_2,
		bool	cacheMat,
		bool	byRef,
		bool	cublas )

Definition at line 4313 of file Utils.cpp.

References CreateSelect(), and is_normal().

getAggregateElementOffset() [1/2]

static uint64_t getAggregateElementOffset ( const llvm::DataLayout & DL, llvm::Type * T, llvm::ArrayRef< unsigned > Idxs )

inlinestatic

The byte offset of the aggregate element addressed by Idxs within T, that is, what gep T, ptr, 0, Idxs... would compute.

The data layout already records this, so read it off directly rather than materializing a throwaway GEP and folding it: callers run this once per field of every aggregate they walk.

Definition at line 1354 of file Utils.h.

Referenced by defaultTypeTreeForLLVM(), getAggregateElementOffset(), getConstantAnalysis(), simplifyLoad(), TypeAnalyzer::visitCallBase(), TypeAnalyzer::visitExtractValueInst(), TypeAnalyzer::visitInsertValueInst(), and TypeAnalyzer::visitShuffleVectorInst().

getAggregateElementOffset() [2/2]

static uint64_t getAggregateElementOffset ( const llvm::DataLayout & DL, llvm::Type * T, uint64_t Idx )

inlinestatic

The byte offset of element Idx of the aggregate type T.

Definition at line 1375 of file Utils.h.

References getAggregateElementOffset().

getAllocationIndexFromCall()

static llvm::Optional< size_t > getAllocationIndexFromCall ( const llvm::CallBase * op )

inlinestatic

Definition at line 1519 of file Utils.h.

References getFunctionFromCall().

Referenced by TypeAnalyzer::visitCallBase(), and zeroKnownAllocation().

getAncestor()

static llvm::Loop * getAncestor ( llvm::Loop * R1, llvm::Loop * R2 )

inlinestatic

Definition at line 1197 of file Utils.h.

Referenced by overwritesToMemoryReadByLoop().

getBaseObject() [1/2]

static const llvm::Value * getBaseObject ( const llvm::Value * V )

inlinestatic

Definition at line 1842 of file Utils.h.

References getBaseObject().

getBaseObject() [2/2]

static llvm::Value * getBaseObject ( llvm::Value * V, bool offsetAllowed = true )

inlinestatic

Definition at line 1711 of file Utils.h.

References Call, getFuncNameFromCall(), getFunctionFromCall(), and isIntelSubscriptIntrinsic().

Referenced by PreProcessCache::AlwaysInline(), arePointersGuaranteedNoAlias(), EnzymeFixupJuliaCallingConvention(), fortranDescriptorTree(), getBaseObject(), getBaseObjects(), isAtomic(), moveSRetToFromRoots(), needsReRooting(), AdjointGenerator::recursivelyHandleSubfunction(), RecursivelyReplaceAddressSpace(), replaceToDense(), ReadOnlyOrThrowDetector::run(), AdjointGenerator::shadowStoreSchedule(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitOMPCall().

getBaseObjects()

static llvm::SetVector< llvm::Value * > getBaseObjects ( llvm::Value * V, bool offsetAllowed = true )

inlinestatic

Definition at line 1847 of file Utils.h.

References getBaseObject().

getBranchCondition()

static llvm::Value * getBranchCondition ( llvm::Value * V )

inlinestatic

The condition of a branch isConditionalBranch says yes to.

Definition at line 100 of file Utils.h.

Referenced by CanonicalizeLatches(), closedFormOfCountedAccumulator(), MustExitScalarEvolution::computeExitLimit(), fixSparse_inner(), fixSparseIndices(), and SelectOptimization().

getDeallocationIndicesFromCall()

template<typename T >

static std::vector< ssize_t > getDeallocationIndicesFromCall ( T * op )

inlinestatic

Definition at line 1582 of file Utils.h.

References getFunctionFromCall().

getDeallocatorFnFromCall()

template<typename T >

static llvm::Function * getDeallocatorFnFromCall ( T * op )

inlinestatic

Definition at line 1561 of file Utils.h.

References getFunctionFromCall(), and hasMetadata().

getDefaultAnonymousTapeType()

llvm::PointerType * getDefaultAnonymousTapeType

(

llvm::LLVMContext &

C

)

Definition at line 531 of file Utils.cpp.

References EnzymeDefaultTapeType, and getInt8PtrTy().

Referenced by getFunctionTypeForClone(), and traceType().

getFast()

llvm::FastMathFlags getFast

(

)

Get LLVM fast math flags.

Definition at line 4026 of file Utils.cpp.

References EnzymeFastMath().

Referenced by callSPMVDiagUpdate(), CacheUtility::createCacheForScope(), fixSparse_inner(), DiffeGradientUtils::freeCache(), DiffeGradientUtils::getDifferential(), getOrInsertDifferentialFloatMemcpy(), getorInsertInnerProd(), getOrInsertMemcpyStrided(), getOrInsertOpFloatSum(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), lowerSparsePlaceholders(), AdjointGenerator::recursivelyHandleSubfunction(), CacheUtility::storeInstructionInCache(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitIntrinsicInst(), and AdjointGenerator::visitOMPCall().

getFirstFunctionDefinition()

llvm::Function * getFirstFunctionDefinition

(

llvm::Module &

M

)

Referenced by EmitFailure().

getFirstNonPHI()

static llvm::Instruction * getFirstNonPHI ( llvm::BasicBlock * B )

inlinestatic

Definition at line 2550 of file Utils.h.

Referenced by CanonicalizeLatches(), fixSparse_inner(), fixSparseIndices(), RemoveRedundantIVs(), SimplifyMPIQueries(), and CacheUtility::storeInstructionInCache().

getFirstNonPHIOrDbg()

static llvm::Instruction * getFirstNonPHIOrDbg ( llvm::BasicBlock * B )

inlinestatic

Definition at line 2580 of file Utils.h.

Referenced by DynamicTraceInterface::DynamicTraceInterface(), FindCanonicalIV(), and InsertNewCanonicalIV().

getFirstNonPHIOrDbgOrLifetime()

static llvm::Instruction * getFirstNonPHIOrDbgOrLifetime ( llvm::BasicBlock * B )

inlinestatic

Definition at line 2591 of file Utils.h.

Referenced by TraceUtils::GetChoice(), and TraceGenerator::visitFunction().

getFuncName()

static llvm::StringRef getFuncName ( llvm::Function * called )

inlinestatic

Definition at line 1461 of file Utils.h.

Referenced by attributeKnownFunctions(), getFuncNameFromCall(), mayReadThroughCustomRule(), and SimplifyMPIQueries().

getFuncNameFromCall()

static llvm::StringRef getFuncNameFromCall ( const llvm::CallBase * op )

inlinestatic

Definition at line 1470 of file Utils.h.

References getFuncName(), and getFunctionFromCall().

Referenced by CanonicalizeGCPreserveEnds(), DetectPointerArgOfFn(), getBaseObject(), getOrInsertCheckedFree(), hasNoCache(), hasSingleGCPreserveEnd(), DifferentialUseAnalysis::is_value_needed_in_reverse(), isAllocationCall(), isDeallocationCall(), isIntelSubscriptIntrinsic(), isPointerArithmeticInst(), LowerSparsification(), AdjointGenerator::recursivelyHandleSubfunction(), replaceToDense(), ReadOnlyOrThrowDetector::run(), TypeAnalyzer::run(), shouldDisableNoWrite(), TypeAnalyzer::visitCallBase(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitMemSetCommon(), AdjointGenerator::visitOMPCall(), and writesToMemoryReadBy().

getFunctionFromCall()

template<typename T >

static llvm::Function * getFunctionFromCall ( T * op )

inlinestatic

Definition at line 1437 of file Utils.h.

Referenced by getAllocationIndexFromCall(), getBaseObject(), getDeallocationIndicesFromCall(), getDeallocatorFnFromCall(), getFuncNameFromCall(), hasNoCache(), isLocalReadOnlyOrThrow(), isNoAlias(), isNoCapture(), isNoEscapingAllocation(), isReadOnly(), isReadOnlyOrThrow(), isWriteOnly(), and shouldDisableNoWrite().

GetFunctionFromValue()

llvm::Function * GetFunctionFromValue

(

llvm::Value *

fn

)

getGPUSharedAddrSpace()

static unsigned getGPUSharedAddrSpace ( const llvm::Triple & TT )

inlinestatic

Definition at line 169 of file Utils.h.

getInt8PtrTy()

static llvm::PointerType * getInt8PtrTy ( llvm::LLVMContext & Context, unsigned AddressSpace = 0 )

inlinestatic

Definition at line 1324 of file Utils.h.

References getPointerType().

Referenced by addressType(), CreateDealloc(), AdjointGenerator::DifferentiableMemCopyFloats(), ErrorIfRuntimeInactive(), TraceUtils::GetChoice(), getDefaultAnonymousTapeType(), getMPIHelper(), getOrInsertDifferentialMPI_Wait(), getOrInsertOpFloatSum(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), TraceUtils::InsertFunction(), AdjointGenerator::MPI_COMM_RANK(), AdjointGenerator::MPI_COMM_SIZE(), AdjointGenerator::MPI_TYPE_SIZE(), PreProcessCache::preprocessForClone(), AdjointGenerator::recursivelyHandleSubfunction(), PreProcessCache::ReplaceReallocs(), SanitizeDerivatives(), TraceInterface::stringType(), to_blas_callconv(), AdjointGenerator::visitMemTransferCommon(), and zeroKnownAllocation().

getIntrinsicDeclaration()

static llvm::Function * getIntrinsicDeclaration ( llvm::Module * M, llvm::Intrinsic::ID id, llvm::ArrayRef< llvm::Type * > Tys = {} )

inlinestatic

Definition at line 2541 of file Utils.h.

Referenced by CreateAllocation(), getOrInsertExponentialAllocator(), AdjointGenerator::handleAdjointForIntrinsic(), AdjointGenerator::handleMPI(), PreProcessCache::preprocessForClone(), RecursivelyReplaceAddressSpace(), PreProcessCache::ReplaceReallocs(), and AdjointGenerator::visitCommonStore().

getJuliaObjects()

llvm::SmallVector< llvm::Value *, 1 > getJuliaObjects	(	llvm::Value *	v,
		llvm::IRBuilder<> &	B )

Definition at line 5347 of file Utils.cpp.

References anyJuliaObjects(), and isSpecialPtr().

Referenced by RecursivelyReplaceAddressSpace().

getMPIHelper()

static llvm::StructType * getMPIHelper ( llvm::LLVMContext & Context )

inlinestatic

Definition at line 1381 of file Utils.h.

References getInt8PtrTy().

Referenced by AdjointGenerator::handleMPI().

getMPIMemberPtr()

template<MPI_Elem E, bool Pointer = true>

static llvm::Value * getMPIMemberPtr ( llvm::IRBuilder<> & B, llvm::Value * V, llvm::Type * T )

inlinestatic

Definition at line 1398 of file Utils.h.

References Pointer.

Referenced by AdjointGenerator::handleMPI().

getNextNonDebugInstruction()

static llvm::Instruction * getNextNonDebugInstruction ( llvm::Instruction * Z )

inlinestatic

Get the next non-debug instruction, erring if none exists.

Definition at line 471 of file Utils.h.

References getNextNonDebugInstructionOrNull().

Referenced by CacheUtility::storeInstructionInCache().

getNextNonDebugInstructionOrNull()

static llvm::Instruction * getNextNonDebugInstructionOrNull ( llvm::Instruction * Z )

inlinestatic

Get the next non-debug instruction, if one exists.

Definition at line 462 of file Utils.h.

Referenced by getNextNonDebugInstruction().

getOrInsertCheckedFree()

llvm::Function * getOrInsertCheckedFree	(	llvm::Module &	M,
		llvm::CallInst *	call,
		llvm::Type *	Type,
		unsigned	width )

getOrInsertDifferentialFloatMemcpy()

llvm::Function * getOrInsertDifferentialFloatMemcpy	(	llvm::Module &	M,
		llvm::Type *	T,
		unsigned	dstalign,
		unsigned	srcalign,
		unsigned	dstaddr,
		unsigned	srcaddr,
		unsigned	bitwidth,
		bool	runtimeActivity,
		bool	atomic )

Create function for type that performs the derivative memcpy on floating point memory.

getOrInsertDifferentialFloatMemcpyMat()

llvm::Function * getOrInsertDifferentialFloatMemcpyMat	(	llvm::Module &	M,
		llvm::Type *	elementType,
		llvm::PointerType *	PT,
		llvm::IntegerType *	IT,
		llvm::IntegerType *	CT,
		unsigned	dstalign,
		unsigned	srcalign,
		bool	zeroSrc )

getOrInsertDifferentialFloatMemmove()

llvm::Function * getOrInsertDifferentialFloatMemmove	(	llvm::Module &	M,
		llvm::Type *	T,
		unsigned	dstalign,
		unsigned	srcalign,
		unsigned	dstaddr,
		unsigned	srcaddr,
		unsigned	bitwidth,
		bool	runtimeActivity,
		bool	atomic )

Create function for type that performs the derivative memmove on floating point memory.

getOrInsertDifferentialMPI_Wait()

llvm::Function * getOrInsertDifferentialMPI_Wait	(	llvm::Module &	M,
		llvm::ArrayRef< llvm::Type * >	T,
		llvm::Type *	reqType,
		llvm::StringRef	caller )

Create function for type that performs the derivative MPI_Wait.

getOrInsertDifferentialWaitallSave()

llvm::Function * getOrInsertDifferentialWaitallSave	(	llvm::Module &	M,
		llvm::ArrayRef< llvm::Type * >	T,
		llvm::PointerType *	reqType )

getorInsertInnerProd()

llvm::CallInst * getorInsertInnerProd	(	llvm::IRBuilder<> &	B,
		llvm::Module &	M,
		BlasInfo	blas,
		llvm::IntegerType *	IT,
		llvm::Type *	BlasPT,
		llvm::Type *	BlasIT,
		llvm::Type *	fpTy,
		llvm::ArrayRef< llvm::Value * >	args,
		const llvm::ArrayRef< llvm::OperandBundleDef >	bundles,
		bool	byRef,
		bool	cublas,
		bool	julia_decl )

getOrInsertMemcpyMat()

llvm::Function * getOrInsertMemcpyMat	(	llvm::Module &	M,
		llvm::Type *	elementType,
		llvm::PointerType *	PT,
		llvm::IntegerType *	IT,
		unsigned	dstalign,
		unsigned	srcalign )

Turned out to be a faster alternatives to lapacks lacpy function.

getOrInsertMemcpyStrided()

llvm::Function * getOrInsertMemcpyStrided	(	llvm::Module &	M,
		llvm::Type *	elementType,
		llvm::PointerType *	T,
		llvm::Type *	IT,
		unsigned	dstalign,
		unsigned	srcalign )

Create function for type that performs memcpy with a stride.

getOrInsertOpFloatSum()

llvm::Value * getOrInsertOpFloatSum	(	llvm::Module &	M,
		llvm::Function *	templateFn,
		llvm::Type *	OpPtr,
		llvm::Type *	OpType,
		ConcreteType	CT,
		llvm::Type *	intType,
		llvm::IRBuilder<> &	B2 )

templateFn is the MPI entry point this reduction is being built for; the MPI_Op_create it needs is named to match.

getOrInsertPerCallingConv()

llvm::FunctionCallee getOrInsertPerCallingConv	(	llvm::Module &	M,
		llvm::Function *	templateFn,
		llvm::StringRef	callee,
		llvm::FunctionType *	FT )

Declare callee in M under whatever naming convention the frontend used to reach templateFn, recording the plain name in enzyme_math so that the declaration is still recognized afterwards.

A helper introduced next to a call has to follow that call's convention to resolve at link time: Julia, for one, names its lazily bound ccalls "ejlstr$<function>$<library>" and loads those libraries RTLD_LOCAL, so a plainly named declaration would not be reachable via dlsym.

getPointerType()

static llvm::PointerType * getPointerType ( llvm::Type * T, unsigned AddressSpace = 0 )

inlinestatic

Pointer into the given address space.

From LLVM 17 on, and on LLVM 15/16 whenever the context has opaque pointers enabled, the element type is ignored – so callers with no element type to hand may pass any type belonging to the right context (see getInt8PtrTy).

Definition at line 1313 of file Utils.h.

Referenced by addValueToCache(), callSPMVDiagUpdate(), changePointerAddrSpace(), convertSRetTypeFromString(), CreateAllocation(), AdjointGenerator::DifferentiableMemCopyFloats(), EnzymeFixupJuliaCallingConvention(), getInt8PtrTy(), getOrInsertDifferentialFloatMemcpy(), getOrInsertExponentialAllocator(), getorInsertInnerProd(), getUnqual(), load_if_ref(), lookup_with_layout(), PreProcessCache::LowerAllocAddr(), moveSRetToFromRoots(), AdjointGenerator::recursivelyHandleSubfunction(), RecursivelyReplaceAddressSpace(), SimplifyMPIQueries(), and AdjointGenerator::visitOMPCall().

getRenamedPerCallingConv()

static std::string getRenamedPerCallingConv ( llvm::StringRef caller, llvm::StringRef callee )

inlinestatic

Definition at line 2812 of file Utils.h.

References getRenamedPerCallingConv(), isFortranMPICall(), startsWith(), and tripleSplitDollar().

Referenced by getOrInsertDifferentialMPI_Wait(), getOrInsertOpFloatSum(), getOrInsertPerCallingConv(), getRenamedPerCallingConv(), AdjointGenerator::handleMPI(), AdjointGenerator::MPI_COMM_RANK(), AdjointGenerator::MPI_COMM_SIZE(), and AdjointGenerator::MPI_TYPE_SIZE().

getSubType() [1/2]

static llvm::Type * getSubType ( llvm::Type * T )

inlinestatic

Definition at line 2633 of file Utils.h.

Referenced by getSubType().

getSubType() [2/2]

template<typename Arg1 , typename... Args>

static llvm::Type * getSubType ( llvm::Type * T, Arg1 i, Args... args )

inlinestatic

Definition at line 2636 of file Utils.h.

References getSubType().

getUndefinedValueForType()

llvm::Constant * getUndefinedValueForType	(	llvm::Module &	M,
		llvm::Type *	T,
		bool	forceZero = false )

Definition at line 3918 of file Utils.cpp.

References EnzymeUndefinedValueForType, and EnzymeZeroCache.

Referenced by copyNonJLValueInto(), CacheUtility::createCacheForScope(), AdjointGenerator::handleKnownCallDerivatives(), moveSRetToFromRoots(), and AdjointGenerator::recursivelyHandleSubfunction().

getUnqual()

static llvm::PointerType * getUnqual ( llvm::Type * T )

inlinestatic

Definition at line 1329 of file Utils.h.

References getPointerType().

Referenced by EnzymeFixupJuliaCallingConvention(), TraceUtils::FromClone(), getOrInsertDifferentialWaitallSave(), getOrInsertOpFloatSum(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), load_if_ref(), lookup_with_layout(), AdjointGenerator::MPI_COMM_RANK(), AdjointGenerator::MPI_COMM_SIZE(), AdjointGenerator::MPI_TYPE_SIZE(), needsReRooting(), AdjointGenerator::recursivelyHandleSubfunction(), SimplifyMPIQueries(), and AdjointGenerator::visitOMPCall().

hasCustomRuleMetadata()

template<typename T >

static bool hasCustomRuleMetadata ( const T * V )

inlinestatic

Definition at line 1626 of file Utils.h.

References hasMetadata().

Referenced by mayReadThroughCustomRule(), and shouldDisableNoWrite().

hasMetadata() [1/3]

static llvm::MDNode * hasMetadata ( const llvm::GlobalObject * O, llvm::StringRef kind )

inlinestatic

Check if a global has metadata.

Definition at line 483 of file Utils.h.

Referenced by PreProcessCache::AlwaysInline(), arePointersGuaranteedNoAlias(), CoaleseTrivialMallocs(), EnzymeHasFromStack(), fixSparse_inner(), getDeallocatorFnFromCall(), AdjointGenerator::handleKnownCallDerivatives(), hasCustomRuleMetadata(), hasNoCache(), PreProcessCache::LowerAllocAddr(), ReadOnlyOrThrowDetector::run(), TypeAnalyzer::visitCallBase(), AdjointGenerator::visitInsertValueInst(), AdjointGenerator::visitLoadLike(), AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitMemTransferCommon().

hasMetadata() [2/3]

static llvm::MDNode * hasMetadata ( const llvm::Instruction * O, llvm::StringRef kind )

inlinestatic

Check if an instruction has metadata.

Definition at line 489 of file Utils.h.

hasMetadata() [3/3]

static llvm::MDNode * hasMetadata ( const llvm::Instruction * O, unsigned kind )

inlinestatic

Definition at line 493 of file Utils.h.

hasNoCache()

static bool hasNoCache ( llvm::Value * op )

inlinestatic

Definition at line 1484 of file Utils.h.

References EnzymeJuliaAddrLoad, EnzymeNonPower2Cache, getFuncNameFromCall(), getFunctionFromCall(), and hasMetadata().

hasSRetRRootsOrUnionSRet()

static bool hasSRetRRootsOrUnionSRet ( llvm::CallBase * CB )

inlinestatic

Definition at line 2918 of file Utils.h.

hasTerminator()

static bool hasTerminator ( llvm::BasicBlock * BB )

static

Definition at line 2981 of file Utils.h.

Referenced by DiffeGradientUtils::freeCache().

insert_or_assign()

template<typename K , typename V >

static std::map< K, V >::iterator insert_or_assign ( std::map< K, V > & map, K & key, V && val )

inlinestatic

Insert into a map.

Definition at line 979 of file Utils.h.

insert_or_assign2()

template<typename K , typename V >

static std::map< K, V >::iterator insert_or_assign2 ( std::map< K, V > & map, K key, V val )

inlinestatic

Insert into a map.

Definition at line 990 of file Utils.h.

Referenced by AdjointGenerator::recursivelyHandleSubfunction(), CacheUtility::replaceAWithB(), AdjointGenerator::visitCallInst(), and AdjointGenerator::visitOMPCall().

insertBeforeInst()

static void insertBeforeInst ( llvm::Instruction * I, llvm::Instruction * Before )

inlinestatic

Definition at line 2562 of file Utils.h.

Referenced by getConstantAnalysis(), and TypeAnalyzer::visitConstantExpr().

IntToFloatTy()

static llvm::Type * IntToFloatTy ( llvm::Type * T )

inlinestatic

Convert a integer type to a floating point type of the same size.

Definition at line 799 of file Utils.h.

References IntToFloatTy().

Referenced by IntToFloatTy(), and AdjointGenerator::visitOMPCall().

is_left()

llvm::Value * is_left	(	llvm::IRBuilder<> &	B,
		llvm::Value *	side,
		bool	byRef,
		bool	cublas )

is_lower()

llvm::Value * is_lower	(	llvm::IRBuilder<> &	B,
		llvm::Value *	uplo,
		bool	byRef,
		bool	cublas )

is_nonunit()

llvm::Value * is_nonunit	(	llvm::IRBuilder<> &	B,
		llvm::Value *	uplo,
		bool	byRef,
		bool	cublas )

is_normal()

llvm::Value * is_normal	(	llvm::IRBuilder<> &	B,
		llvm::Value *	trans,
		bool	byRef,
		bool	cublas )

isAnyBranch()

static bool isAnyBranch ( const llvm::Value * V )

inlinestatic

Definition at line 57 of file Utils.h.

Referenced by MustExitScalarEvolution::computeExitLimit(), fixSparse_inner(), fixSparseIndices(), getExitBlocks(), DifferentialUseAnalysis::is_value_needed_in_reverse(), PreProcessCache::preprocessForClone(), and SelectOptimization().

isAtomic()

bool isAtomic	(	llvm::Value *	origptr,
		bool	AtomicAdd,
		llvm::Function *	newFunc )

isCalledFunction()

static llvm::Function * isCalledFunction ( llvm::Value * val )

inlinestatic

Definition at line 383 of file Utils.h.

isCertainPrint()

static bool isCertainPrint ( const llvm::StringRef name )

inlinestatic

Definition at line 863 of file Utils.h.

References startsWith().

Referenced by writesToMemoryReadBy().

isConditionalBranch()

static bool isConditionalBranch ( const llvm::Value * V )

inlinestatic

Definition at line 65 of file Utils.h.

Referenced by CanonicalizeLatches(), closedFormOfCountedAccumulator(), MustExitScalarEvolution::computeExitLimit(), fixSparse_inner(), fixSparseIndices(), and SelectOptimization().

isDebugFunction()

static bool isDebugFunction ( llvm::Function * called )

inlinestatic

Definition at line 824 of file Utils.h.

Referenced by ReadOnlyOrThrowDetector::run(), and writesToMemoryReadBy().

isFlangRuntimeNoFree()

bool isFlangRuntimeNoFree

(

llvm::StringRef

name

)

LLVM flang runtime functions that neither free memory of the program nor return or store an allocation into it: I/O (which frees only the I/O statement state that _FortranAioBegin* allocated and _FortranAioEndIoStatement releases), FLUSH, the scalar character inquiries and comparison, and MOD/MODULO.

Definition at line 115 of file Utils.cpp.

References startsWith().

Referenced by attributeKnownFunctions().

isFortranMPICall()

static bool isFortranMPICall ( llvm::StringRef Name )

inlinestatic

True if Name uses a Fortran ABI mangling of an MPI routine (trailing underscore(s), e.g.

"mpi_recv_", "mpi_comm_rank__"). Such calls pass all arguments by reference and take an extra trailing ierr argument.

Definition at line 2798 of file Utils.h.

References canonicalizeMPIName(), and endsWith().

Referenced by getRenamedPerCallingConv(), AdjointGenerator::handleMPI(), AdjointGenerator::MPI_COMM_RANK(), AdjointGenerator::MPI_COMM_SIZE(), AdjointGenerator::MPI_TYPE_SIZE(), and TypeAnalyzer::visitCallBase().

isGPUArch()

static bool isGPUArch ( const llvm::Triple & TT )

inlinestatic

Definition at line 162 of file Utils.h.

isIntelSubscriptIntrinsic() [1/2]

static bool isIntelSubscriptIntrinsic ( const llvm::IntrinsicInst & II )

inlinestatic

Definition at line 1649 of file Utils.h.

References getFuncNameFromCall(), and startsWith().

Referenced by analyzeIntelSubscriptIntrinsic(), getBaseObject(), DifferentialUseAnalysis::is_value_needed_in_reverse(), isIntelSubscriptIntrinsic(), isPointerArithmeticInst(), PreProcessCache::preprocessForClone(), RecursivelyReplaceAddressSpace(), and AdjointGenerator::visitIntrinsicInst().

isIntelSubscriptIntrinsic() [2/2]

static bool isIntelSubscriptIntrinsic ( const llvm::Value * val )

inlinestatic

Definition at line 1653 of file Utils.h.

References isIntelSubscriptIntrinsic().

isLocalReadOnlyOrThrow() [1/2]

static bool isLocalReadOnlyOrThrow ( const llvm::CallBase * call )

inlinestatic

Definition at line 1970 of file Utils.h.

References getFunctionFromCall(), isLocalReadOnlyOrThrow(), and isReadOnly().

isLocalReadOnlyOrThrow() [2/2]

static bool isLocalReadOnlyOrThrow ( const llvm::Function * F )

inlinestatic

Definition at line 1959 of file Utils.h.

References isReadOnly().

Referenced by isLocalReadOnlyOrThrow(), and ReadOnlyOrThrowDetector::run().

isNoAlias() [1/2]

static bool isNoAlias ( const llvm::CallBase * call )

inlinestatic

Definition at line 2127 of file Utils.h.

References getFunctionFromCall().

Referenced by arePointersGuaranteedNoAlias(), and isNoAlias().

isNoAlias() [2/2]

static bool isNoAlias ( const llvm::Value * val )

inlinestatic

Definition at line 2138 of file Utils.h.

References isNoAlias().

isNoCapture()

static bool isNoCapture ( const llvm::CallBase * call, size_t idx )

inlinestatic

Definition at line 2109 of file Utils.h.

References getFunctionFromCall().

Referenced by DetectPointerArgOfFn(), notCapturedBefore(), AdjointGenerator::recursivelyHandleSubfunction(), and mlir::enzyme::PointsToPointerAnalysis::visitCallControlFlowTransfer().

isNoEscapingAllocation() [1/2]

static bool isNoEscapingAllocation ( const llvm::CallBase * call )

inlinestatic

Definition at line 2259 of file Utils.h.

References getFunctionFromCall(), and isNoEscapingAllocation().

isNoEscapingAllocation() [2/2]

static bool isNoEscapingAllocation ( const llvm::Function * F )

inlinestatic

Definition at line 2156 of file Utils.h.

Referenced by isNoEscapingAllocation().

isNVLoad()

bool isNVLoad

(

const llvm::Value *

V

)

Definition at line 4821 of file Utils.cpp.

isPointerArithmeticInst()

static bool isPointerArithmeticInst ( const llvm::Value * V, bool includephi = true, bool includebin = true )

inlinestatic

Definition at line 1660 of file Utils.h.

References Call, getFuncNameFromCall(), and isIntelSubscriptIntrinsic().

Referenced by arePointersGuaranteedNoAlias(), DetectPointerArgOfFn(), and notCapturedBefore().

isReadNone() [1/2]

static bool isReadNone ( const llvm::CallBase * call, ssize_t arg = -1 )

inlinestatic

Definition at line 2101 of file Utils.h.

References isReadOnly(), and isWriteOnly().

isReadNone() [2/2]

static bool isReadNone ( const llvm::Function * F, ssize_t arg = -1 )

inlinestatic

Definition at line 2105 of file Utils.h.

References isReadOnly(), and isWriteOnly().

isReadOnly() [1/2]

static bool isReadOnly ( const llvm::CallBase * call, ssize_t arg = -1 )

inlinestatic

Definition at line 1915 of file Utils.h.

References getFunctionFromCall(), and isReadOnly().

isReadOnly() [2/2]

static bool isReadOnly ( const llvm::Function * F, ssize_t arg = -1 )

inlinestatic

Definition at line 1883 of file Utils.h.

Referenced by isLocalReadOnlyOrThrow(), isLocalReadOnlyOrThrow(), isReadNone(), isReadNone(), isReadOnly(), isReadOnlyOrThrow(), and isReadOnlyOrThrow().

isReadOnlyOrThrow() [1/2]

static bool isReadOnlyOrThrow ( const llvm::CallBase * call )

inlinestatic

Definition at line 2014 of file Utils.h.

References getFunctionFromCall(), isReadOnly(), and isReadOnlyOrThrow().

isReadOnlyOrThrow() [2/2]

static bool isReadOnlyOrThrow ( const llvm::Function * F )

inlinestatic

Definition at line 2004 of file Utils.h.

References isReadOnly().

Referenced by isReadOnlyOrThrow(), ReadOnlyOrThrowDetector::run(), and writesToMemoryReadBy().

isReturned()

static bool isReturned ( llvm::Instruction * inst )

inlinestatic

Check whether this instruction is returned.

Definition at line 765 of file Utils.h.

isSpecialPtr()

static bool isSpecialPtr ( llvm::Type * Ty )

inlinestatic

Definition at line 2664 of file Utils.h.

References FirstSpecial, and LastSpecial.

Referenced by anyJuliaObjects(), CountTrackedPointers::CountTrackedPointers(), getJuliaObjects(), moveSRetToFromRoots(), needsReRooting(), and AdjointGenerator::visitCallInst().

isUnconditionalBranch()

static bool isUnconditionalBranch ( const llvm::Value * V )

inlinestatic

Definition at line 75 of file Utils.h.

Referenced by fixSparseIndices().

isWriteOnly() [1/2]

static bool isWriteOnly ( const llvm::CallBase * call, ssize_t arg = -1 )

inlinestatic

Definition at line 2061 of file Utils.h.

References getFunctionFromCall(), and isWriteOnly().

isWriteOnly() [2/2]

static bool isWriteOnly ( const llvm::Function * F, ssize_t arg = -1 )

inlinestatic

Definition at line 2033 of file Utils.h.

Referenced by DetectPointerArgOfFn(), EnzymeFixupJuliaCallingConvention(), isReadNone(), isReadNone(), isWriteOnly(), and AdjointGenerator::recursivelyHandleSubfunction().

load_if_ref()

llvm::Value * load_if_ref	(	llvm::IRBuilder<> &	B,
		llvm::Type *	intType,
		llvm::Value *	V,
		bool	byRef )

Definition at line 4370 of file Utils.cpp.

References getPointerType(), and getUnqual().

Referenced by callSPMVDiagUpdate(), and getorInsertInnerProd().

lookup_with_layout()

llvm::Value * lookup_with_layout	(	llvm::IRBuilder<> &	B,
		llvm::Type *	fpType,
		llvm::Value *	layout,
		llvm::Value *	base,
		llvm::Value *	lda,
		llvm::Value *	row,
		llvm::Value *	col )

max()

template<typename T >

static T max ( T a, T b )

inlinestatic

Pick the maximum value.

Definition at line 406 of file Utils.h.

Referenced by TypeAnalyzer::considerTBAA(), TypeAnalyzer::visitCallBase(), and TypeAnalyzer::visitMemTransferCommon().

mayExecuteAfter()

void mayExecuteAfter	(	llvm::SmallVectorImpl< llvm::Instruction * > &	results,
		llvm::Instruction *	inst,
		const llvm::SetVector< llvm::Instruction * > &	stores,
		const llvm::Loop *	region )

min()

template<typename T >

static T min ( T a, T b )

inlinestatic

Pick the maximum value.

Definition at line 412 of file Utils.h.

moveBeforeInst()

static void moveBeforeInst ( llvm::Instruction * I, llvm::Instruction * Before )

inlinestatic

Definition at line 2571 of file Utils.h.

Referenced by EnzymeMoveBefore(), FindCanonicalIV(), and fixSparse_inner().

moveSRetToFromRoots()

llvm::Value * moveSRetToFromRoots	(	llvm::IRBuilder<> &	B,
		llvm::Type *	jltype,
		llvm::Value *	sret,
		llvm::Type *	root_ty,
		llvm::Value *	rootRet,
		size_t	rootOffset,
		SRetRootMovement	direction )

Definition at line 5148 of file Utils.cpp.

References constantInBoundsGEPHelper(), CountTrackedPointers::count, GradientUtils::extractMeta(), getBaseObject(), getPointerType(), getUndefinedValueForType(), isSpecialPtr(), NullifySRetValue, RootPointerToSRetPointer, RootPointerToSRetValue, SRetPointerToRootPointer, SRetValueToRootPointer, and Tracked.

Referenced by EnzymeFixupJuliaCallingConvention().

nextPowerOfTwo()

llvm::Value * nextPowerOfTwo	(	llvm::IRBuilder<> &	B,
		llvm::Value *	V )

Create function to computer nearest power of two.

Definition at line 2415 of file Utils.cpp.

Referenced by PreProcessCache::ReplaceReallocs().

notCaptured()

bool notCaptured	(	llvm::Value *	V,
		llvm::TargetLibraryInfo *	TLI = nullptr )

Check if value if b captured.

Definition at line 4956 of file Utils.cpp.

References notCapturedBefore().

notCapturedBefore()

bool notCapturedBefore	(	llvm::Value *	V,
		llvm::Instruction *	inst,
		size_t	checkLoadCaptured,
		llvm::Instruction *	startinst = nullptr,
		llvm::TargetLibraryInfo *	TLI = nullptr )

Check if value if b captured after definition before executing inst.

If checkLoadCaptured != 0, also consider catpures of any loads of the value as a capture (for the number of loads set). With TLI, passing the value to a deallocation function is not a capture.

operator<<() [1/4]

static llvm::raw_ostream & operator<< ( llvm::raw_ostream & os, DerivativeMode mode )

inlinestatic

Definition at line 592 of file Utils.h.

References to_string().

operator<<() [2/4]

static llvm::raw_ostream & operator<< ( llvm::raw_ostream & os, DIFFE_TYPE mode )

inlinestatic

Definition at line 614 of file Utils.h.

References to_string().

operator<<() [3/4]

static llvm::raw_ostream & operator<< ( llvm::raw_ostream & os, ReturnType mode )

inlinestatic

Definition at line 644 of file Utils.h.

References to_string().

operator<<() [4/4]

static llvm::raw_ostream & operator<< ( llvm::raw_ostream & os, ValueType mode )

inlinestatic

Definition at line 569 of file Utils.h.

References to_string().

overwritesToMemoryReadBy()

bool overwritesToMemoryReadBy	(	const TypeResults *	TR,
		llvm::AAResults &	AA,
		llvm::TargetLibraryInfo &	TLI,
		llvm::ScalarEvolution &	SE,
		llvm::LoopInfo &	LI,
		llvm::DominatorTree &	DT,
		llvm::Instruction *	maybeReader,
		llvm::Instruction *	maybeWriter,
		llvm::Loop *	scope = nullptr )

Definition at line 3040 of file Utils.cpp.

References arePointersGuaranteedNoAlias(), overwritesToMemoryReadByLoop(), and writesToMemoryReadBy().

parseTrueType()

std::vector< std::tuple< llvm::Type *, size_t, size_t > > parseTrueType	(	const llvm::MDNode *	md,
		DerivativeMode	Mode,
		bool	const_src )

Definition at line 4749 of file Utils.cpp.

References Anything, ForwardMode, and ForwardModeError.

Referenced by AdjointGenerator::visitInsertValueInst(), AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitMemTransferCommon().

PostCacheStore()

llvm::SmallVector< llvm::Instruction *, 2 > PostCacheStore	(	llvm::StoreInst *	SI,
		llvm::IRBuilder<> &	B )

Definition at line 517 of file Utils.cpp.

References EnzymePostCacheStore.

Referenced by CacheUtility::createCacheForScope(), EnzymeFixupJuliaCallingConvention(), and RecursivelyReplaceAddressSpace().

resetClonedGUID()

static void resetClonedGUID ( llvm::Function * F )

inlinestatic

Definition at line 91 of file Utils.h.

Referenced by PreProcessCache::CloneFunctionWithReturns(), EnzymeCloneFunctionWithoutReturnOrArgs(), EnzymeFixupBatchedJuliaCallingConvention(), EnzymeFixupJuliaCallingConvention(), TraceUtils::FromClone(), and PreProcessCache::preprocessForClone().

SanitizeDerivatives()

llvm::Value * SanitizeDerivatives	(	llvm::Value *	val,
		llvm::Value *	toset,
		llvm::IRBuilder<> &	BuilderM,
		llvm::Value *	mask = nullptr )

Definition at line 3929 of file Utils.cpp.

References CustomErrorHandler, emit_backtrace(), EnzymeCheckDerivativeNaN(), EnzymeSanitizeDerivatives, getInt8PtrTy(), getString(), NaNError, and str().

Referenced by DiffeGradientUtils::setDiffe().

setBranchCondition()

static void setBranchCondition ( llvm::Value * V, llvm::Value * Cond )

inlinestatic

Definition at line 108 of file Utils.h.

Referenced by fixSparseIndices().

shouldDisableNoWrite()

static bool shouldDisableNoWrite ( const llvm::CallInst * CI )

inlinestatic

Definition at line 1633 of file Utils.h.

References getFuncNameFromCall(), getFunctionFromCall(), and hasCustomRuleMetadata().

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

simplifyLoad()

llvm::Value * simplifyLoad	(	llvm::Value *	LI,
		size_t	valSz = 0,
		size_t	preOffset = 0 )

sliceTrueType()

llvm::MDNode * sliceTrueType	(	const llvm::MDNode *	md,
		size_t	start,
		size_t	length )

Restrict an enzyme_truetype annotation describing a memory region to the byte range [start, start + length), rebasing its offsets to that range.

Returns nullptr if no type information covers the range.

Definition at line 4713 of file Utils.cpp.

Referenced by AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitMemTransferCommon().

startsWith()

static bool startsWith ( llvm::StringRef string, llvm::StringRef prefix )

inlinestatic

Definition at line 847 of file Utils.h.

Referenced by copyForwardedCalleeAttrs(), ForceRecursiveInlining(), getConstantAnalysis(), getRenamedPerCallingConv(), handleCustomDerivative(), AdjointGenerator::handleKnownCallDerivatives(), isCertainPrint(), mlir::enzyme::ActivityAnalyzer::isConstantValue(), isFlangRuntimeNoFree(), isIntelSubscriptIntrinsic(), isItaniumEncoding(), isMemFreeLibMFunction(), isProduct(), isSum(), lowerSparsePlaceholders(), PreProcessCache::preprocessForClone(), preserveNVVM(), tripleSplitDollar(), TypeAnalyzer::visitCallBase(), AdjointGenerator::visitCallInst(), and AdjointGenerator::visitMemSetCommon().

to_blas_callconv()

llvm::Value * to_blas_callconv	(	llvm::IRBuilder<> &	B,
		llvm::Value *	V,
		bool	byRef,
		bool	cublas,
		llvm::IntegerType *	julia_decl,
		llvm::IRBuilder<> &	entryBuilder,
		llvm::Twine const &	= "" )

to_blas_fp_callconv()

llvm::Value * to_blas_fp_callconv	(	llvm::IRBuilder<> &	B,
		llvm::Value *	V,
		bool	byRef,
		llvm::Type *	julia_decl,
		llvm::IRBuilder<> &	entryBuilder,
		llvm::Twine const &	= "" )

to_string() [1/5]

template<typename T >

static std::string to_string ( const std::set< T > & us )

inlinestatic

Output a set as a string.

Definition at line 420 of file Utils.h.

Referenced by operator<<(), operator<<(), operator<<(), and operator<<().

to_string() [2/5]

static std::string to_string ( DerivativeMode mode )

inlinestatic

Definition at line 574 of file Utils.h.

References ForwardMode, ForwardModeError, ForwardModeSplit, ReverseModeCombined, ReverseModeGradient, and ReverseModePrimal.

to_string() [3/5]

static std::string to_string ( DIFFE_TYPE t )

inlinestatic

Convert DIFFE_TYPE to a string.

Definition at line 598 of file Utils.h.

References CONSTANT, DUP_ARG, DUP_NONEED, and OUT_DIFF.

to_string() [4/5]

static std::string to_string ( ReturnType t )

inlinestatic

Convert ReturnType to a string.

Definition at line 620 of file Utils.h.

References Args, ArgsWithReturn, ArgsWithTwoReturns, Return, Tape, TapeAndReturn, TapeAndTwoReturns, TwoReturns, and Void.

to_string() [5/5]

static std::string to_string ( ValueType mode )

inlinestatic

Definition at line 555 of file Utils.h.

References Both, None, Primal, and Shadow.

transpose() [1/2]

llvm::Value * transpose	(	std::string	floatType,
		llvm::IRBuilder<> &	B,
		llvm::Value *	V,
		bool	byRef,
		bool	cublas,
		llvm::IntegerType *	IT,
		llvm::IRBuilder<> &	entryBuilder,
		const llvm::Twine &	name )

Definition at line 4329 of file Utils.cpp.

References to_blas_callconv(), and transpose().

transpose() [2/2]

llvm::Value * transpose	(	std::string	floatType,
		llvm::IRBuilder<> &	B,
		llvm::Value *	V,
		bool	cublas )

tripleSplitDollar()

static std::tuple< llvm::StringRef, llvm::StringRef, llvm::StringRef > tripleSplitDollar ( llvm::StringRef caller )

inlinestatic

Definition at line 2803 of file Utils.h.

References startsWith().

Referenced by getOrInsertDifferentialMPI_Wait(), and getRenamedPerCallingConv().

whatType() [1/2]

static DIFFE_TYPE whatType ( llvm::Type * arg, DerivativeMode mode )

inlinestatic

Definition at line 759 of file Utils.h.

References whatType().

whatType() [2/2]

static DIFFE_TYPE whatType ( llvm::Type * arg, DerivativeMode mode, bool integersAreConstant, std::set< llvm::Type * > & seen )

inlinestatic

Attempt to automatically detect the differentiable classification based off of a given type.

Definition at line 653 of file Utils.h.

References CONSTANT, DUP_ARG, DUP_NONEED, ForwardMode, ForwardModeError, ForwardModeSplit, OUT_DIFF, and whatType().

Referenced by AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitOMPCall(), whatType(), and whatType().

writesToMemoryReadBy()

bool writesToMemoryReadBy	(	const TypeResults *	TR,
		llvm::AAResults &	AA,
		llvm::TargetLibraryInfo &	TLI,
		llvm::Instruction *	maybeReader,
		llvm::Instruction *	maybeWriter )

Return whether maybeReader can read from memory written to by maybeWriter.

Definition at line 3155 of file Utils.cpp.

References Anything, getFuncNameFromCall(), isAllocationFunction(), isCertainPrint(), isDeallocationFunction(), isDebugFunction(), ConcreteType::isKnown(), isMemFreeLibMFunction(), isReadOnlyOrThrow(), TypeTree::Lookup(), parseTBAA(), Pointer, TypeResults::query(), and Unknown.

Referenced by arePointersGuaranteedNoAlias(), and overwritesToMemoryReadBy().

ZeroMemory()

void ZeroMemory	(	llvm::IRBuilder<> &	Builder,
		llvm::Type *	T,
		llvm::Value *	obj,
		bool	isTape )

Definition at line 508 of file Utils.cpp.

References CustomZero.

Referenced by DiffeGradientUtils::getDifferential().

Variable Documentation

CustomErrorHandler

LLVMValueRef(* CustomErrorHandler) (const char *, LLVMValueRef, ErrorType, const void *, LLVMValueRef, LLVMBuilderRef) ( const char * , LLVMValueRef , ErrorType , const void * , LLVMValueRef , LLVMBuilderRef  )

extern

Definition at line 62 of file Utils.cpp.

Referenced by analyzeIntelSubscriptIntrinsic(), checkEnzymeTypeExtent(), EmitNoDerivativeError(), EmitNoDerivativeError(), EmitNoTypeError(), EnzymeFixupJuliaCallingConvention(), CacheUtility::erase(), TypeResults::firstPointer(), AdjointGenerator::handleKnownCallDerivatives(), TypeTree::insert(), needsReRooting(), TypeTree::Only(), AdjointGenerator::recursivelyHandleSubfunction(), RecursivelyReplaceAddressSpace(), SanitizeDerivatives(), transpose(), TypeAnalyzer::visitBinaryOperation(), AdjointGenerator::visitCommonStore(), TypeAnalyzer::visitGEPOperator(), TypeAnalyzer::visitIntrinsicInst(), TypeAnalyzer::visitMemTransferCommon(), and TypeAnalyzer::visitPHINode().

EnzymeBlasCopy

llvm::cl::opt<bool> EnzymeBlasCopy

extern

EnzymeJuliaAddrLoad

llvm::cl::opt<bool> EnzymeJuliaAddrLoad

extern

Referenced by TypeAnalyzer::getAnalysis(), AdjointGenerator::handleKnownCallDerivatives(), hasNoCache(), RecursivelyReplaceAddressSpace(), ReadOnlyOrThrowDetector::run(), AdjointGenerator::visitCallInst(), and AdjointGenerator::visitFenceInst().

EnzymeLapackCopy

llvm::cl::opt<bool> EnzymeLapackCopy

extern

EnzymeNonPower2Cache

llvm::cl::opt<bool> EnzymeNonPower2Cache

extern

Referenced by hasNoCache().

EnzymePrintPerf

llvm::cl::opt<bool> EnzymePrintPerf

extern

Print additional debug info relevant to performance.

Referenced by EmitWarning(), EmitWarning(), EmitWarningEnabled(), and ReadOnlyOrThrowDetector::run().

KnownMPIFunctions

const std::pair<const char *, const char *> KnownMPIFunctions[]

static

MPI routines recognized by Enzyme, keyed by their lowercase, underscore-free base name and mapped to the canonical C name (without the profiling "P" prefix).

Definition at line 2720 of file Utils.h.

Referenced by canonicalizeMPIName().

MixedActivityHint

const char* MixedActivityHint

staticconstexpr

Initial value:

=
    ". If this value may be active at runtime, enable runtime activity "
    "analysis (pass \"enzyme_runtime_activity\" to __enzyme_autodiff); "
    "otherwise its derivative will be assumed zero."

Remedy appended to mismatched-activity warnings, which are emitted when a value Enzyme proved inactive is used where an active value may be required.

Definition at line 315 of file Utils.h.

Referenced by AdjointGenerator::handleKnownCallDerivatives(), and AdjointGenerator::visitCommonStore().

PrimalParamAttrsToPreserve

llvm::Attribute::AttrKind PrimalParamAttrsToPreserve[]

inlinestatic

Initial value:

= {
  llvm::Attribute::AttrKind::ReadOnly,
  llvm::Attribute::AttrKind::WriteOnly,
  llvm::Attribute::AttrKind::ZExt,
  llvm::Attribute::AttrKind::SExt,
  llvm::Attribute::AttrKind::InReg,
  llvm::Attribute::AttrKind::ByVal,
 
 
 
  llvm::Attribute::AttrKind::Preallocated,
  llvm::Attribute::AttrKind::InAlloca,
 
 
 
 
 
 
  llvm::Attribute::AttrKind::NoFree,
  llvm::Attribute::AttrKind::Alignment,
  llvm::Attribute::AttrKind::StackAlignment,
 
 
 
  llvm::Attribute::AttrKind::NoCapture,
 
  llvm::Attribute::AttrKind::ReadNone
}

Definition at line 2486 of file Utils.h.

Referenced by PreProcessCache::CloneFunctionWithReturns(), and AdjointGenerator::recursivelyHandleSubfunction().

shadowHandlers

llvm::StringMap<std::function<llvm::Value *( llvm::IRBuilder<> &, llvm::CallInst *, llvm::ArrayRef<llvm::Value *>, GradientUtils *)> > shadowHandlers

extern

Definition at line 71 of file GradientUtils.cpp.

Referenced by EnzymeRegisterAllocationHandler(), AdjointGenerator::handleKnownCallDerivatives(), and TypeAnalyzer::visitCallBase().

ShadowParamAttrsToPreserve

llvm::Attribute::AttrKind ShadowParamAttrsToPreserve[]

inlinestatic

Initial value:

= {
  llvm::Attribute::AttrKind::ZExt,
  llvm::Attribute::AttrKind::SExt,
 
 
 
  llvm::Attribute::AttrKind::NoFree,
  llvm::Attribute::AttrKind::Alignment,
  llvm::Attribute::AttrKind::StackAlignment,
 
 
 
  llvm::Attribute::AttrKind::NoCapture,
 
  llvm::Attribute::AttrKind::ReadNone,
}

Definition at line 2518 of file Utils.h.

Referenced by PreProcessCache::CloneFunctionWithReturns(), and AdjointGenerator::recursivelyHandleSubfunction().