Utils.cpp File Reference

#include "Utils.h"
#include "GradientUtils.h"
#include "TypeAnalysis/TypeAnalysis.h"
#include "SCEV/ScalarEvolution.h"
#include "SCEV/ScalarEvolutionExpander.h"
#include "TypeAnalysis/TBAA.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Verifier.h"
#include "llvm/ADT/Triple.h"
#include "llvm-c/Core.h"
#include "BlasAttributor.inc"
#include "LibraryFuncs.h"

Include dependency graph for Utils.cpp:

Go to the source code of this file.

Macros
#define	addAttribute   addAttributeAtIndex
#define	getAttribute   getAttributeAtIndex

Functions
llvm::cl::opt< bool >	EnzymeLapackCopy ("enzyme-lapack-copy", cl::init(false), cl::Hidden, cl::desc("Use blas copy calls to cache matrices"))
llvm::cl::opt< bool >	EnzymeBlasCopy ("enzyme-blas-copy", cl::init(true), cl::Hidden, cl::desc("Use blas copy calls to cache vectors"))
llvm::cl::opt< bool >	EnzymeFastMath ("enzyme-fast-math", cl::init(true), cl::Hidden, cl::desc("Use fast math on derivative compuation"))
llvm::cl::opt< bool >	EnzymeMemmoveWarning ("enzyme-memmove-warning", cl::init(true), cl::Hidden, cl::desc("Warn if using memmove implementation as a fallback for memmove"))
llvm::cl::opt< bool >	EnzymeRuntimeError ("enzyme-runtime-error", cl::init(false), cl::Hidden, cl::desc("Emit Runtime errors instead of compile time ones"))
llvm::cl::opt< bool >	EnzymeCheckDerivativeNaN ("enzyme-check-nan", cl::init(false), cl::Hidden, cl::desc("Add NaN checks to all derivative intermediate values"))
llvm::cl::opt< bool >	EnzymeNonPower2Cache ("enzyme-non-power2-cache", cl::init(false), cl::Hidden, cl::desc("Disable caching of integers which are not a power of 2"))
bool	isFlangRuntimeNoFree (llvm::StringRef name)
	LLVM flang runtime functions that neither free memory of the program nor return or store an allocation into it: I/O (which frees only the I/O statement state that _FortranAioBegin* allocated and _FortranAioEndIoStatement releases), FLUSH, the scalar character inquiries and comparison, and MOD/MODULO.
bool	attributeKnownFunctions (llvm::Function &F)
void	ZeroMemory (llvm::IRBuilder<> &Builder, llvm::Type *T, llvm::Value *obj, bool isTape)
llvm::SmallVector< llvm::Instruction *, 2 >	PostCacheStore (llvm::StoreInst *SI, llvm::IRBuilder<> &B)
llvm::PointerType *	getDefaultAnonymousTapeType (llvm::LLVMContext &C)
Function *	getOrInsertExponentialAllocator (Module &M, Function *newFunc, bool ZeroInit, llvm::Type *RT)
llvm::Value *	CreateReAllocation (llvm::IRBuilder<> &B, llvm::Value *prev, llvm::Type *T, llvm::Value *OuterCount, llvm::Value *InnerCount, const llvm::Twine &Name, llvm::CallInst **caller, bool ZeroMem)
Value *	CreateAllocation (IRBuilder<> &Builder, llvm::Type *T, Value *Count, const Twine &Name, CallInst **caller, Instruction **ZeroMem, bool isDefault)
CallInst *	CreateDealloc (llvm::IRBuilder<> &Builder, llvm::Value *ToFree)
static std::string	tofltstr (Type *T)
	Convert a floating type to a string.
Constant *	getString (Module &M, StringRef Str)
void	emit_backtrace (llvm::Instruction *inst, llvm::raw_ostream &ss)
void	ErrorIfRuntimeInactive (llvm::IRBuilder<> &B, llvm::Value *primal, llvm::Value *shadow, const char *Message, llvm::DebugLoc &&loc, llvm::Instruction *orig)
bool	isAtomic (Value *origptr, bool AtomicAdd, Function *newFunc)
Function *	getOrInsertDifferentialFloatMemcpy (Module &M, Type *elementType, unsigned dstalign, unsigned srcalign, unsigned dstaddr, unsigned srcaddr, unsigned bitwidth, bool runtimeActivity, bool atomic)
	Create function for type that is equivalent to memcpy but adds to destination rather than a direct copy; dst, src, numelems.
Value *	lookup_with_layout (IRBuilder<> &B, Type *fpType, Value *layout, Value *const base, Value *lda, Value *row, Value *col)
void	copy_lower_to_upper (llvm::IRBuilder<> &B, llvm::Type *fpType, BlasInfo blas, bool byRef, llvm::Value *layout, llvm::Value *islower, llvm::Value *A, llvm::Value *lda, llvm::Value *N)
void	callMemcpyStridedBlas (llvm::IRBuilder<> &B, llvm::Module &M, BlasInfo blas, llvm::ArrayRef< llvm::Value * > args, llvm::Type *copy_retty, llvm::ArrayRef< llvm::OperandBundleDef > bundles)
	Create function for type that performs memcpy with a stride using blas copy.
void	callMemcpyStridedLapack (llvm::IRBuilder<> &B, llvm::Module &M, BlasInfo blas, llvm::ArrayRef< llvm::Value * > args, llvm::ArrayRef< llvm::OperandBundleDef > bundles)
	Create function for type that performs memcpy using lapack copy.
void	callSPMVDiagUpdate (IRBuilder<> &B, Module &M, BlasInfo blas, IntegerType *IT, Type *BlasCT, Type *BlasFPT, Type *BlasPT, Type *BlasIT, Type *fpTy, ArrayRef< Value * > args, ArrayRef< OperandBundleDef > bundles, bool byRef, bool julia_decl)
llvm::CallInst *	getorInsertInnerProd (llvm::IRBuilder<> &B, llvm::Module &M, BlasInfo blas, IntegerType *IT, Type *BlasPT, Type *BlasIT, Type *fpTy, llvm::ArrayRef< llvm::Value * > args, const llvm::ArrayRef< llvm::OperandBundleDef > bundles, bool byRef, bool cublas, bool julia_decl)
Function *	getOrInsertMemcpyStrided (Module &M, Type *elementType, PointerType *T, Type *IT, unsigned dstalign, unsigned srcalign)
Function *	getOrInsertMemcpyMat (Module &Mod, Type *elementType, PointerType *PT, IntegerType *IT, unsigned dstalign, unsigned srcalign)
Function *	getOrInsertDifferentialFloatMemcpyMat (Module &Mod, Type *elementType, PointerType *PT, IntegerType *IT, IntegerType *CT, unsigned dstalign, unsigned srcalign, bool zeroSrc)
Function *	getOrInsertDifferentialFloatMemmove (Module &M, Type *T, unsigned dstalign, unsigned srcalign, unsigned dstaddr, unsigned srcaddr, unsigned bitwidth, bool runtimeActivity, bool atomic)
FunctionCallee	getOrInsertPerCallingConv (Module &M, Function *templateFn, StringRef callee, FunctionType *FT)
Function *	getOrInsertCheckedFree (Module &M, CallInst *call, Type *Ty, unsigned width)
llvm::Value *	nextPowerOfTwo (llvm::IRBuilder<> &B, llvm::Value *V)
	Create function to computer nearest power of two.
llvm::Function *	getOrInsertDifferentialWaitallSave (llvm::Module &M, ArrayRef< llvm::Type * > T, PointerType *reqType)
llvm::Function *	getOrInsertDifferentialMPI_Wait (llvm::Module &M, ArrayRef< llvm::Type * > T, Type *reqType, StringRef caller)
llvm::Value *	getOrInsertOpFloatSum (llvm::Module &M, llvm::Function *templateFn, llvm::Type *OpPtr, llvm::Type *OpType, ConcreteType CT, llvm::Type *intType, IRBuilder<> &B2)
void	mayExecuteAfter (llvm::SmallVectorImpl< llvm::Instruction * > &results, llvm::Instruction *inst, const llvm::SetVector< Instruction * > &stores, const llvm::Loop *region)
const SCEV *	evaluateAtIterationWithoutExt (const SCEVAddRecExpr *AR, const SCEV *It, ScalarEvolution &SE)
bool	overwritesToMemoryReadByLoop (llvm::ScalarEvolution &SE, llvm::LoopInfo &LI, llvm::DominatorTree &DT, llvm::Instruction *maybeReader, const llvm::SCEV *LoadStart, const llvm::SCEV *LoadEnd, llvm::Instruction *maybeWriter, const llvm::SCEV *StoreStart, const llvm::SCEV *StoreEnd, llvm::Loop *scope)
bool	overwritesToMemoryReadBy (const TypeResults *TR, llvm::AAResults &AA, llvm::TargetLibraryInfo &TLI, ScalarEvolution &SE, llvm::LoopInfo &LI, llvm::DominatorTree &DT, llvm::Instruction *maybeReader, llvm::Instruction *maybeWriter, llvm::Loop *scope)
bool	writesToMemoryReadBy (const TypeResults *TR, llvm::AAResults &AA, llvm::TargetLibraryInfo &TLI, llvm::Instruction *maybeReader, llvm::Instruction *maybeWriter)
	Return whether maybeReader can read from memory written to by maybeWriter.
AllocaInst *	getBaseAndOffset (Value *ptr, size_t &offset)
SmallVector< std::tuple< Instruction *, Value *, size_t >, 1 >	findAllUsersOf (Value *AI)
SmallVector< std::pair< Value *, size_t >, 1 >	getAllLoadedValuesFrom (AllocaInst *ptr0, size_t offset, size_t valSz, bool &legal)
Value *	simplifyLoad (Value *V, size_t valSz, size_t preOffset)
Value *	GetFunctionValFromValue (Value *fn)
Function *	GetFunctionFromValue (Value *fn)
Function *	getFirstFunctionDefinition (Module &M)
static bool	matchBLASName (StringRef in, ArrayRef< const char * > prefixes, ArrayRef< const char * > floatTypes, ArrayRef< const char * > stems, ArrayRef< const char * > suffixes, const char *&prefixOut, const char *&floatTypeOut, const char *&stemOut, const char *&suffixOut)
	Decompose in as prefix + floatType + stem + suffix using the supplied tables, setting the out-parameters to the entries that matched.
llvm::Optional< BlasInfo >	extractBLAS (llvm::StringRef in)
llvm::Constant *	getUndefinedValueForType (llvm::Module &M, llvm::Type *T, bool forceZero)
llvm::Value *	SanitizeDerivatives (llvm::Value *val, llvm::Value *toset, llvm::IRBuilder<> &BuilderM, llvm::Value *mask)
llvm::FastMathFlags	getFast ()
	Get LLVM fast math flags.
void	addValueToCache (llvm::Value *arg, bool cache_arg, llvm::Type *ty, llvm::SmallVectorImpl< llvm::Value * > &cacheValues, llvm::IRBuilder<> &BuilderZ, const Twine &name)
llvm::Value *	to_blas_callconv (IRBuilder<> &B, llvm::Value *V, bool byRef, bool cublas, IntegerType *julia_decl, IRBuilder<> &entryBuilder, llvm::Twine const &name)
llvm::Value *	to_blas_fp_callconv (IRBuilder<> &B, llvm::Value *V, bool byRef, Type *fpTy, IRBuilder<> &entryBuilder, llvm::Twine const &name)
Value *	is_lower (IRBuilder<> &B, Value *uplo, bool byRef, bool cublas)
Value *	is_nonunit (IRBuilder<> &B, Value *uplo, bool byRef, bool cublas)
llvm::Value *	is_normal (IRBuilder<> &B, llvm::Value *trans, bool byRef, bool cublas)
llvm::Value *	is_left (IRBuilder<> &B, llvm::Value *side, bool byRef, bool cublas)
llvm::Value *	transpose (std::string floatType, IRBuilder<> &B, llvm::Value *V, bool cublas)
llvm::Value *	get_cached_mat_width (llvm::IRBuilder<> &B, llvm::ArrayRef< llvm::Value * > trans, llvm::Value *arg_ld, llvm::Value *dim1, llvm::Value *dim2, bool cacheMat, bool byRef, bool cublas)
llvm::Value *	transpose (std::string floatType, llvm::IRBuilder<> &B, llvm::Value *V, bool byRef, bool cublas, llvm::IntegerType *julia_decl, llvm::IRBuilder<> &entryBuilder, const llvm::Twine &name)
llvm::Value *	load_if_ref (llvm::IRBuilder<> &B, llvm::Type *intType, llvm::Value *V, bool byRef)
SmallVector< llvm::Value *, 1 >	get_blas_row (llvm::IRBuilder<> &B, ArrayRef< llvm::Value * > transA, bool byRef, bool cublas)
SmallVector< llvm::Value *, 1 >	get_blas_row (llvm::IRBuilder<> &B, ArrayRef< llvm::Value * > transA, ArrayRef< llvm::Value * > row, ArrayRef< llvm::Value * > col, bool byRef, bool cublas)
bool	collectOffset (GEPOperator *gep, const DataLayout &DL, unsigned BitWidth, MapVector< Value *, APInt > &VariableOffsets, APInt &ConstantOffset)
llvm::CallInst *	createIntrinsicCall (llvm::IRBuilderBase &B, llvm::Intrinsic::ID ID, llvm::Type *RetTy, llvm::ArrayRef< llvm::Value * > Args, llvm::Instruction *FMFSource, const llvm::Twine &Name)
llvm::Value *	get1ULP (llvm::IRBuilder<> &builder, llvm::Value *res)
llvm::Value *	EmitNoDerivativeError (const std::string &message, llvm::Instruction &inst, GradientUtils *gutils, llvm::IRBuilder<> &Builder2, llvm::Value *condition)
bool	EmitNoDerivativeError (const std::string &message, Value *todiff, RequestContext &context)
void	EmitNoTypeError (const std::string &message, llvm::Instruction &inst, GradientUtils *gutils, llvm::IRBuilder<> &Builder2)
llvm::MDNode *	sliceTrueType (const llvm::MDNode *md, size_t start, size_t length)
	Restrict an enzyme_truetype annotation describing a memory region to the byte range [start, start + length), rebasing its offsets to that range.
std::vector< std::tuple< llvm::Type *, size_t, size_t > >	parseTrueType (const llvm::MDNode *md, DerivativeMode Mode, bool const_src)
void	dumpModule (llvm::Module *mod)
void	dumpValue (llvm::Value *val)
void	dumpBlock (llvm::BasicBlock *blk)
void	dumpType (llvm::Type *ty)
void	dumpTypeResults (TypeResults &TR)
bool	isNVLoad (const llvm::Value *V)
bool	notCapturedBefore (llvm::Value *V, Instruction *inst, size_t checkLoadCaptures, Instruction *startinst, llvm::TargetLibraryInfo *TLI)
bool	notCaptured (llvm::Value *V, llvm::TargetLibraryInfo *TLI)
	Check if value if b captured.
llvm::Optional< bool >	arePointersGuaranteedNoAlias (TargetLibraryInfo &TLI, llvm::AAResults &AA, llvm::DominatorTree &DT, llvm::LoopInfo &LI, llvm::Value *op0, llvm::Value *op1, bool offsetAllowed)
static Value *	constantInBoundsGEPHelper (llvm::IRBuilder<> &B, llvm::Type *type, llvm::Value *value, ArrayRef< unsigned > path)
llvm::Value *	moveSRetToFromRoots (llvm::IRBuilder<> &B, llvm::Type *jltype, llvm::Value *sret, llvm::Type *root_ty, llvm::Value *rootRet, size_t rootOffset, SRetRootMovement direction)
void	copyNonJLValueInto (llvm::IRBuilder<> &B, llvm::Type *curType, llvm::Type *dstType, llvm::Value *dst, llvm::ArrayRef< unsigned > dstPrefix0, llvm::Type *srcType, llvm::Value *src, llvm::ArrayRef< unsigned > srcPrefix0, bool shouldZero)
llvm::SmallVector< llvm::Value *, 1 >	getJuliaObjects (llvm::Value *v, llvm::IRBuilder<> &B)

Variables
LLVMValueRef(*	CustomErrorHandler )(const char *, LLVMValueRef, ErrorType, const void *, LLVMValueRef, LLVMBuilderRef) = nullptr
LLVMValueRef(*	CustomAllocator )(LLVMBuilderRef, LLVMTypeRef, LLVMValueRef, LLVMValueRef, uint8_t, LLVMValueRef *) = nullptr
void(*	CustomZero )(LLVMBuilderRef, LLVMTypeRef, LLVMValueRef, uint8_t) = nullptr
LLVMValueRef(*	CustomDeallocator )(LLVMBuilderRef, LLVMValueRef) = nullptr
void(*	CustomRuntimeInactiveError )(LLVMBuilderRef, LLVMValueRef, LLVMValueRef) = nullptr
LLVMValueRef *(*	EnzymePostCacheStore )(LLVMValueRef, LLVMBuilderRef, uint64_t *size) = nullptr
LLVMTypeRef(*	EnzymeDefaultTapeType )(LLVMContextRef) = nullptr
LLVMValueRef(*	EnzymeUndefinedValueForType )(LLVMModuleRef, LLVMTypeRef, uint8_t) = nullptr
LLVMValueRef(*	EnzymeSanitizeDerivatives )(LLVMValueRef, LLVMValueRef toset, LLVMBuilderRef, LLVMValueRef) = nullptr
llvm::cl::opt< bool >	EnzymeZeroCache

Macro Definition Documentation

addAttribute

#define addAttribute   addAttributeAtIndex

Definition at line 112 of file Utils.cpp.

getAttribute

#define getAttribute   getAttributeAtIndex

Definition at line 113 of file Utils.cpp.

Function Documentation

addValueToCache()

void addValueToCache	(	llvm::Value *	arg,
		bool	cache_arg,
		llvm::Type *	ty,
		llvm::SmallVectorImpl< llvm::Value * > &	cacheValues,
		llvm::IRBuilder<> &	BuilderZ,
		const Twine &	name )

Definition at line 4033 of file Utils.cpp.

References getPointerType().

arePointersGuaranteedNoAlias()

llvm::Optional< bool > arePointersGuaranteedNoAlias	(	TargetLibraryInfo &	TLI,
		llvm::AAResults &	AA,
		llvm::DominatorTree &	DT,
		llvm::LoopInfo &	LI,
		llvm::Value *	op0,
		llvm::Value *	op1,
		bool	offsetAllowed )

Definition at line 4968 of file Utils.cpp.

References allInstructionsBetween(), getBaseObject(), hasMetadata(), isAllocationCall(), isNoAlias(), isPointerArithmeticInst(), notCapturedBefore(), and writesToMemoryReadBy().

Referenced by overwritesToMemoryReadBy().

attributeKnownFunctions()

bool attributeKnownFunctions

(

llvm::Function &

F

)

Definition at line 139 of file Utils.cpp.

References addFunctionNoCapture(), canonicalizeMPIName(), getFuncName(), and isFlangRuntimeNoFree().

Referenced by callMemcpyStridedBlas(), callMemcpyStridedLapack(), copy_lower_to_upper(), EnzymeAttributeKnownFunctions(), getorInsertInnerProd(), and preserveNVVM().

callMemcpyStridedBlas()

void callMemcpyStridedBlas	(	llvm::IRBuilder<> &	B,
		llvm::Module &	M,
		BlasInfo	blas,
		llvm::ArrayRef< llvm::Value * >	args,
		llvm::Type *	copy_retty,
		llvm::ArrayRef< llvm::OperandBundleDef >	bundles )

Create function for type that performs memcpy with a stride using blas copy.

Definition at line 1526 of file Utils.cpp.

References attributeKnownFunctions(), BlasInfo::floatType, GetFunctionFromValue(), BlasInfo::prefix, and BlasInfo::suffix.

callMemcpyStridedLapack()

void callMemcpyStridedLapack	(	llvm::IRBuilder<> &	B,
		llvm::Module &	M,
		BlasInfo	blas,
		llvm::ArrayRef< llvm::Value * >	args,
		llvm::ArrayRef< llvm::OperandBundleDef >	bundles )

Create function for type that performs memcpy using lapack copy.

Definition at line 1546 of file Utils.cpp.

References attributeKnownFunctions(), BlasInfo::floatType, GetFunctionFromValue(), BlasInfo::prefix, and BlasInfo::suffix.

callSPMVDiagUpdate()

void callSPMVDiagUpdate	(	IRBuilder<> &	B,
		Module &	M,
		BlasInfo	blas,
		IntegerType *	IT,
		Type *	BlasCT,
		Type *	BlasFPT,
		Type *	BlasPT,
		Type *	BlasIT,
		Type *	fpTy,
		ArrayRef< Value * >	args,
		ArrayRef< OperandBundleDef >	bundles,
		bool	byRef,
		bool	julia_decl )

Definition at line 1564 of file Utils.cpp.

References addFunctionNoCapture(), BlasInfo::floatType, getFast(), getPointerType(), is_lower(), load_if_ref(), and BlasInfo::suffix.

collectOffset()

bool collectOffset	(	GEPOperator *	gep,
		const DataLayout &	DL,
		unsigned	BitWidth,
		MapVector< Value *, APInt > &	VariableOffsets,
		APInt &	ConstantOffset )

Definition at line 4464 of file Utils.cpp.

Referenced by EnzymeComputeByteOffsetOfGEP(), findAllUsersOf(), getBaseAndOffset(), needsReRooting(), and TypeAnalyzer::visitGEPOperator().

constantInBoundsGEPHelper()

static Value * constantInBoundsGEPHelper ( llvm::IRBuilder<> & B, llvm::Type * type, llvm::Value * value, ArrayRef< unsigned > path )

static

Definition at line 5137 of file Utils.cpp.

Referenced by copyNonJLValueInto(), and moveSRetToFromRoots().

copy_lower_to_upper()

void copy_lower_to_upper	(	llvm::IRBuilder<> &	B,
		llvm::Type *	fpType,
		BlasInfo	blas,
		bool	byRef,
		llvm::Value *	layout,
		llvm::Value *	islower,
		llvm::Value *	A,
		llvm::Value *	lda,
		llvm::Value *	N )

Definition at line 1382 of file Utils.cpp.

References addFunctionNoCapture(), attributeKnownFunctions(), CreateSelect(), BlasInfo::floatType, getFunctionFromCall(), GetFunctionFromValue(), lookup_with_layout(), BlasInfo::prefix, BlasInfo::suffix, and to_blas_callconv().

copyNonJLValueInto()

void copyNonJLValueInto	(	llvm::IRBuilder<> &	B,
		llvm::Type *	curType,
		llvm::Type *	dstType,
		llvm::Value *	dst,
		llvm::ArrayRef< unsigned >	dstPrefix0,
		llvm::Type *	srcType,
		llvm::Value *	src,
		llvm::ArrayRef< unsigned >	srcPrefix0,
		bool	shouldZero )

Definition at line 5271 of file Utils.cpp.

References constantInBoundsGEPHelper(), CountTrackedPointers::count, and getUndefinedValueForType().

Referenced by EnzymeFixupJuliaCallingConvention().

CreateAllocation()

Value * CreateAllocation	(	IRBuilder<> &	Builder,
		llvm::Type *	T,
		Value *	Count,
		const Twine &	Name,
		CallInst **	caller,
		Instruction **	ZeroMem,
		bool	isDefault )

Definition at line 714 of file Utils.cpp.

References Count, CustomAllocator, getIntrinsicDeclaration(), and getPointerType().

Referenced by CacheUtility::createCacheForScope(), CacheUtility::getCachePointer(), getOrInsertDifferentialWaitallSave(), getOrInsertExponentialAllocator(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), and UpgradeAllocasToMallocs().

CreateDealloc()

CallInst * CreateDealloc	(	llvm::IRBuilder<> &	Builder,
		llvm::Value *	ToFree )

Definition at line 871 of file Utils.cpp.

References CustomDeallocator, and getInt8PtrTy().

Referenced by DiffeGradientUtils::freeCache(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), and AdjointGenerator::visitOMPCall().

createIntrinsicCall()

llvm::CallInst * createIntrinsicCall	(	llvm::IRBuilderBase &	B,
		llvm::Intrinsic::ID	ID,
		llvm::Type *	RetTy,
		llvm::ArrayRef< llvm::Value * >	Args,
		llvm::Instruction *	FMFSource,
		const llvm::Twine &	Name )

Definition at line 4528 of file Utils.cpp.

References Args.

CreateReAllocation()

llvm::Value * CreateReAllocation	(	llvm::IRBuilder<> &	B,
		llvm::Value *	prev,
		llvm::Type *	T,
		llvm::Value *	OuterCount,
		llvm::Value *	InnerCount,
		const llvm::Twine &	Name,
		llvm::CallInst **	caller,
		bool	ZeroMem )

Definition at line 685 of file Utils.cpp.

References getOrInsertExponentialAllocator().

Referenced by CacheUtility::createCacheForScope().

dumpBlock()

void dumpBlock

(

llvm::BasicBlock *

blk

)

Definition at line 4815 of file Utils.cpp.

dumpModule()

void dumpModule

(

llvm::Module *

mod

)

Definition at line 4811 of file Utils.cpp.

dumpType()

void dumpType

(

llvm::Type *

ty

)

Definition at line 4817 of file Utils.cpp.

dumpTypeResults()

void dumpTypeResults

(

TypeResults &

TR

)

Definition at line 4819 of file Utils.cpp.

References TypeResults::dump().

dumpValue()

void dumpValue

(

llvm::Value *

val

)

Definition at line 4813 of file Utils.cpp.

emit_backtrace()

void emit_backtrace	(	llvm::Instruction *	inst,
		llvm::raw_ostream &	ss )

Definition at line 964 of file Utils.cpp.

Referenced by EmitNoDerivativeError(), EmitNoDerivativeError(), EmitNoTypeError(), ErrorIfRuntimeInactive(), and SanitizeDerivatives().

EmitNoDerivativeError() [1/2]

llvm::Value * EmitNoDerivativeError	(	const std::string &	message,
		llvm::Instruction &	inst,
		GradientUtils *	gutils,
		llvm::IRBuilder<> &	Builder2,
		llvm::Value *	condition )

Definition at line 4593 of file Utils.cpp.

References contains(), CustomErrorHandler, TypeResults::dump(), emit_backtrace(), EmitFailure(), EnzymeRuntimeError(), getString(), NoDerivative, str(), and GradientUtils::TR.

Referenced by AdjointGenerator::createBinaryOperatorAdjoint(), AdjointGenerator::createBinaryOperatorDual(), AdjointGenerator::handleAdjointForIntrinsic(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitAtomicRMWInst(), AdjointGenerator::visitCastInst(), AdjointGenerator::visitInstruction(), and AdjointGenerator::visitMemSetCommon().

EmitNoDerivativeError() [2/2]

bool EmitNoDerivativeError	(	const std::string &	message,
		Value *	todiff,
		RequestContext &	context )

Definition at line 4632 of file Utils.cpp.

References CustomErrorHandler, emit_backtrace(), EmitFailure(), EnzymeRuntimeError(), getString(), RequestContext::ip, NoDerivative, RequestContext::req, and str().

EmitNoTypeError()

void EmitNoTypeError	(	const std::string &	message,
		llvm::Instruction &	inst,
		GradientUtils *	gutils,
		llvm::IRBuilder<> &	Builder2 )

Definition at line 4675 of file Utils.cpp.

References TypeResults::analyzer, CustomErrorHandler, TypeResults::dump(), emit_backtrace(), EmitFailure(), EnzymeRuntimeError(), getString(), NoType, str(), and GradientUtils::TR.

Referenced by TypeResults::firstPointer(), TypeResults::intType(), AdjointGenerator::visitCastInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitExtractValueInst(), AdjointGenerator::visitInsertValueInst(), AdjointGenerator::visitLoadLike(), AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitMemTransferCommon().

EnzymeBlasCopy()

llvm::cl::opt< bool > EnzymeBlasCopy	(	"enzyme-blas-copy"	,
		cl::init(true)	,
		cl::Hidden	,
		cl::desc("Use blas copy calls to cache vectors")	)

EnzymeCheckDerivativeNaN()

llvm::cl::opt< bool > EnzymeCheckDerivativeNaN	(	"enzyme-check-nan"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Add NaN checks to all derivative intermediate values")	)

Referenced by SanitizeDerivatives().

EnzymeFastMath()

llvm::cl::opt< bool > EnzymeFastMath	(	"enzyme-fast-math"	,
		cl::init(true)	,
		cl::Hidden	,
		cl::desc("Use fast math on derivative compuation")	)

Referenced by getFast().

EnzymeLapackCopy()

llvm::cl::opt< bool > EnzymeLapackCopy	(	"enzyme-lapack-copy"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Use blas copy calls to cache matrices")	)

EnzymeMemmoveWarning()

llvm::cl::opt< bool > EnzymeMemmoveWarning	(	"enzyme-memmove-warning"	,
		cl::init(true)	,
		cl::Hidden	,
		cl::desc("Warn if using memmove implementation as a fallback for memmove")	)

Referenced by getOrInsertDifferentialFloatMemmove().

EnzymeNonPower2Cache()

llvm::cl::opt< bool > EnzymeNonPower2Cache	(	"enzyme-non-power2-cache"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Disable caching of integers which are not a power of 2")	)

EnzymeRuntimeError()

llvm::cl::opt< bool > EnzymeRuntimeError	(	"enzyme-runtime-error"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Emit Runtime errors instead of compile time ones")	)

Referenced by EmitNoDerivativeError(), EmitNoDerivativeError(), and EmitNoTypeError().

ErrorIfRuntimeInactive()

void ErrorIfRuntimeInactive	(	llvm::IRBuilder<> &	B,
		llvm::Value *	primal,
		llvm::Value *	shadow,
		const char *	Message,
		llvm::DebugLoc &&	loc,
		llvm::Instruction *	orig )

Definition at line 1031 of file Utils.cpp.

References addFunctionNoCapture(), CustomRuntimeInactiveError, emit_backtrace(), getInt8PtrTy(), getString(), and str().

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

evaluateAtIterationWithoutExt()

const SCEV * evaluateAtIterationWithoutExt	(	const SCEVAddRecExpr *	AR,
		const SCEV *	It,
		ScalarEvolution &	SE )

Definition at line 2801 of file Utils.cpp.

Referenced by closedFormOfCountedAccumulator(), evaluateAtLoopIter(), overwritesToMemoryReadByLoop(), and NonAffineClosedFormRewriter::visitAddRecExpr().

extractBLAS()

llvm::Optional< BlasInfo > extractBLAS

(

llvm::StringRef

in

)

Definition at line 3871 of file Utils.cpp.

References matchBLASName().

Referenced by AdjointGenerator::handleKnownCallDerivatives(), and TypeAnalyzer::visitCallBase().

findAllUsersOf()

SmallVector< std::tuple< Instruction *, Value *, size_t >, 1 > findAllUsersOf

(

Value *

AI

)

Definition at line 3488 of file Utils.cpp.

References collectOffset().

Referenced by getAllLoadedValuesFrom(), and needsReRooting().

get1ULP()

llvm::Value * get1ULP	(	llvm::IRBuilder<> &	builder,
		llvm::Value *	res )

Definition at line 4570 of file Utils.cpp.

get_blas_row() [1/2]

SmallVector< llvm::Value *, 1 > get_blas_row	(	llvm::IRBuilder<> &	B,
		ArrayRef< llvm::Value * >	transA,
		ArrayRef< llvm::Value * >	row,
		ArrayRef< llvm::Value * >	col,
		bool	byRef,
		bool	cublas )

Definition at line 4411 of file Utils.cpp.

References get_blas_row().

get_blas_row() [2/2]

SmallVector< llvm::Value *, 1 > get_blas_row	(	llvm::IRBuilder<> &	B,
		ArrayRef< llvm::Value * >	transA,
		bool	byRef,
		bool	cublas )

Definition at line 4384 of file Utils.cpp.

Referenced by get_blas_row().

get_cached_mat_width()

llvm::Value * get_cached_mat_width	(	llvm::IRBuilder<> &	B,
		llvm::ArrayRef< llvm::Value * >	trans,
		llvm::Value *	arg_ld,
		llvm::Value *	dim1,
		llvm::Value *	dim2,
		bool	cacheMat,
		bool	byRef,
		bool	cublas )

Definition at line 4313 of file Utils.cpp.

References CreateSelect(), and is_normal().

getAllLoadedValuesFrom()

SmallVector< std::pair< Value *, size_t >, 1 > getAllLoadedValuesFrom	(	AllocaInst *	ptr0,
		size_t	offset,
		size_t	valSz,
		bool &	legal )

Definition at line 3533 of file Utils.cpp.

References findAllUsersOf(), getAllLoadedValuesFrom(), and getBaseAndOffset().

Referenced by getAllLoadedValuesFrom(), and simplifyLoad().

getBaseAndOffset()

AllocaInst * getBaseAndOffset	(	Value *	ptr,
		size_t &	offset )

Definition at line 3445 of file Utils.cpp.

References collectOffset(), and simplifyLoad().

Referenced by getAllLoadedValuesFrom(), and simplifyLoad().

getDefaultAnonymousTapeType()

llvm::PointerType * getDefaultAnonymousTapeType

(

llvm::LLVMContext &

C

)

Definition at line 531 of file Utils.cpp.

References EnzymeDefaultTapeType, and getInt8PtrTy().

Referenced by getFunctionTypeForClone(), and traceType().

getFast()

llvm::FastMathFlags getFast

(

)

Get LLVM fast math flags.

Definition at line 4026 of file Utils.cpp.

References EnzymeFastMath().

Referenced by callSPMVDiagUpdate(), CacheUtility::createCacheForScope(), fixSparse_inner(), DiffeGradientUtils::freeCache(), DiffeGradientUtils::getDifferential(), getOrInsertDifferentialFloatMemcpy(), getorInsertInnerProd(), getOrInsertMemcpyStrided(), getOrInsertOpFloatSum(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), lowerSparsePlaceholders(), AdjointGenerator::recursivelyHandleSubfunction(), CacheUtility::storeInstructionInCache(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitIntrinsicInst(), and AdjointGenerator::visitOMPCall().

getFirstFunctionDefinition()

Function * getFirstFunctionDefinition

(

Module &

M

)

Definition at line 3815 of file Utils.cpp.

GetFunctionFromValue()

Function * GetFunctionFromValue

(

Value *

fn

)

Definition at line 3811 of file Utils.cpp.

References GetFunctionValFromValue().

Referenced by callMemcpyStridedBlas(), callMemcpyStridedLapack(), copy_lower_to_upper(), getorInsertInnerProd(), TraceGenerator::handleObserveCall(), and TraceGenerator::handleSampleCall().

GetFunctionValFromValue()

Value * GetFunctionValFromValue

(

Value *

fn

)

Definition at line 3730 of file Utils.cpp.

References Call, GetFunctionValFromValue(), and simplifyLoad().

Referenced by GetFunctionFromValue(), and GetFunctionValFromValue().

getJuliaObjects()

llvm::SmallVector< llvm::Value *, 1 > getJuliaObjects	(	llvm::Value *	v,
		llvm::IRBuilder<> &	B )

Definition at line 5347 of file Utils.cpp.

References anyJuliaObjects(), and isSpecialPtr().

Referenced by RecursivelyReplaceAddressSpace().

getOrInsertCheckedFree()

Function * getOrInsertCheckedFree	(	Module &	M,
		CallInst *	call,
		Type *	Ty,
		unsigned	width )

Definition at line 2304 of file Utils.cpp.

References addFunctionNoCapture(), and getFuncNameFromCall().

getOrInsertDifferentialFloatMemcpy()

Function * getOrInsertDifferentialFloatMemcpy	(	Module &	M,
		Type *	elementType,
		unsigned	dstalign,
		unsigned	srcalign,
		unsigned	dstaddr,
		unsigned	srcaddr,
		unsigned	bitwidth,
		bool	runtimeActivity,
		bool	atomic )

Create function for type that is equivalent to memcpy but adds to destination rather than a direct copy; dst, src, numelems.

Definition at line 1160 of file Utils.cpp.

References addFunctionNoCapture(), getFast(), getPointerType(), and tofltstr().

Referenced by AdjointGenerator::DifferentiableMemCopyFloats(), and getOrInsertDifferentialFloatMemmove().

getOrInsertDifferentialFloatMemcpyMat()

Function * getOrInsertDifferentialFloatMemcpyMat	(	Module &	Mod,
		Type *	elementType,
		PointerType *	PT,
		IntegerType *	IT,
		IntegerType *	CT,
		unsigned	dstalign,
		unsigned	srcalign,
		bool	zeroSrc )

Definition at line 2123 of file Utils.cpp.

References tofltstr().

getOrInsertDifferentialFloatMemmove()

Function * getOrInsertDifferentialFloatMemmove	(	Module &	M,
		Type *	T,
		unsigned	dstalign,
		unsigned	srcalign,
		unsigned	dstaddr,
		unsigned	srcaddr,
		unsigned	bitwidth,
		bool	runtimeActivity,
		bool	atomic )

Definition at line 2282 of file Utils.cpp.

References EnzymeMemmoveWarning(), and getOrInsertDifferentialFloatMemcpy().

getOrInsertDifferentialMPI_Wait()

llvm::Function * getOrInsertDifferentialMPI_Wait	(	llvm::Module &	M,
		ArrayRef< llvm::Type * >	T,
		Type *	reqType,
		StringRef	caller )

Definition at line 2494 of file Utils.cpp.

References getInt8PtrTy(), getRenamedPerCallingConv(), ISEND, str(), and tripleSplitDollar().

Referenced by AdjointGenerator::handleMPI().

getOrInsertDifferentialWaitallSave()

llvm::Function * getOrInsertDifferentialWaitallSave	(	llvm::Module &	M,
		ArrayRef< llvm::Type * >	T,
		PointerType *	reqType )

Definition at line 2426 of file Utils.cpp.

References CreateAllocation(), and getUnqual().

Referenced by AdjointGenerator::handleMPI().

getOrInsertExponentialAllocator()

Function * getOrInsertExponentialAllocator	(	Module &	M,
		Function *	newFunc,
		bool	ZeroInit,
		llvm::Type *	RT )

Definition at line 537 of file Utils.cpp.

References CreateAllocation(), getFunctionFromCall(), getIntrinsicDeclaration(), and getPointerType().

Referenced by CreateReAllocation().

getorInsertInnerProd()

llvm::CallInst * getorInsertInnerProd	(	llvm::IRBuilder<> &	B,
		llvm::Module &	M,
		BlasInfo	blas,
		IntegerType *	IT,
		Type *	BlasPT,
		Type *	BlasIT,
		Type *	fpTy,
		llvm::ArrayRef< llvm::Value * >	args,
		const llvm::ArrayRef< llvm::OperandBundleDef >	bundles,
		bool	byRef,
		bool	cublas,
		bool	julia_decl )

Definition at line 1752 of file Utils.cpp.

References addFunctionNoCapture(), attributeKnownFunctions(), BlasInfo::floatType, getFast(), GetFunctionFromValue(), getPointerType(), load_if_ref(), BlasInfo::prefix, BlasInfo::suffix, and to_blas_callconv().

getOrInsertMemcpyMat()

Function * getOrInsertMemcpyMat	(	Module &	Mod,
		Type *	elementType,
		PointerType *	PT,
		IntegerType *	IT,
		unsigned	dstalign,
		unsigned	srcalign )

Definition at line 2003 of file Utils.cpp.

References addFunctionNoCapture(), and tofltstr().

getOrInsertMemcpyStrided()

Function * getOrInsertMemcpyStrided	(	Module &	M,
		Type *	elementType,
		PointerType *	T,
		Type *	IT,
		unsigned	dstalign,
		unsigned	srcalign )

Definition at line 1901 of file Utils.cpp.

References addFunctionNoCapture(), getFast(), and tofltstr().

getOrInsertOpFloatSum()

llvm::Value * getOrInsertOpFloatSum	(	llvm::Module &	M,
		llvm::Function *	templateFn,
		llvm::Type *	OpPtr,
		llvm::Type *	OpType,
		ConcreteType	CT,
		llvm::Type *	intType,
		IRBuilder<> &	B2 )

Definition at line 2601 of file Utils.cpp.

References addFunctionNoCapture(), getFast(), getInt8PtrTy(), getOrInsertPerCallingConv(), getRenamedPerCallingConv(), getUnqual(), ConcreteType::isFloat(), and ConcreteType::str().

Referenced by AdjointGenerator::handleMPI().

getOrInsertPerCallingConv()

FunctionCallee getOrInsertPerCallingConv	(	Module &	M,
		Function *	templateFn,
		StringRef	callee,
		FunctionType *	FT )

Definition at line 2294 of file Utils.cpp.

References getRenamedPerCallingConv().

Referenced by getOrInsertOpFloatSum(), and AdjointGenerator::handleKnownCallDerivatives().

getString()

Constant * getString	(	Module &	M,
		StringRef	Str )

Definition at line 954 of file Utils.cpp.

Referenced by EmitNoDerivativeError(), EmitNoDerivativeError(), EmitNoTypeError(), ErrorIfRuntimeInactive(), and SanitizeDerivatives().

getUndefinedValueForType()

llvm::Constant * getUndefinedValueForType	(	llvm::Module &	M,
		llvm::Type *	T,
		bool	forceZero )

Definition at line 3918 of file Utils.cpp.

References EnzymeUndefinedValueForType, and EnzymeZeroCache.

Referenced by copyNonJLValueInto(), CacheUtility::createCacheForScope(), AdjointGenerator::handleKnownCallDerivatives(), moveSRetToFromRoots(), and AdjointGenerator::recursivelyHandleSubfunction().

is_left()

llvm::Value * is_left	(	IRBuilder<> &	B,
		llvm::Value *	side,
		bool	byRef,
		bool	cublas )

Definition at line 4199 of file Utils.cpp.

is_lower()

Value * is_lower	(	IRBuilder<> &	B,
		Value *	uplo,
		bool	byRef,
		bool	cublas )

Definition at line 4095 of file Utils.cpp.

Referenced by callSPMVDiagUpdate().

is_nonunit()

Value * is_nonunit	(	IRBuilder<> &	B,
		Value *	uplo,
		bool	byRef,
		bool	cublas )

Definition at line 4129 of file Utils.cpp.

is_normal()

llvm::Value * is_normal	(	IRBuilder<> &	B,
		llvm::Value *	trans,
		bool	byRef,
		bool	cublas )

Definition at line 4163 of file Utils.cpp.

Referenced by get_cached_mat_width().

isAtomic()

bool isAtomic	(	Value *	origptr,
		bool	AtomicAdd,
		Function *	newFunc )

Definition at line 1136 of file Utils.cpp.

References getBaseObject().

isFlangRuntimeNoFree()

bool isFlangRuntimeNoFree

(

llvm::StringRef

name

)

LLVM flang runtime functions that neither free memory of the program nor return or store an allocation into it: I/O (which frees only the I/O statement state that _FortranAioBegin* allocated and _FortranAioEndIoStatement releases), FLUSH, the scalar character inquiries and comparison, and MOD/MODULO.

Definition at line 115 of file Utils.cpp.

References startsWith().

Referenced by attributeKnownFunctions().

isNVLoad()

bool isNVLoad

(

const llvm::Value *

V

)

Definition at line 4821 of file Utils.cpp.

load_if_ref()

llvm::Value * load_if_ref	(	llvm::IRBuilder<> &	B,
		llvm::Type *	intType,
		llvm::Value *	V,
		bool	byRef )

Definition at line 4370 of file Utils.cpp.

References getPointerType(), and getUnqual().

Referenced by callSPMVDiagUpdate(), and getorInsertInnerProd().

lookup_with_layout()

Value * lookup_with_layout	(	IRBuilder<> &	B,
		Type *	fpType,
		Value *	layout,
		Value *const	base,
		Value *	lda,
		Value *	row,
		Value *	col )

Definition at line 1333 of file Utils.cpp.

References CreateSelect(), getPointerType(), and getUnqual().

Referenced by copy_lower_to_upper().

matchBLASName()

static bool matchBLASName ( StringRef in, ArrayRef< const char * > prefixes, ArrayRef< const char * > floatTypes, ArrayRef< const char * > stems, ArrayRef< const char * > suffixes, const char *& prefixOut, const char *& floatTypeOut, const char *& stemOut, const char *& suffixOut )

static

Decompose in as prefix + floatType + stem + suffix using the supplied tables, setting the out-parameters to the entries that matched.

No name the tables in extractBLAS can spell decomposes in more than one way, so peeling the fixed parts off either end of in and looking up what remains is equivalent to comparing in against every name the tables can generate – but it allocates nothing. That matters: extractBLAS runs for every call instruction visited by type analysis and by is_use_directly_needed_in_reverse, and materialising the ~1200 candidate std::strings per query dominated compile time on call-heavy modules that contain no BLAS at all.

Definition at line 3835 of file Utils.cpp.

Referenced by extractBLAS().

mayExecuteAfter()

void mayExecuteAfter	(	llvm::SmallVectorImpl< llvm::Instruction * > &	results,
		llvm::Instruction *	inst,
		const llvm::SetVector< Instruction * > &	stores,
		const llvm::Loop *	region )

Definition at line 2740 of file Utils.cpp.

moveSRetToFromRoots()

llvm::Value * moveSRetToFromRoots	(	llvm::IRBuilder<> &	B,
		llvm::Type *	jltype,
		llvm::Value *	sret,
		llvm::Type *	root_ty,
		llvm::Value *	rootRet,
		size_t	rootOffset,
		SRetRootMovement	direction )

Definition at line 5148 of file Utils.cpp.

References constantInBoundsGEPHelper(), CountTrackedPointers::count, GradientUtils::extractMeta(), getBaseObject(), getPointerType(), getUndefinedValueForType(), isSpecialPtr(), NullifySRetValue, RootPointerToSRetPointer, RootPointerToSRetValue, SRetPointerToRootPointer, SRetValueToRootPointer, and Tracked.

Referenced by EnzymeFixupJuliaCallingConvention().

nextPowerOfTwo()

llvm::Value * nextPowerOfTwo	(	llvm::IRBuilder<> &	B,
		llvm::Value *	V )

Create function to computer nearest power of two.

Definition at line 2415 of file Utils.cpp.

Referenced by PreProcessCache::ReplaceReallocs().

notCaptured()

bool notCaptured	(	llvm::Value *	V,
		llvm::TargetLibraryInfo *	TLI )

Check if value if b captured.

Definition at line 4956 of file Utils.cpp.

References notCapturedBefore().

notCapturedBefore()

bool notCapturedBefore	(	llvm::Value *	V,
		Instruction *	inst,
		size_t	checkLoadCaptures,
		Instruction *	startinst,
		llvm::TargetLibraryInfo *	TLI )

Definition at line 4841 of file Utils.cpp.

References isDeallocationCall(), isNoCapture(), and isPointerArithmeticInst().

Referenced by arePointersGuaranteedNoAlias(), and notCaptured().

overwritesToMemoryReadBy()

bool overwritesToMemoryReadBy	(	const TypeResults *	TR,
		llvm::AAResults &	AA,
		llvm::TargetLibraryInfo &	TLI,
		ScalarEvolution &	SE,
		llvm::LoopInfo &	LI,
		llvm::DominatorTree &	DT,
		llvm::Instruction *	maybeReader,
		llvm::Instruction *	maybeWriter,
		llvm::Loop *	scope )

Definition at line 3040 of file Utils.cpp.

References arePointersGuaranteedNoAlias(), overwritesToMemoryReadByLoop(), and writesToMemoryReadBy().

overwritesToMemoryReadByLoop()

bool overwritesToMemoryReadByLoop	(	llvm::ScalarEvolution &	SE,
		llvm::LoopInfo &	LI,
		llvm::DominatorTree &	DT,
		llvm::Instruction *	maybeReader,
		const llvm::SCEV *	LoadStart,
		const llvm::SCEV *	LoadEnd,
		llvm::Instruction *	maybeWriter,
		const llvm::SCEV *	StoreStart,
		const llvm::SCEV *	StoreEnd,
		llvm::Loop *	scope )

We force all ranges for all loops in range ... [scope, anc], .... cur to expand the number of iterations

Definition at line 2849 of file Utils.cpp.

References evaluateAtIterationWithoutExt(), and getAncestor().

Referenced by overwritesToMemoryReadBy().

parseTrueType()

std::vector< std::tuple< llvm::Type *, size_t, size_t > > parseTrueType	(	const llvm::MDNode *	md,
		DerivativeMode	Mode,
		bool	const_src )

Definition at line 4749 of file Utils.cpp.

References Anything, ForwardMode, and ForwardModeError.

Referenced by AdjointGenerator::visitInsertValueInst(), AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitMemTransferCommon().

PostCacheStore()

llvm::SmallVector< llvm::Instruction *, 2 > PostCacheStore	(	llvm::StoreInst *	SI,
		llvm::IRBuilder<> &	B )

Definition at line 517 of file Utils.cpp.

References EnzymePostCacheStore.

Referenced by CacheUtility::createCacheForScope(), EnzymeFixupJuliaCallingConvention(), and RecursivelyReplaceAddressSpace().

SanitizeDerivatives()

llvm::Value * SanitizeDerivatives	(	llvm::Value *	val,
		llvm::Value *	toset,
		llvm::IRBuilder<> &	BuilderM,
		llvm::Value *	mask )

Definition at line 3929 of file Utils.cpp.

References CustomErrorHandler, emit_backtrace(), EnzymeCheckDerivativeNaN(), EnzymeSanitizeDerivatives, getInt8PtrTy(), getString(), NaNError, and str().

Referenced by DiffeGradientUtils::setDiffe().

simplifyLoad()

Value * simplifyLoad	(	Value *	V,
		size_t	valSz,
		size_t	preOffset )

Definition at line 3664 of file Utils.cpp.

References GradientUtils::extractMeta(), getAggregateElementOffset(), getAllLoadedValuesFrom(), getBaseAndOffset(), and simplifyLoad().

Referenced by getBaseAndOffset(), GetFunctionValFromValue(), OldAllocationSize(), and simplifyLoad().

sliceTrueType()

llvm::MDNode * sliceTrueType	(	const llvm::MDNode *	md,
		size_t	start,
		size_t	length )

Restrict an enzyme_truetype annotation describing a memory region to the byte range [start, start + length), rebasing its offsets to that range.

Returns nullptr if no type information covers the range.

Definition at line 4713 of file Utils.cpp.

Referenced by AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitMemTransferCommon().

to_blas_callconv()

llvm::Value * to_blas_callconv	(	IRBuilder<> &	B,
		llvm::Value *	V,
		bool	byRef,
		bool	cublas,
		IntegerType *	julia_decl,
		IRBuilder<> &	entryBuilder,
		llvm::Twine const &	name )

Definition at line 4062 of file Utils.cpp.

References getInt8PtrTy().

Referenced by copy_lower_to_upper(), getorInsertInnerProd(), and transpose().

to_blas_fp_callconv()

llvm::Value * to_blas_fp_callconv	(	IRBuilder<> &	B,
		llvm::Value *	V,
		bool	byRef,
		Type *	fpTy,
		IRBuilder<> &	entryBuilder,
		llvm::Twine const &	name )

Definition at line 4079 of file Utils.cpp.

tofltstr()

static std::string tofltstr ( Type * T )

inlinestatic

Convert a floating type to a string.

Definition at line 925 of file Utils.cpp.

References tofltstr().

Referenced by getOrInsertDifferentialFloatMemcpy(), getOrInsertDifferentialFloatMemcpyMat(), getOrInsertMemcpyMat(), getOrInsertMemcpyStrided(), and tofltstr().

transpose() [1/2]

llvm::Value * transpose	(	std::string	floatType,
		IRBuilder<> &	B,
		llvm::Value *	V,
		bool	cublas )

Definition at line 4242 of file Utils.cpp.

References CustomErrorHandler, EmitFailure(), and NoDerivative.

Referenced by transpose().

transpose() [2/2]

llvm::Value * transpose	(	std::string	floatType,
		llvm::IRBuilder<> &	B,
		llvm::Value *	V,
		bool	byRef,
		bool	cublas,
		llvm::IntegerType *	julia_decl,
		llvm::IRBuilder<> &	entryBuilder,
		const llvm::Twine &	name )

Definition at line 4329 of file Utils.cpp.

References to_blas_callconv(), and transpose().

writesToMemoryReadBy()

bool writesToMemoryReadBy	(	const TypeResults *	TR,
		llvm::AAResults &	AA,
		llvm::TargetLibraryInfo &	TLI,
		llvm::Instruction *	maybeReader,
		llvm::Instruction *	maybeWriter )

Return whether maybeReader can read from memory written to by maybeWriter.

Definition at line 3155 of file Utils.cpp.

References Anything, getFuncNameFromCall(), isAllocationFunction(), isCertainPrint(), isDeallocationFunction(), isDebugFunction(), ConcreteType::isKnown(), isMemFreeLibMFunction(), isReadOnlyOrThrow(), TypeTree::Lookup(), parseTBAA(), Pointer, TypeResults::query(), and Unknown.

Referenced by arePointersGuaranteedNoAlias(), and overwritesToMemoryReadBy().

ZeroMemory()

void ZeroMemory	(	llvm::IRBuilder<> &	Builder,
		llvm::Type *	T,
		llvm::Value *	obj,
		bool	isTape )

Definition at line 508 of file Utils.cpp.

References CustomZero.

Referenced by DiffeGradientUtils::getDifferential().

Variable Documentation

CustomAllocator

LLVMValueRef(* CustomAllocator) (LLVMBuilderRef, LLVMTypeRef, LLVMValueRef, LLVMValueRef, uint8_t, LLVMValueRef *)	(	LLVMBuilderRef	,
		LLVMTypeRef	,
		LLVMValueRef	,
		LLVMValueRef	,
		uint8_t	,
		LLVMValueRef *	) = nullptr

Definition at line 65 of file Utils.cpp.

Referenced by CreateAllocation().

CustomDeallocator

LLVMValueRef(* CustomDeallocator) (LLVMBuilderRef, LLVMValueRef)	(	LLVMBuilderRef	,
		LLVMValueRef	) = nullptr

Definition at line 71 of file Utils.cpp.

Referenced by CreateDealloc().

CustomErrorHandler

LLVMValueRef(* CustomErrorHandler) (const char *, LLVMValueRef, ErrorType, const void *, LLVMValueRef, LLVMBuilderRef)	(	const char *	,
		LLVMValueRef	,
		ErrorType	,
		const void *	,
		LLVMValueRef	,
		LLVMBuilderRef	) = nullptr

Definition at line 62 of file Utils.cpp.

Referenced by analyzeIntelSubscriptIntrinsic(), checkEnzymeTypeExtent(), EmitNoDerivativeError(), EmitNoDerivativeError(), EmitNoTypeError(), EnzymeFixupJuliaCallingConvention(), CacheUtility::erase(), TypeResults::firstPointer(), AdjointGenerator::handleKnownCallDerivatives(), TypeTree::insert(), needsReRooting(), TypeTree::Only(), AdjointGenerator::recursivelyHandleSubfunction(), RecursivelyReplaceAddressSpace(), SanitizeDerivatives(), transpose(), TypeAnalyzer::visitBinaryOperation(), AdjointGenerator::visitCommonStore(), TypeAnalyzer::visitGEPOperator(), TypeAnalyzer::visitIntrinsicInst(), TypeAnalyzer::visitMemTransferCommon(), and TypeAnalyzer::visitPHINode().

CustomRuntimeInactiveError

void(* CustomRuntimeInactiveError) (LLVMBuilderRef, LLVMValueRef, LLVMValueRef)	(	LLVMBuilderRef	,
		LLVMValueRef	,
		LLVMValueRef	) = nullptr

Definition at line 72 of file Utils.cpp.

Referenced by ErrorIfRuntimeInactive().

CustomZero

void(* CustomZero) (LLVMBuilderRef, LLVMTypeRef, LLVMValueRef, uint8_t)	(	LLVMBuilderRef	,
		LLVMTypeRef	,
		LLVMValueRef	,
		uint8_t	) = nullptr

Definition at line 69 of file Utils.cpp.

Referenced by ZeroMemory().

EnzymeDefaultTapeType

LLVMTypeRef(* EnzymeDefaultTapeType) (LLVMContextRef)

(

LLVMContextRef

)

= nullptr

Definition at line 76 of file Utils.cpp.

Referenced by getDefaultAnonymousTapeType().

EnzymePostCacheStore

LLVMValueRef *(* EnzymePostCacheStore) (LLVMValueRef, LLVMBuilderRef, uint64_t *size)	(	LLVMValueRef	,
		LLVMBuilderRef	,
		uint64_t *	size ) = nullptr

Definition at line 74 of file Utils.cpp.

Referenced by PostCacheStore().

EnzymeSanitizeDerivatives

LLVMValueRef(* EnzymeSanitizeDerivatives) (LLVMValueRef, LLVMValueRef toset, LLVMBuilderRef, LLVMValueRef)	(	LLVMValueRef	,
		LLVMValueRef	toset,
		LLVMBuilderRef	,
		LLVMValueRef	) = nullptr

Definition at line 80 of file Utils.cpp.

Referenced by SanitizeDerivatives().

EnzymeUndefinedValueForType

LLVMValueRef(* EnzymeUndefinedValueForType) (LLVMModuleRef, LLVMTypeRef, uint8_t)	(	LLVMModuleRef	,
		LLVMTypeRef	,
		uint8_t	) = nullptr

Definition at line 77 of file Utils.cpp.

Referenced by getUndefinedValueForType().

EnzymeZeroCache

llvm::cl::opt<bool> EnzymeZeroCache

extern

Referenced by getUndefinedValueForType().