Utils.cpp File Reference

#include "Utils.h"
#include "GradientUtils.h"
#include "TypeAnalysis/TypeAnalysis.h"
#include "SCEV/ScalarEvolution.h"
#include "SCEV/ScalarEvolutionExpander.h"
#include "TypeAnalysis/TBAA.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Verifier.h"
#include "llvm/ADT/Triple.h"
#include "llvm-c/Core.h"
#include "BlasAttributor.inc"
#include "LibraryFuncs.h"

Include dependency graph for Utils.cpp:

Go to the source code of this file.

Macros
#define	addAttribute   addAttributeAtIndex
#define	getAttribute   getAttributeAtIndex

Functions
llvm::cl::opt< bool >	EnzymeLapackCopy ("enzyme-lapack-copy", cl::init(false), cl::Hidden, cl::desc("Use blas copy calls to cache matrices"))
llvm::cl::opt< bool >	EnzymeBlasCopy ("enzyme-blas-copy", cl::init(true), cl::Hidden, cl::desc("Use blas copy calls to cache vectors"))
llvm::cl::opt< bool >	EnzymeFastMath ("enzyme-fast-math", cl::init(true), cl::Hidden, cl::desc("Use fast math on derivative compuation"))
llvm::cl::opt< bool >	EnzymeMemmoveWarning ("enzyme-memmove-warning", cl::init(true), cl::Hidden, cl::desc("Warn if using memmove implementation as a fallback for memmove"))
llvm::cl::opt< bool >	EnzymeRuntimeError ("enzyme-runtime-error", cl::init(false), cl::Hidden, cl::desc("Emit Runtime errors instead of compile time ones"))
llvm::cl::opt< bool >	EnzymeCheckDerivativeNaN ("enzyme-check-nan", cl::init(false), cl::Hidden, cl::desc("Add NaN checks to all derivative intermediate values"))
llvm::cl::opt< bool >	EnzymeNonPower2Cache ("enzyme-non-power2-cache", cl::init(false), cl::Hidden, cl::desc("Disable caching of integers which are not a power of 2"))
bool	attributeKnownFunctions (llvm::Function &F)
void	ZeroMemory (llvm::IRBuilder<> &Builder, llvm::Type *T, llvm::Value *obj, bool isTape)
llvm::SmallVector< llvm::Instruction *, 2 >	PostCacheStore (llvm::StoreInst *SI, llvm::IRBuilder<> &B)
llvm::PointerType *	getDefaultAnonymousTapeType (llvm::LLVMContext &C)
Function *	getOrInsertExponentialAllocator (Module &M, Function *newFunc, bool ZeroInit, llvm::Type *RT)
llvm::Value *	CreateReAllocation (llvm::IRBuilder<> &B, llvm::Value *prev, llvm::Type *T, llvm::Value *OuterCount, llvm::Value *InnerCount, const llvm::Twine &Name, llvm::CallInst **caller, bool ZeroMem)
Value *	CreateAllocation (IRBuilder<> &Builder, llvm::Type *T, Value *Count, const Twine &Name, CallInst **caller, Instruction **ZeroMem, bool isDefault)
CallInst *	CreateDealloc (llvm::IRBuilder<> &Builder, llvm::Value *ToFree)
static std::string	tofltstr (Type *T)
	Convert a floating type to a string.
Constant *	getString (Module &M, StringRef Str)
void	emit_backtrace (llvm::Instruction *inst, llvm::raw_ostream &ss)
void	ErrorIfRuntimeInactive (llvm::IRBuilder<> &B, llvm::Value *primal, llvm::Value *shadow, const char *Message, llvm::DebugLoc &&loc, llvm::Instruction *orig)
Function *	getOrInsertDifferentialFloatMemcpy (Module &M, Type *elementType, unsigned dstalign, unsigned srcalign, unsigned dstaddr, unsigned srcaddr, unsigned bitwidth)
	Create function for type that is equivalent to memcpy but adds to destination rather than a direct copy; dst, src, numelems.
Value *	lookup_with_layout (IRBuilder<> &B, Type *fpType, Value *layout, Value *const base, Value *lda, Value *row, Value *col)
void	copy_lower_to_upper (llvm::IRBuilder<> &B, llvm::Type *fpType, BlasInfo blas, bool byRef, llvm::Value *layout, llvm::Value *islower, llvm::Value *A, llvm::Value *lda, llvm::Value *N)
void	callMemcpyStridedBlas (llvm::IRBuilder<> &B, llvm::Module &M, BlasInfo blas, llvm::ArrayRef< llvm::Value * > args, llvm::Type *copy_retty, llvm::ArrayRef< llvm::OperandBundleDef > bundles)
	Create function for type that performs memcpy with a stride using blas copy.
void	callMemcpyStridedLapack (llvm::IRBuilder<> &B, llvm::Module &M, BlasInfo blas, llvm::ArrayRef< llvm::Value * > args, llvm::ArrayRef< llvm::OperandBundleDef > bundles)
	Create function for type that performs memcpy using lapack copy.
void	callSPMVDiagUpdate (IRBuilder<> &B, Module &M, BlasInfo blas, IntegerType *IT, Type *BlasCT, Type *BlasFPT, Type *BlasPT, Type *BlasIT, Type *fpTy, ArrayRef< Value * > args, ArrayRef< OperandBundleDef > bundles, bool byRef, bool julia_decl)
llvm::CallInst *	getorInsertInnerProd (llvm::IRBuilder<> &B, llvm::Module &M, BlasInfo blas, IntegerType *IT, Type *BlasPT, Type *BlasIT, Type *fpTy, llvm::ArrayRef< llvm::Value * > args, const llvm::ArrayRef< llvm::OperandBundleDef > bundles, bool byRef, bool cublas, bool julia_decl)
Function *	getOrInsertMemcpyStrided (Module &M, Type *elementType, PointerType *T, Type *IT, unsigned dstalign, unsigned srcalign)
Function *	getOrInsertMemcpyMat (Module &Mod, Type *elementType, PointerType *PT, IntegerType *IT, unsigned dstalign, unsigned srcalign)
Function *	getOrInsertDifferentialFloatMemcpyMat (Module &Mod, Type *elementType, PointerType *PT, IntegerType *IT, IntegerType *CT, unsigned dstalign, unsigned srcalign, bool zeroSrc)
Function *	getOrInsertDifferentialFloatMemmove (Module &M, Type *T, unsigned dstalign, unsigned srcalign, unsigned dstaddr, unsigned srcaddr, unsigned bitwidth)
Function *	getOrInsertCheckedFree (Module &M, CallInst *call, Type *Ty, unsigned width)
llvm::Value *	nextPowerOfTwo (llvm::IRBuilder<> &B, llvm::Value *V)
	Create function to computer nearest power of two.
llvm::Function *	getOrInsertDifferentialWaitallSave (llvm::Module &M, ArrayRef< llvm::Type * > T, PointerType *reqType)
llvm::Function *	getOrInsertDifferentialMPI_Wait (llvm::Module &M, ArrayRef< llvm::Type * > T, Type *reqType, StringRef caller)
llvm::Value *	getOrInsertOpFloatSum (llvm::Module &M, llvm::Type *OpPtr, llvm::Type *OpType, ConcreteType CT, llvm::Type *intType, IRBuilder<> &B2)
void	mayExecuteAfter (llvm::SmallVectorImpl< llvm::Instruction * > &results, llvm::Instruction *inst, const llvm::SmallPtrSetImpl< Instruction * > &stores, const llvm::Loop *region)
bool	overwritesToMemoryReadByLoop (llvm::ScalarEvolution &SE, llvm::LoopInfo &LI, llvm::DominatorTree &DT, llvm::Instruction *maybeReader, const llvm::SCEV *LoadStart, const llvm::SCEV *LoadEnd, llvm::Instruction *maybeWriter, const llvm::SCEV *StoreStart, const llvm::SCEV *StoreEnd, llvm::Loop *scope)
bool	overwritesToMemoryReadBy (const TypeResults *TR, llvm::AAResults &AA, llvm::TargetLibraryInfo &TLI, ScalarEvolution &SE, llvm::LoopInfo &LI, llvm::DominatorTree &DT, llvm::Instruction *maybeReader, llvm::Instruction *maybeWriter, llvm::Loop *scope)
bool	writesToMemoryReadBy (const TypeResults *TR, llvm::AAResults &AA, llvm::TargetLibraryInfo &TLI, llvm::Instruction *maybeReader, llvm::Instruction *maybeWriter)
	Return whether maybeReader can read from memory written to by maybeWriter.
AllocaInst *	getBaseAndOffset (Value *ptr, size_t &offset)
SmallVector< std::tuple< Instruction *, Value *, size_t >, 1 >	findAllUsersOf (Value *AI)
SmallVector< std::pair< Value *, size_t >, 1 >	getAllLoadedValuesFrom (AllocaInst *ptr0, size_t offset, size_t valSz, bool &legal)
Value *	simplifyLoad (Value *V, size_t valSz, size_t preOffset)
Value *	GetFunctionValFromValue (Value *fn)
Function *	GetFunctionFromValue (Value *fn)
Function *	getFirstFunctionDefinition (Module &M)
llvm::Optional< BlasInfo >	extractBLAS (llvm::StringRef in)
llvm::Constant *	getUndefinedValueForType (llvm::Module &M, llvm::Type *T, bool forceZero)
llvm::Value *	SanitizeDerivatives (llvm::Value *val, llvm::Value *toset, llvm::IRBuilder<> &BuilderM, llvm::Value *mask)
llvm::FastMathFlags	getFast ()
	Get LLVM fast math flags.
void	addValueToCache (llvm::Value *arg, bool cache_arg, llvm::Type *ty, llvm::SmallVectorImpl< llvm::Value * > &cacheValues, llvm::IRBuilder<> &BuilderZ, const Twine &name)
llvm::Value *	to_blas_callconv (IRBuilder<> &B, llvm::Value *V, bool byRef, bool cublas, IntegerType *julia_decl, IRBuilder<> &entryBuilder, llvm::Twine const &name)
llvm::Value *	to_blas_fp_callconv (IRBuilder<> &B, llvm::Value *V, bool byRef, Type *fpTy, IRBuilder<> &entryBuilder, llvm::Twine const &name)
Value *	is_lower (IRBuilder<> &B, Value *uplo, bool byRef, bool cublas)
Value *	is_nonunit (IRBuilder<> &B, Value *uplo, bool byRef, bool cublas)
llvm::Value *	is_normal (IRBuilder<> &B, llvm::Value *trans, bool byRef, bool cublas)
llvm::Value *	is_left (IRBuilder<> &B, llvm::Value *side, bool byRef, bool cublas)
llvm::Value *	transpose (std::string floatType, IRBuilder<> &B, llvm::Value *V, bool cublas)
llvm::Value *	get_cached_mat_width (llvm::IRBuilder<> &B, llvm::ArrayRef< llvm::Value * > trans, llvm::Value *arg_ld, llvm::Value *dim1, llvm::Value *dim2, bool cacheMat, bool byRef, bool cublas)
llvm::Value *	transpose (std::string floatType, llvm::IRBuilder<> &B, llvm::Value *V, bool byRef, bool cublas, llvm::IntegerType *julia_decl, llvm::IRBuilder<> &entryBuilder, const llvm::Twine &name)
llvm::Value *	load_if_ref (llvm::IRBuilder<> &B, llvm::Type *intType, llvm::Value *V, bool byRef)
SmallVector< llvm::Value *, 1 >	get_blas_row (llvm::IRBuilder<> &B, ArrayRef< llvm::Value * > transA, bool byRef, bool cublas)
SmallVector< llvm::Value *, 1 >	get_blas_row (llvm::IRBuilder<> &B, ArrayRef< llvm::Value * > transA, ArrayRef< llvm::Value * > row, ArrayRef< llvm::Value * > col, bool byRef, bool cublas)
bool	collectOffset (GEPOperator *gep, const DataLayout &DL, unsigned BitWidth, MapVector< Value *, APInt > &VariableOffsets, APInt &ConstantOffset)
llvm::CallInst *	createIntrinsicCall (llvm::IRBuilderBase &B, llvm::Intrinsic::ID ID, llvm::Type *RetTy, llvm::ArrayRef< llvm::Value * > Args, llvm::Instruction *FMFSource, const llvm::Twine &Name)
llvm::Value *	get1ULP (llvm::IRBuilder<> &builder, llvm::Value *res)
llvm::Value *	EmitNoDerivativeError (const std::string &message, llvm::Instruction &inst, GradientUtils *gutils, llvm::IRBuilder<> &Builder2, llvm::Value *condition)
bool	EmitNoDerivativeError (const std::string &message, Value *todiff, RequestContext &context)
void	EmitNoTypeError (const std::string &message, llvm::Instruction &inst, GradientUtils *gutils, llvm::IRBuilder<> &Builder2)
std::vector< std::tuple< llvm::Type *, size_t, size_t > >	parseTrueType (const llvm::MDNode *md, DerivativeMode Mode, bool const_src)
void	dumpModule (llvm::Module *mod)
void	dumpValue (llvm::Value *val)
void	dumpBlock (llvm::BasicBlock *blk)
void	dumpType (llvm::Type *ty)
void	dumpTypeResults (TypeResults &TR)
bool	isNVLoad (const llvm::Value *V)
bool	notCapturedBefore (llvm::Value *V, Instruction *inst, size_t checkLoadCaptures)
bool	notCaptured (llvm::Value *V)
	Check if value if b captured.
llvm::Optional< bool >	arePointersGuaranteedNoAlias (TargetLibraryInfo &TLI, llvm::AAResults &AA, llvm::LoopInfo &LI, llvm::Value *op0, llvm::Value *op1, bool offsetAllowed)
static Value *	constantInBoundsGEPHelper (llvm::IRBuilder<> &B, llvm::Type *type, llvm::Value *value, ArrayRef< unsigned > path)
llvm::Value *	moveSRetToFromRoots (llvm::IRBuilder<> &B, llvm::Type *jltype, llvm::Value *sret, llvm::Type *root_ty, llvm::Value *rootRet, size_t rootOffset, SRetRootMovement direction)
void	copyNonJLValueInto (llvm::IRBuilder<> &B, llvm::Type *curType, llvm::Type *dstType, llvm::Value *dst, llvm::ArrayRef< unsigned > dstPrefix0, llvm::Type *srcType, llvm::Value *src, llvm::ArrayRef< unsigned > srcPrefix0, bool shouldZero)
llvm::SmallVector< llvm::Value *, 1 >	getJuliaObjects (llvm::Value *v, llvm::IRBuilder<> &B)

Variables
LLVMValueRef(*	CustomErrorHandler )(const char *, LLVMValueRef, ErrorType, const void *, LLVMValueRef, LLVMBuilderRef) = nullptr
LLVMValueRef(*	CustomAllocator )(LLVMBuilderRef, LLVMTypeRef, LLVMValueRef, LLVMValueRef, uint8_t, LLVMValueRef *) = nullptr
void(*	CustomZero )(LLVMBuilderRef, LLVMTypeRef, LLVMValueRef, uint8_t) = nullptr
LLVMValueRef(*	CustomDeallocator )(LLVMBuilderRef, LLVMValueRef) = nullptr
void(*	CustomRuntimeInactiveError )(LLVMBuilderRef, LLVMValueRef, LLVMValueRef) = nullptr
LLVMValueRef *(*	EnzymePostCacheStore )(LLVMValueRef, LLVMBuilderRef, uint64_t *size) = nullptr
LLVMTypeRef(*	EnzymeDefaultTapeType )(LLVMContextRef) = nullptr
LLVMValueRef(*	EnzymeUndefinedValueForType )(LLVMModuleRef, LLVMTypeRef, uint8_t) = nullptr
LLVMValueRef(*	EnzymeSanitizeDerivatives )(LLVMValueRef, LLVMValueRef toset, LLVMBuilderRef, LLVMValueRef) = nullptr
llvm::cl::opt< bool >	EnzymeZeroCache

Macro Definition Documentation

addAttribute

#define addAttribute   addAttributeAtIndex

Definition at line 112 of file Utils.cpp.

getAttribute

#define getAttribute   getAttributeAtIndex

Definition at line 113 of file Utils.cpp.

Function Documentation

addValueToCache()

void addValueToCache	(	llvm::Value *	arg,
		bool	cache_arg,
		llvm::Type *	ty,
		llvm::SmallVectorImpl< llvm::Value * > &	cacheValues,
		llvm::IRBuilder<> &	BuilderZ,
		const Twine &	name )

Definition at line 3738 of file Utils.cpp.

arePointersGuaranteedNoAlias()

llvm::Optional< bool > arePointersGuaranteedNoAlias	(	TargetLibraryInfo &	TLI,
		llvm::AAResults &	AA,
		llvm::LoopInfo &	LI,
		llvm::Value *	op0,
		llvm::Value *	op1,
		bool	offsetAllowed )

Definition at line 4618 of file Utils.cpp.

References allInstructionsBetween(), getBaseObject(), hasMetadata(), isAllocationCall(), isNoAlias(), notCapturedBefore(), and writesToMemoryReadBy().

Referenced by overwritesToMemoryReadBy().

attributeKnownFunctions()

bool attributeKnownFunctions

(

llvm::Function &

F

)

Definition at line 114 of file Utils.cpp.

References addFunctionNoCapture(), Constant, and getFuncName().

Referenced by callMemcpyStridedBlas(), callMemcpyStridedLapack(), copy_lower_to_upper(), EnzymeAttributeKnownFunctions(), getorInsertInnerProd(), and preserveNVVM().

callMemcpyStridedBlas()

void callMemcpyStridedBlas	(	llvm::IRBuilder<> &	B,
		llvm::Module &	M,
		BlasInfo	blas,
		llvm::ArrayRef< llvm::Value * >	args,
		llvm::Type *	copy_retty,
		llvm::ArrayRef< llvm::OperandBundleDef >	bundles )

Create function for type that performs memcpy with a stride using blas copy.

Definition at line 1316 of file Utils.cpp.

References attributeKnownFunctions(), BlasInfo::floatType, GetFunctionFromValue(), BlasInfo::prefix, and BlasInfo::suffix.

callMemcpyStridedLapack()

void callMemcpyStridedLapack	(	llvm::IRBuilder<> &	B,
		llvm::Module &	M,
		BlasInfo	blas,
		llvm::ArrayRef< llvm::Value * >	args,
		llvm::ArrayRef< llvm::OperandBundleDef >	bundles )

Create function for type that performs memcpy using lapack copy.

Definition at line 1336 of file Utils.cpp.

References attributeKnownFunctions(), BlasInfo::floatType, GetFunctionFromValue(), BlasInfo::prefix, and BlasInfo::suffix.

callSPMVDiagUpdate()

void callSPMVDiagUpdate	(	IRBuilder<> &	B,
		Module &	M,
		BlasInfo	blas,
		IntegerType *	IT,
		Type *	BlasCT,
		Type *	BlasFPT,
		Type *	BlasPT,
		Type *	BlasIT,
		Type *	fpTy,
		ArrayRef< Value * >	args,
		ArrayRef< OperandBundleDef >	bundles,
		bool	byRef,
		bool	julia_decl )

Definition at line 1354 of file Utils.cpp.

References addFunctionNoCapture(), BlasInfo::floatType, getFast(), is_lower(), load_if_ref(), and BlasInfo::suffix.

collectOffset()

bool collectOffset	(	GEPOperator *	gep,
		const DataLayout &	DL,
		unsigned	BitWidth,
		MapVector< Value *, APInt > &	VariableOffsets,
		APInt &	ConstantOffset )

Definition at line 4169 of file Utils.cpp.

Referenced by EnzymeComputeByteOffsetOfGEP(), findAllUsersOf(), getBaseAndOffset(), needsReRooting(), and TypeAnalyzer::visitGEPOperator().

constantInBoundsGEPHelper()

static Value * constantInBoundsGEPHelper ( llvm::IRBuilder<> & B, llvm::Type * type, llvm::Value * value, ArrayRef< unsigned > path )

static

Definition at line 4703 of file Utils.cpp.

Referenced by copyNonJLValueInto(), and moveSRetToFromRoots().

copy_lower_to_upper()

void copy_lower_to_upper	(	llvm::IRBuilder<> &	B,
		llvm::Type *	fpType,
		BlasInfo	blas,
		bool	byRef,
		llvm::Value *	layout,
		llvm::Value *	islower,
		llvm::Value *	A,
		llvm::Value *	lda,
		llvm::Value *	N )

Definition at line 1184 of file Utils.cpp.

References addFunctionNoCapture(), attributeKnownFunctions(), CreateSelect(), BlasInfo::floatType, getFunctionFromCall(), GetFunctionFromValue(), lookup_with_layout(), BlasInfo::prefix, BlasInfo::suffix, and to_blas_callconv().

copyNonJLValueInto()

void copyNonJLValueInto	(	llvm::IRBuilder<> &	B,
		llvm::Type *	curType,
		llvm::Type *	dstType,
		llvm::Value *	dst,
		llvm::ArrayRef< unsigned >	dstPrefix0,
		llvm::Type *	srcType,
		llvm::Value *	src,
		llvm::ArrayRef< unsigned >	srcPrefix0,
		bool	shouldZero )

Definition at line 4837 of file Utils.cpp.

References constantInBoundsGEPHelper(), CountTrackedPointers::count, and getUndefinedValueForType().

Referenced by EnzymeFixupJuliaCallingConvention().

CreateAllocation()

Value * CreateAllocation	(	IRBuilder<> &	Builder,
		llvm::Type *	T,
		Value *	Count,
		const Twine &	Name,
		CallInst **	caller,
		Instruction **	ZeroMem,
		bool	isDefault )

Definition at line 619 of file Utils.cpp.

References Count, CustomAllocator, and getIntrinsicDeclaration().

Referenced by CacheUtility::createCacheForScope(), CacheUtility::getCachePointer(), getOrInsertDifferentialWaitallSave(), getOrInsertExponentialAllocator(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), and UpgradeAllocasToMallocs().

CreateDealloc()

CallInst * CreateDealloc	(	llvm::IRBuilder<> &	Builder,
		llvm::Value *	ToFree )

Definition at line 742 of file Utils.cpp.

References CustomDeallocator, and getInt8PtrTy().

Referenced by DiffeGradientUtils::freeCache(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), and AdjointGenerator::visitOMPCall().

createIntrinsicCall()

llvm::CallInst * createIntrinsicCall	(	llvm::IRBuilderBase &	B,
		llvm::Intrinsic::ID	ID,
		llvm::Type *	RetTy,
		llvm::ArrayRef< llvm::Value * >	Args,
		llvm::Instruction *	FMFSource,
		const llvm::Twine &	Name )

Definition at line 4233 of file Utils.cpp.

References Args.

CreateReAllocation()

llvm::Value * CreateReAllocation	(	llvm::IRBuilder<> &	B,
		llvm::Value *	prev,
		llvm::Type *	T,
		llvm::Value *	OuterCount,
		llvm::Value *	InnerCount,
		const llvm::Twine &	Name,
		llvm::CallInst **	caller,
		bool	ZeroMem )

Definition at line 590 of file Utils.cpp.

References getOrInsertExponentialAllocator().

Referenced by CacheUtility::createCacheForScope().

dumpBlock()

void dumpBlock

(

llvm::BasicBlock *

blk

)

Definition at line 4477 of file Utils.cpp.

dumpModule()

void dumpModule

(

llvm::Module *

mod

)

Definition at line 4473 of file Utils.cpp.

dumpType()

void dumpType

(

llvm::Type *

ty

)

Definition at line 4479 of file Utils.cpp.

dumpTypeResults()

void dumpTypeResults

(

TypeResults &

TR

)

Definition at line 4481 of file Utils.cpp.

References TypeResults::dump().

dumpValue()

void dumpValue

(

llvm::Value *

val

)

Definition at line 4475 of file Utils.cpp.

emit_backtrace()

void emit_backtrace	(	llvm::Instruction *	inst,
		llvm::raw_ostream &	ss )

Definition at line 835 of file Utils.cpp.

Referenced by EmitNoDerivativeError(), EmitNoDerivativeError(), EmitNoTypeError(), ErrorIfRuntimeInactive(), and SanitizeDerivatives().

EmitNoDerivativeError() [1/2]

llvm::Value * EmitNoDerivativeError	(	const std::string &	message,
		llvm::Instruction &	inst,
		GradientUtils *	gutils,
		llvm::IRBuilder<> &	Builder2,
		llvm::Value *	condition )

Definition at line 4295 of file Utils.cpp.

References contains(), CustomErrorHandler, TypeResults::dump(), emit_backtrace(), EmitFailure(), EnzymeRuntimeError(), getString(), NoDerivative, str(), and GradientUtils::TR.

Referenced by AdjointGenerator::createBinaryOperatorAdjoint(), AdjointGenerator::createBinaryOperatorDual(), AdjointGenerator::handleAdjointForIntrinsic(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), AdjointGenerator::visitAtomicRMWInst(), AdjointGenerator::visitCastInst(), AdjointGenerator::visitInstruction(), and AdjointGenerator::visitMemSetCommon().

EmitNoDerivativeError() [2/2]

bool EmitNoDerivativeError	(	const std::string &	message,
		Value *	todiff,
		RequestContext &	context )

Definition at line 4334 of file Utils.cpp.

References CustomErrorHandler, emit_backtrace(), EmitFailure(), EnzymeRuntimeError(), getString(), RequestContext::ip, NoDerivative, RequestContext::req, and str().

EmitNoTypeError()

void EmitNoTypeError	(	const std::string &	message,
		llvm::Instruction &	inst,
		GradientUtils *	gutils,
		llvm::IRBuilder<> &	Builder2 )

Definition at line 4377 of file Utils.cpp.

References TypeResults::analyzer, CustomErrorHandler, TypeResults::dump(), emit_backtrace(), EmitFailure(), EnzymeRuntimeError(), getString(), NoType, str(), and GradientUtils::TR.

Referenced by AdjointGenerator::visitCastInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitExtractValueInst(), AdjointGenerator::visitInsertValueInst(), AdjointGenerator::visitLoadLike(), AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitMemTransferCommon().

EnzymeBlasCopy()

llvm::cl::opt< bool > EnzymeBlasCopy	(	"enzyme-blas-copy"	,
		cl::init(true)	,
		cl::Hidden	,
		cl::desc("Use blas copy calls to cache vectors")	)

EnzymeCheckDerivativeNaN()

llvm::cl::opt< bool > EnzymeCheckDerivativeNaN	(	"enzyme-check-nan"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Add NaN checks to all derivative intermediate values")	)

Referenced by SanitizeDerivatives().

EnzymeFastMath()

llvm::cl::opt< bool > EnzymeFastMath	(	"enzyme-fast-math"	,
		cl::init(true)	,
		cl::Hidden	,
		cl::desc("Use fast math on derivative compuation")	)

Referenced by getFast().

EnzymeLapackCopy()

llvm::cl::opt< bool > EnzymeLapackCopy	(	"enzyme-lapack-copy"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Use blas copy calls to cache matrices")	)

EnzymeMemmoveWarning()

llvm::cl::opt< bool > EnzymeMemmoveWarning	(	"enzyme-memmove-warning"	,
		cl::init(true)	,
		cl::Hidden	,
		cl::desc("Warn if using memmove implementation as a fallback for memmove")	)

Referenced by getOrInsertDifferentialFloatMemmove().

EnzymeNonPower2Cache()

llvm::cl::opt< bool > EnzymeNonPower2Cache	(	"enzyme-non-power2-cache"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Disable caching of integers which are not a power of 2")	)

EnzymeRuntimeError()

llvm::cl::opt< bool > EnzymeRuntimeError	(	"enzyme-runtime-error"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Emit Runtime errors instead of compile time ones")	)

Referenced by EmitNoDerivativeError(), EmitNoDerivativeError(), and EmitNoTypeError().

ErrorIfRuntimeInactive()

void ErrorIfRuntimeInactive	(	llvm::IRBuilder<> &	B,
		llvm::Value *	primal,
		llvm::Value *	shadow,
		const char *	Message,
		llvm::DebugLoc &&	loc,
		llvm::Instruction *	orig )

Definition at line 902 of file Utils.cpp.

References addFunctionNoCapture(), CustomRuntimeInactiveError, emit_backtrace(), getInt8PtrTy(), getString(), and str().

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

extractBLAS()

llvm::Optional< BlasInfo > extractBLAS

(

llvm::StringRef

in

)

Definition at line 3563 of file Utils.cpp.

References str().

Referenced by AdjointGenerator::handleKnownCallDerivatives(), and TypeAnalyzer::visitCallBase().

findAllUsersOf()

SmallVector< std::tuple< Instruction *, Value *, size_t >, 1 > findAllUsersOf

(

Value *

AI

)

Definition at line 3210 of file Utils.cpp.

References collectOffset().

Referenced by getAllLoadedValuesFrom(), and needsReRooting().

get1ULP()

llvm::Value * get1ULP	(	llvm::IRBuilder<> &	builder,
		llvm::Value *	res )

Definition at line 4272 of file Utils.cpp.

get_blas_row() [1/2]

SmallVector< llvm::Value *, 1 > get_blas_row	(	llvm::IRBuilder<> &	B,
		ArrayRef< llvm::Value * >	transA,
		ArrayRef< llvm::Value * >	row,
		ArrayRef< llvm::Value * >	col,
		bool	byRef,
		bool	cublas )

Definition at line 4116 of file Utils.cpp.

References get_blas_row().

get_blas_row() [2/2]

SmallVector< llvm::Value *, 1 > get_blas_row	(	llvm::IRBuilder<> &	B,
		ArrayRef< llvm::Value * >	transA,
		bool	byRef,
		bool	cublas )

Definition at line 4089 of file Utils.cpp.

Referenced by get_blas_row().

get_cached_mat_width()

llvm::Value * get_cached_mat_width	(	llvm::IRBuilder<> &	B,
		llvm::ArrayRef< llvm::Value * >	trans,
		llvm::Value *	arg_ld,
		llvm::Value *	dim1,
		llvm::Value *	dim2,
		bool	cacheMat,
		bool	byRef,
		bool	cublas )

Definition at line 4018 of file Utils.cpp.

References CreateSelect(), and is_normal().

getAllLoadedValuesFrom()

SmallVector< std::pair< Value *, size_t >, 1 > getAllLoadedValuesFrom	(	AllocaInst *	ptr0,
		size_t	offset,
		size_t	valSz,
		bool &	legal )

Definition at line 3255 of file Utils.cpp.

References findAllUsersOf(), getAllLoadedValuesFrom(), and getBaseAndOffset().

Referenced by getAllLoadedValuesFrom(), and simplifyLoad().

getBaseAndOffset()

AllocaInst * getBaseAndOffset	(	Value *	ptr,
		size_t &	offset )

Definition at line 3167 of file Utils.cpp.

References collectOffset(), and simplifyLoad().

Referenced by getAllLoadedValuesFrom(), and simplifyLoad().

getDefaultAnonymousTapeType()

llvm::PointerType * getDefaultAnonymousTapeType

(

llvm::LLVMContext &

C

)

Definition at line 437 of file Utils.cpp.

References EnzymeDefaultTapeType, and getInt8PtrTy().

Referenced by getFunctionTypeForClone(), and traceType().

getFast()

llvm::FastMathFlags getFast

(

)

Get LLVM fast math flags.

Definition at line 3731 of file Utils.cpp.

References EnzymeFastMath().

Referenced by callSPMVDiagUpdate(), CacheUtility::createCacheForScope(), fixSparse_inner(), fixSparseIndices(), DiffeGradientUtils::freeCache(), DiffeGradientUtils::getDifferential(), getOrInsertDifferentialFloatMemcpy(), getorInsertInnerProd(), getOrInsertMemcpyStrided(), getOrInsertOpFloatSum(), AdjointGenerator::handleKnownCallDerivatives(), AdjointGenerator::handleMPI(), AdjointGenerator::recursivelyHandleSubfunction(), CacheUtility::storeInstructionInCache(), AdjointGenerator::visitCallInst(), AdjointGenerator::visitCommonStore(), AdjointGenerator::visitIntrinsicInst(), and AdjointGenerator::visitOMPCall().

getFirstFunctionDefinition()

Function * getFirstFunctionDefinition

(

Module &

M

)

Definition at line 3551 of file Utils.cpp.

GetFunctionFromValue()

Function * GetFunctionFromValue

(

Value *

fn

)

Definition at line 3547 of file Utils.cpp.

References GetFunctionValFromValue().

Referenced by callMemcpyStridedBlas(), callMemcpyStridedLapack(), copy_lower_to_upper(), getorInsertInnerProd(), TraceGenerator::handleObserveCall(), and TraceGenerator::handleSampleCall().

GetFunctionValFromValue()

Value * GetFunctionValFromValue

(

Value *

fn

)

Definition at line 3466 of file Utils.cpp.

References Call, GetFunctionValFromValue(), and simplifyLoad().

Referenced by GetFunctionFromValue(), and GetFunctionValFromValue().

getJuliaObjects()

llvm::SmallVector< llvm::Value *, 1 > getJuliaObjects	(	llvm::Value *	v,
		llvm::IRBuilder<> &	B )

Definition at line 4913 of file Utils.cpp.

References anyJuliaObjects(), and isSpecialPtr().

Referenced by RecursivelyReplaceAddressSpace().

getOrInsertCheckedFree()

Function * getOrInsertCheckedFree	(	Module &	M,
		CallInst *	call,
		Type *	Ty,
		unsigned	width )

Definition at line 2084 of file Utils.cpp.

References addFunctionNoCapture(), and getFuncNameFromCall().

getOrInsertDifferentialFloatMemcpy()

Function * getOrInsertDifferentialFloatMemcpy	(	Module &	M,
		Type *	elementType,
		unsigned	dstalign,
		unsigned	srcalign,
		unsigned	dstaddr,
		unsigned	srcaddr,
		unsigned	bitwidth )

Create function for type that is equivalent to memcpy but adds to destination rather than a direct copy; dst, src, numelems.

Definition at line 1009 of file Utils.cpp.

References addFunctionNoCapture(), getFast(), and tofltstr().

Referenced by AdjointGenerator::DifferentiableMemCopyFloats(), and getOrInsertDifferentialFloatMemmove().

getOrInsertDifferentialFloatMemcpyMat()

Function * getOrInsertDifferentialFloatMemcpyMat	(	Module &	Mod,
		Type *	elementType,
		PointerType *	PT,
		IntegerType *	IT,
		IntegerType *	CT,
		unsigned	dstalign,
		unsigned	srcalign,
		bool	zeroSrc )

Definition at line 1913 of file Utils.cpp.

References tofltstr().

getOrInsertDifferentialFloatMemmove()

Function * getOrInsertDifferentialFloatMemmove	(	Module &	M,
		Type *	T,
		unsigned	dstalign,
		unsigned	srcalign,
		unsigned	dstaddr,
		unsigned	srcaddr,
		unsigned	bitwidth )

Definition at line 2073 of file Utils.cpp.

References EnzymeMemmoveWarning(), and getOrInsertDifferentialFloatMemcpy().

getOrInsertDifferentialMPI_Wait()

llvm::Function * getOrInsertDifferentialMPI_Wait	(	llvm::Module &	M,
		ArrayRef< llvm::Type * >	T,
		Type *	reqType,
		StringRef	caller )

Definition at line 2271 of file Utils.cpp.

References getInt8PtrTy(), getRenamedPerCallingConv(), ISEND, str(), and tripleSplitDollar().

Referenced by AdjointGenerator::handleMPI().

getOrInsertDifferentialWaitallSave()

llvm::Function * getOrInsertDifferentialWaitallSave	(	llvm::Module &	M,
		ArrayRef< llvm::Type * >	T,
		PointerType *	reqType )

Definition at line 2203 of file Utils.cpp.

References CreateAllocation(), and getUnqual().

Referenced by AdjointGenerator::handleMPI().

getOrInsertExponentialAllocator()

Function * getOrInsertExponentialAllocator	(	Module &	M,
		Function *	newFunc,
		bool	ZeroInit,
		llvm::Type *	RT )

Definition at line 443 of file Utils.cpp.

References CreateAllocation(), getFunctionFromCall(), and getIntrinsicDeclaration().

Referenced by CreateReAllocation().

getorInsertInnerProd()

llvm::CallInst * getorInsertInnerProd	(	llvm::IRBuilder<> &	B,
		llvm::Module &	M,
		BlasInfo	blas,
		IntegerType *	IT,
		Type *	BlasPT,
		Type *	BlasIT,
		Type *	fpTy,
		llvm::ArrayRef< llvm::Value * >	args,
		const llvm::ArrayRef< llvm::OperandBundleDef >	bundles,
		bool	byRef,
		bool	cublas,
		bool	julia_decl )

Definition at line 1542 of file Utils.cpp.

References addFunctionNoCapture(), attributeKnownFunctions(), BlasInfo::floatType, getFast(), GetFunctionFromValue(), load_if_ref(), BlasInfo::prefix, BlasInfo::suffix, and to_blas_callconv().

getOrInsertMemcpyMat()

Function * getOrInsertMemcpyMat	(	Module &	Mod,
		Type *	elementType,
		PointerType *	PT,
		IntegerType *	IT,
		unsigned	dstalign,
		unsigned	srcalign )

Definition at line 1793 of file Utils.cpp.

References addFunctionNoCapture(), and tofltstr().

getOrInsertMemcpyStrided()

Function * getOrInsertMemcpyStrided	(	Module &	M,
		Type *	elementType,
		PointerType *	T,
		Type *	IT,
		unsigned	dstalign,
		unsigned	srcalign )

Definition at line 1691 of file Utils.cpp.

References addFunctionNoCapture(), getFast(), and tofltstr().

getOrInsertOpFloatSum()

llvm::Value * getOrInsertOpFloatSum	(	llvm::Module &	M,
		llvm::Type *	OpPtr,
		llvm::Type *	OpType,
		ConcreteType	CT,
		llvm::Type *	intType,
		IRBuilder<> &	B2 )

Definition at line 2378 of file Utils.cpp.

References addFunctionNoCapture(), Constant, getFast(), getInt8PtrTy(), getUnqual(), ConcreteType::isFloat(), and ConcreteType::str().

Referenced by AdjointGenerator::handleMPI().

getString()

Constant * getString	(	Module &	M,
		StringRef	Str )

Definition at line 825 of file Utils.cpp.

Referenced by EmitNoDerivativeError(), EmitNoDerivativeError(), EmitNoTypeError(), ErrorIfRuntimeInactive(), and SanitizeDerivatives().

getUndefinedValueForType()

llvm::Constant * getUndefinedValueForType	(	llvm::Module &	M,
		llvm::Type *	T,
		bool	forceZero )

Definition at line 3623 of file Utils.cpp.

References EnzymeUndefinedValueForType, and EnzymeZeroCache.

Referenced by copyNonJLValueInto(), CacheUtility::createCacheForScope(), AdjointGenerator::handleKnownCallDerivatives(), moveSRetToFromRoots(), and AdjointGenerator::recursivelyHandleSubfunction().

is_left()

llvm::Value * is_left	(	IRBuilder<> &	B,
		llvm::Value *	side,
		bool	byRef,
		bool	cublas )

Definition at line 3904 of file Utils.cpp.

is_lower()

Value * is_lower	(	IRBuilder<> &	B,
		Value *	uplo,
		bool	byRef,
		bool	cublas )

Definition at line 3800 of file Utils.cpp.

Referenced by callSPMVDiagUpdate().

is_nonunit()

Value * is_nonunit	(	IRBuilder<> &	B,
		Value *	uplo,
		bool	byRef,
		bool	cublas )

Definition at line 3834 of file Utils.cpp.

is_normal()

llvm::Value * is_normal	(	IRBuilder<> &	B,
		llvm::Value *	trans,
		bool	byRef,
		bool	cublas )

Definition at line 3868 of file Utils.cpp.

Referenced by get_cached_mat_width().

isNVLoad()

bool isNVLoad

(

const llvm::Value *

V

)

Definition at line 4483 of file Utils.cpp.

load_if_ref()

llvm::Value * load_if_ref	(	llvm::IRBuilder<> &	B,
		llvm::Type *	intType,
		llvm::Value *	V,
		bool	byRef )

Definition at line 4075 of file Utils.cpp.

References getUnqual().

Referenced by callSPMVDiagUpdate(), and getorInsertInnerProd().

lookup_with_layout()

Value * lookup_with_layout	(	IRBuilder<> &	B,
		Type *	fpType,
		Value *	layout,
		Value *const	base,
		Value *	lda,
		Value *	row,
		Value *	col )

Definition at line 1135 of file Utils.cpp.

References CreateSelect(), and getUnqual().

Referenced by copy_lower_to_upper().

mayExecuteAfter()

void mayExecuteAfter	(	llvm::SmallVectorImpl< llvm::Instruction * > &	results,
		llvm::Instruction *	inst,
		const llvm::SmallPtrSetImpl< Instruction * > &	stores,
		const llvm::Loop *	region )

Definition at line 2513 of file Utils.cpp.

moveSRetToFromRoots()

llvm::Value * moveSRetToFromRoots	(	llvm::IRBuilder<> &	B,
		llvm::Type *	jltype,
		llvm::Value *	sret,
		llvm::Type *	root_ty,
		llvm::Value *	rootRet,
		size_t	rootOffset,
		SRetRootMovement	direction )

Definition at line 4714 of file Utils.cpp.

References constantInBoundsGEPHelper(), CountTrackedPointers::count, GradientUtils::extractMeta(), getBaseObject(), getUndefinedValueForType(), isSpecialPtr(), NullifySRetValue, RootPointerToSRetPointer, RootPointerToSRetValue, SRetPointerToRootPointer, SRetValueToRootPointer, and Tracked.

Referenced by EnzymeFixupJuliaCallingConvention().

nextPowerOfTwo()

llvm::Value * nextPowerOfTwo	(	llvm::IRBuilder<> &	B,
		llvm::Value *	V )

Create function to computer nearest power of two.

Definition at line 2192 of file Utils.cpp.

Referenced by PreProcessCache::ReplaceReallocs().

notCaptured()

bool notCaptured

(

llvm::Value *

V

)

Check if value if b captured.

Definition at line 4608 of file Utils.cpp.

References notCapturedBefore().

Referenced by DetectReadonlyOrThrowFn().

notCapturedBefore()

bool notCapturedBefore	(	llvm::Value *	V,
		Instruction *	inst,
		size_t	checkLoadCaptures )

Definition at line 4503 of file Utils.cpp.

References isNoCapture(), and isPointerArithmeticInst().

Referenced by arePointersGuaranteedNoAlias(), and notCaptured().

overwritesToMemoryReadBy()

bool overwritesToMemoryReadBy	(	const TypeResults *	TR,
		llvm::AAResults &	AA,
		llvm::TargetLibraryInfo &	TLI,
		ScalarEvolution &	SE,
		llvm::LoopInfo &	LI,
		llvm::DominatorTree &	DT,
		llvm::Instruction *	maybeReader,
		llvm::Instruction *	maybeWriter,
		llvm::Loop *	scope )

Definition at line 2765 of file Utils.cpp.

References arePointersGuaranteedNoAlias(), overwritesToMemoryReadByLoop(), and writesToMemoryReadBy().

overwritesToMemoryReadByLoop()

bool overwritesToMemoryReadByLoop	(	llvm::ScalarEvolution &	SE,
		llvm::LoopInfo &	LI,
		llvm::DominatorTree &	DT,
		llvm::Instruction *	maybeReader,
		const llvm::SCEV *	LoadStart,
		const llvm::SCEV *	LoadEnd,
		llvm::Instruction *	maybeWriter,
		const llvm::SCEV *	StoreStart,
		const llvm::SCEV *	StoreEnd,
		llvm::Loop *	scope )

We force all ranges for all loops in range ... [scope, anc], .... cur to expand the number of iterations

Definition at line 2574 of file Utils.cpp.

References getAncestor().

Referenced by overwritesToMemoryReadBy().

parseTrueType()

std::vector< std::tuple< llvm::Type *, size_t, size_t > > parseTrueType	(	const llvm::MDNode *	md,
		DerivativeMode	Mode,
		bool	const_src )

Definition at line 4411 of file Utils.cpp.

References Anything, ForwardMode, and ForwardModeError.

Referenced by AdjointGenerator::visitMemSetCommon(), and AdjointGenerator::visitMemTransferCommon().

PostCacheStore()

llvm::SmallVector< llvm::Instruction *, 2 > PostCacheStore	(	llvm::StoreInst *	SI,
		llvm::IRBuilder<> &	B )

Definition at line 423 of file Utils.cpp.

References EnzymePostCacheStore.

Referenced by CacheUtility::createCacheForScope(), EnzymeFixupJuliaCallingConvention(), and RecursivelyReplaceAddressSpace().

SanitizeDerivatives()

llvm::Value * SanitizeDerivatives	(	llvm::Value *	val,
		llvm::Value *	toset,
		llvm::IRBuilder<> &	BuilderM,
		llvm::Value *	mask )

Definition at line 3634 of file Utils.cpp.

References CustomErrorHandler, emit_backtrace(), EnzymeCheckDerivativeNaN(), EnzymeSanitizeDerivatives, getInt8PtrTy(), getString(), NaNError, and str().

Referenced by DiffeGradientUtils::setDiffe().

simplifyLoad()

Value * simplifyLoad	(	Value *	V,
		size_t	valSz,
		size_t	preOffset )

Definition at line 3386 of file Utils.cpp.

References GradientUtils::extractMeta(), getAllLoadedValuesFrom(), getBaseAndOffset(), getUnqual(), and simplifyLoad().

Referenced by getBaseAndOffset(), GetFunctionValFromValue(), OldAllocationSize(), and simplifyLoad().

to_blas_callconv()

llvm::Value * to_blas_callconv	(	IRBuilder<> &	B,
		llvm::Value *	V,
		bool	byRef,
		bool	cublas,
		IntegerType *	julia_decl,
		IRBuilder<> &	entryBuilder,
		llvm::Twine const &	name )

Definition at line 3767 of file Utils.cpp.

References getInt8PtrTy().

Referenced by copy_lower_to_upper(), getorInsertInnerProd(), and transpose().

to_blas_fp_callconv()

llvm::Value * to_blas_fp_callconv	(	IRBuilder<> &	B,
		llvm::Value *	V,
		bool	byRef,
		Type *	fpTy,
		IRBuilder<> &	entryBuilder,
		llvm::Twine const &	name )

Definition at line 3784 of file Utils.cpp.

tofltstr()

static std::string tofltstr ( Type * T )

inlinestatic

Convert a floating type to a string.

Definition at line 796 of file Utils.cpp.

References tofltstr().

Referenced by getOrInsertDifferentialFloatMemcpy(), getOrInsertDifferentialFloatMemcpyMat(), getOrInsertMemcpyMat(), getOrInsertMemcpyStrided(), and tofltstr().

transpose() [1/2]

llvm::Value * transpose	(	std::string	floatType,
		IRBuilder<> &	B,
		llvm::Value *	V,
		bool	cublas )

Definition at line 3947 of file Utils.cpp.

References CustomErrorHandler, EmitFailure(), and NoDerivative.

Referenced by transpose().

transpose() [2/2]

llvm::Value * transpose	(	std::string	floatType,
		llvm::IRBuilder<> &	B,
		llvm::Value *	V,
		bool	byRef,
		bool	cublas,
		llvm::IntegerType *	julia_decl,
		llvm::IRBuilder<> &	entryBuilder,
		const llvm::Twine &	name )

Definition at line 4034 of file Utils.cpp.

References to_blas_callconv(), and transpose().

writesToMemoryReadBy()

bool writesToMemoryReadBy	(	const TypeResults *	TR,
		llvm::AAResults &	AA,
		llvm::TargetLibraryInfo &	TLI,
		llvm::Instruction *	maybeReader,
		llvm::Instruction *	maybeWriter )

Return whether maybeReader can read from memory written to by maybeWriter.

Definition at line 2880 of file Utils.cpp.

References Anything, Constant, getFuncNameFromCall(), isAllocationFunction(), isCertainPrint(), isDeallocationFunction(), isDebugFunction(), ConcreteType::isKnown(), isMemFreeLibMFunction(), TypeTree::Lookup(), parseTBAA(), Pointer, TypeResults::query(), and Unknown.

Referenced by arePointersGuaranteedNoAlias(), and overwritesToMemoryReadBy().

ZeroMemory()

void ZeroMemory	(	llvm::IRBuilder<> &	Builder,
		llvm::Type *	T,
		llvm::Value *	obj,
		bool	isTape )

Definition at line 414 of file Utils.cpp.

References CustomZero.

Referenced by DiffeGradientUtils::getDifferential().

Variable Documentation

CustomAllocator

LLVMValueRef(* CustomAllocator) (LLVMBuilderRef, LLVMTypeRef, LLVMValueRef, LLVMValueRef, uint8_t, LLVMValueRef *)	(	LLVMBuilderRef	,
		LLVMTypeRef	,
		LLVMValueRef	,
		LLVMValueRef	,
		uint8_t	,
		LLVMValueRef *	) = nullptr

Definition at line 65 of file Utils.cpp.

Referenced by CreateAllocation().

CustomDeallocator

LLVMValueRef(* CustomDeallocator) (LLVMBuilderRef, LLVMValueRef)	(	LLVMBuilderRef	,
		LLVMValueRef	) = nullptr

Definition at line 71 of file Utils.cpp.

Referenced by CreateDealloc().

CustomErrorHandler

LLVMValueRef(* CustomErrorHandler) (const char *, LLVMValueRef, ErrorType, const void *, LLVMValueRef, LLVMBuilderRef)	(	const char *	,
		LLVMValueRef	,
		ErrorType	,
		const void *	,
		LLVMValueRef	,
		LLVMBuilderRef	) = nullptr

Definition at line 62 of file Utils.cpp.

Referenced by analyzeIntelSubscriptIntrinsic(), EmitNoDerivativeError(), EmitNoDerivativeError(), EmitNoTypeError(), EnzymeFixupJuliaCallingConvention(), CacheUtility::erase(), TypeResults::firstPointer(), TypeTree::insert(), needsReRooting(), TypeTree::Only(), AdjointGenerator::recursivelyHandleSubfunction(), RecursivelyReplaceAddressSpace(), SanitizeDerivatives(), transpose(), TypeAnalyzer::visitBinaryOperation(), AdjointGenerator::visitCommonStore(), TypeAnalyzer::visitGEPOperator(), TypeAnalyzer::visitIntrinsicInst(), TypeAnalyzer::visitMemTransferCommon(), and TypeAnalyzer::visitPHINode().

CustomRuntimeInactiveError

void(* CustomRuntimeInactiveError) (LLVMBuilderRef, LLVMValueRef, LLVMValueRef)	(	LLVMBuilderRef	,
		LLVMValueRef	,
		LLVMValueRef	) = nullptr

Definition at line 72 of file Utils.cpp.

Referenced by ErrorIfRuntimeInactive().

CustomZero

void(* CustomZero) (LLVMBuilderRef, LLVMTypeRef, LLVMValueRef, uint8_t)	(	LLVMBuilderRef	,
		LLVMTypeRef	,
		LLVMValueRef	,
		uint8_t	) = nullptr

Definition at line 69 of file Utils.cpp.

Referenced by ZeroMemory().

EnzymeDefaultTapeType

LLVMTypeRef(* EnzymeDefaultTapeType) (LLVMContextRef)

(

LLVMContextRef

)

= nullptr

Definition at line 76 of file Utils.cpp.

Referenced by getDefaultAnonymousTapeType().

EnzymePostCacheStore

LLVMValueRef *(* EnzymePostCacheStore) (LLVMValueRef, LLVMBuilderRef, uint64_t *size)	(	LLVMValueRef	,
		LLVMBuilderRef	,
		uint64_t *	size ) = nullptr

Definition at line 74 of file Utils.cpp.

Referenced by PostCacheStore().

EnzymeSanitizeDerivatives

LLVMValueRef(* EnzymeSanitizeDerivatives) (LLVMValueRef, LLVMValueRef toset, LLVMBuilderRef, LLVMValueRef)	(	LLVMValueRef	,
		LLVMValueRef	toset,
		LLVMBuilderRef	,
		LLVMValueRef	) = nullptr

Definition at line 80 of file Utils.cpp.

Referenced by SanitizeDerivatives().

EnzymeUndefinedValueForType

LLVMValueRef(* EnzymeUndefinedValueForType) (LLVMModuleRef, LLVMTypeRef, uint8_t)	(	LLVMModuleRef	,
		LLVMTypeRef	,
		uint8_t	) = nullptr

Definition at line 77 of file Utils.cpp.

Referenced by getUndefinedValueForType().

EnzymeZeroCache

llvm::cl::opt<bool> EnzymeZeroCache

extern

Referenced by getUndefinedValueForType().