#include "EnzymeLogic.h"
#include "ActivityAnalysis.h"
#include "AdjointGenerator.h"
#include "TypeAnalysis/TypeAnalysis.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/Support/ErrorHandling.h"
#include <cmath>
#include "SCEV/ScalarEvolution.h"
#include "SCEV/ScalarEvolutionExpander.h"
#include "llvm/Analysis/DependenceAnalysis.h"
#include <deque>
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Demangle/Demangle.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Support/AMDGPUMetadata.h"
#include "llvm/Support/TimeProfiler.h"
#include "llvm/ADT/StringSet.h"
#include "DiffeGradientUtils.h"
#include "FunctionUtils.h"
#include "GradientUtils.h"
#include "InstructionBatcher.h"
#include "LibraryFuncs.h"
#include "TraceGenerator.h"
#include "Utils.h"
#include "Dialect/Ops.h"
#include "Implementations/CoreDialectsAutoDiffImplementations.h"
#include "Interfaces/AutoDiffOpInterface.h"
#include "Interfaces/AutoDiffTypeInterface.h"
#include "Interfaces/GradientUtils.h"
#include "Interfaces/GradientUtilsReverse.h"
#include "mlir/IR/Matchers.h"
#include "mlir/Interfaces/FunctionInterfaces.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/IR/Dominance.h"
#include "mlir/Pass/PassManager.h"
#include "mlir/Pass/PassRegistry.h"
#include "llvm/ADT/BreadthFirstIterator.h"
#include "EnzymeLogic.h"

Classes
struct	CacheAnalysis

class	TruncateUtils

class	TruncateGenerator

Macros
#define	addAttribute addAttributeAtIndex

#define	getAttribute getAttributeAtIndex

#define	removeAttribute removeAttributeAtIndex

Functions
llvm::cl::opt< bool >	EnzymePrint ("enzyme-print", cl::init(false), cl::Hidden, cl::desc("Print before and after fns for autodiff"))

llvm::cl::opt< bool >	EnzymePrintUnnecessary ("enzyme-print-unnecessary", cl::init(false), cl::Hidden, cl::desc("Print unnecessary values in function"))

cl::opt< bool >	looseTypeAnalysis ("enzyme-loose-types", cl::init(false), cl::Hidden, cl::desc("Allow looser use of types"))

cl::opt< bool >	nonmarkedglobals_inactiveloads ("enzyme_nonmarkedglobals_inactiveloads", cl::init(true), cl::Hidden, cl::desc("Consider loads of nonmarked globals to be inactive"))

cl::opt< bool >	EnzymeJuliaAddrLoad ("enzyme-julia-addr-load", cl::init(false), cl::Hidden, cl::desc("Mark all loads resulting in an addr(13)* to be legal to redo"))

cl::opt< bool >	EnzymeAssumeUnknownNoFree ("enzyme-assume-unknown-nofree", cl::init(false), cl::Hidden, cl::desc("Assume unknown instructions are nofree as needed"))

void	calculateUnusedValuesInFunction (Function &func, llvm::SmallPtrSetImpl< const Value * > &unnecessaryValues, llvm::SmallPtrSetImpl< const Instruction * > &unnecessaryInstructions, bool returnValue, DerivativeMode mode, GradientUtils gutils, TargetLibraryInfo &TLI, ArrayRef< DIFFE_TYPE > constant_args, const llvm::SmallPtrSetImpl< BasicBlock > &oldUnreachable)

void	calculateUnusedStoresInFunction (Function &func, llvm::SmallPtrSetImpl< const Instruction * > &unnecessaryStores, const llvm::SmallPtrSetImpl< const Instruction * > &unnecessaryInstructions, GradientUtils *gutils, TargetLibraryInfo &TLI)

std::string	to_string (Function &F, const std::vector< bool > &us)

std::pair< SmallVector< Type , 4 >, SmallVector< Type , 4 > >	getDefaultFunctionTypeForAugmentation (FunctionType *called, bool returnUsed, DIFFE_TYPE retType)
	assuming not top level

std::pair< SmallVector< Type , 4 >, SmallVector< Type , 4 > >	getDefaultFunctionTypeForGradient (FunctionType *called, DIFFE_TYPE retType, ArrayRef< DIFFE_TYPE > tys)
	assuming not top level

std::pair< SmallVector< Type , 4 >, SmallVector< Type , 4 > >	getDefaultFunctionTypeForGradient (FunctionType *called, DIFFE_TYPE retType)
	assuming not top level

bool	shouldAugmentCall (CallInst op, const GradientUtils gutils)

bool	legalCombinedForwardReverse (CallInst origop, const std::map< ReturnInst , StoreInst * > &replacedReturns, SmallVectorImpl< Instruction * > &postCreate, SmallVectorImpl< Instruction * > &userReplace, const GradientUtils gutils, const SmallPtrSetImpl< const Instruction > &unnecessaryInstructions, const SmallPtrSetImpl< BasicBlock * > &oldUnreachable, const bool subretused)

void	clearFunctionAttributes (Function *f)

void	cleanupInversionAllocs (DiffeGradientUtils gutils, BasicBlock entry)

void	restoreCache (DiffeGradientUtils gutils, const std::map< std::pair< Instruction , CacheType >, int > &mapping, const SmallPtrSetImpl< BasicBlock * > &guaranteedUnreachable)

void	createTerminator (DiffeGradientUtils gutils, BasicBlock oBB, DIFFE_TYPE retType, bool returnPrimal, bool returnShadow)

Value *	selectByWidth (IRBuilder<> &B, DiffeGradientUtils gutils, Value cond, Value tval, Value fval)

void	createInvertedTerminator (DiffeGradientUtils gutils, ArrayRef< DIFFE_TYPE > argTypes, BasicBlock oBB, AllocaInst retAlloca, AllocaInst dretAlloca, unsigned extraArgs, DIFFE_TYPE retType)

static Value *	floatValTruncate (IRBuilderBase &B, Value *v, FloatTruncation truncation)

static Value *	floatValExpand (IRBuilderBase &B, Value *v, FloatTruncation truncation)

static Value *	floatMemTruncate (IRBuilderBase &B, Value *v, FloatTruncation truncation)

static Value *	floatMemExpand (IRBuilderBase &B, Value *v, FloatTruncation truncation)

void	createTerminator (MGradientUtils gutils, mlir::Block oBB, const ArrayRef< bool > returnPrimals, const ArrayRef< bool > returnShadows)

Variables
LLVMValueRef(*	EnzymeFixupReturn )(LLVMBuilderRef, LLVMValueRef) = nullptr

Macro Definition Documentation

◆ addAttribute

#define addAttribute addAttributeAtIndex

◆ getAttribute

#define getAttribute getAttributeAtIndex

◆ removeAttribute

#define removeAttribute removeAttributeAtIndex

Function Documentation

◆ calculateUnusedStoresInFunction()

void calculateUnusedStoresInFunction	(	Function &	func,
		llvm::SmallPtrSetImpl< const Instruction * > &	unnecessaryStores,
		const llvm::SmallPtrSetImpl< const Instruction * > &	unnecessaryInstructions,
		GradientUtils *	gutils,
		TargetLibraryInfo &	TLI )

Definition at line 1176 of file EnzymeLogic.cpp.

◆ calculateUnusedValuesInFunction()

void calculateUnusedValuesInFunction	(	Function &	func,
		llvm::SmallPtrSetImpl< const Value * > &	unnecessaryValues,
		llvm::SmallPtrSetImpl< const Instruction * > &	unnecessaryInstructions,
		bool	returnValue,
		DerivativeMode	mode,
		GradientUtils *	gutils,
		TargetLibraryInfo &	TLI,
		ArrayRef< DIFFE_TYPE >	constant_args,
		const llvm::SmallPtrSetImpl< BasicBlock * > &	oldUnreachable )

Definition at line 682 of file EnzymeLogic.cpp.

◆ cleanupInversionAllocs()

void cleanupInversionAllocs	(	DiffeGradientUtils *	gutils,
		BasicBlock *	entry )

Definition at line 1804 of file EnzymeLogic.cpp.

◆ clearFunctionAttributes()

void clearFunctionAttributes ( Function * f )

Definition at line 1729 of file EnzymeLogic.cpp.

◆ createInvertedTerminator()

void createInvertedTerminator	(	DiffeGradientUtils *	gutils,
		ArrayRef< DIFFE_TYPE >	argTypes,
		BasicBlock *	oBB,
		AllocaInst *	retAlloca,
		AllocaInst *	dretAlloca,
		unsigned	extraArgs,
		DIFFE_TYPE	retType )

Definition at line 3302 of file EnzymeLogic.cpp.

◆ createTerminator() [1/2]

void createTerminator	(	DiffeGradientUtils *	gutils,
		BasicBlock *	oBB,
		DIFFE_TYPE	retType,
		bool	returnPrimal,
		bool	returnShadow )

Definition at line 3193 of file EnzymeLogic.cpp.

◆ createTerminator() [2/2]

void createTerminator	(	MGradientUtils *	gutils,
		mlir::Block *	oBB,
		const ArrayRef< bool >	returnPrimals,
		const ArrayRef< bool >	returnShadows )

Definition at line 27 of file EnzymeLogic.cpp.

References mlir::enzyme::detail::branchingForwardHandler(), mlir::enzyme::MGradientUtils::erase(), mlir::enzyme::MGradientUtils::getNewFromOriginal(), mlir::enzyme::MGradientUtils::invertPointerM(), and mlir::enzyme::MGradientUtils::isConstantValue().

◆ EnzymeAssumeUnknownNoFree()

cl::opt< bool > EnzymeAssumeUnknownNoFree	(	"enzyme-assume-unknown-nofree"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Assume unknown instructions are nofree as needed")	)

◆ EnzymeJuliaAddrLoad()

cl::opt< bool > EnzymeJuliaAddrLoad	(	"enzyme-julia-addr-load"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Mark all loads resulting in an addr(13)* to be legal to redo")	)

◆ EnzymePrint()

llvm::cl::opt< bool > EnzymePrint	(	"enzyme-print"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Print before and after fns for autodiff")	)

◆ EnzymePrintUnnecessary()

llvm::cl::opt< bool > EnzymePrintUnnecessary	(	"enzyme-print-unnecessary"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Print unnecessary values in function")	)

◆ floatMemExpand()

static Value * floatMemExpand	(	IRBuilderBase &	B,
		Value *	v,
		FloatTruncation	truncation )

static

Definition at line 5170 of file EnzymeLogic.cpp.

◆ floatMemTruncate()

static Value * floatMemTruncate	(	IRBuilderBase &	B,
		Value *	v,
		FloatTruncation	truncation )

static

Definition at line 5161 of file EnzymeLogic.cpp.

◆ floatValExpand()

static Value * floatValExpand	(	IRBuilderBase &	B,
		Value *	v,
		FloatTruncation	truncation )

static

Definition at line 5150 of file EnzymeLogic.cpp.

◆ floatValTruncate()

static Value * floatValTruncate	(	IRBuilderBase &	B,
		Value *	v,
		FloatTruncation	truncation )

static

Definition at line 5139 of file EnzymeLogic.cpp.

◆ getDefaultFunctionTypeForAugmentation()

std::pair< SmallVector< Type , 4 >, SmallVector< Type , 4 > > getDefaultFunctionTypeForAugmentation	(	FunctionType *	called,
		bool	returnUsed,
		DIFFE_TYPE	retType )

assuming not top level

Definition at line 1238 of file EnzymeLogic.cpp.

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

◆ getDefaultFunctionTypeForGradient() [1/2]

std::pair< SmallVector< Type , 4 >, SmallVector< Type , 4 > > getDefaultFunctionTypeForGradient	(	FunctionType *	called,
		DIFFE_TYPE	retType )

assuming not top level

Definition at line 1302 of file EnzymeLogic.cpp.

◆ getDefaultFunctionTypeForGradient() [2/2]

std::pair< SmallVector< Type , 4 >, SmallVector< Type , 4 > > getDefaultFunctionTypeForGradient	(	FunctionType *	called,
		DIFFE_TYPE	retType,
		ArrayRef< DIFFE_TYPE >	tys )

assuming not top level

Definition at line 1268 of file EnzymeLogic.cpp.

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

◆ legalCombinedForwardReverse()

bool legalCombinedForwardReverse	(	CallInst *	origop,
		const std::map< ReturnInst , StoreInst > &	replacedReturns,
		SmallVectorImpl< Instruction * > &	postCreate,
		SmallVectorImpl< Instruction * > &	userReplace,
		const GradientUtils *	gutils,
		const SmallPtrSetImpl< const Instruction * > &	unnecessaryInstructions,
		const SmallPtrSetImpl< BasicBlock * > &	oldUnreachable,
		const bool	subretused )

Definition at line 1391 of file EnzymeLogic.cpp.

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

◆ looseTypeAnalysis()

cl::opt< bool > looseTypeAnalysis	(	"enzyme-loose-types"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Allow looser use of types")	)

◆ nonmarkedglobals_inactiveloads()

cl::opt< bool > nonmarkedglobals_inactiveloads	(	"enzyme_nonmarkedglobals_inactiveloads"	,
		cl::init(true)	,
		cl::Hidden	,
		cl::desc("Consider loads of nonmarked globals to be inactive")	)

◆ restoreCache()

void restoreCache	(	DiffeGradientUtils *	gutils,
		const std::map< std::pair< Instruction *, CacheType >, int > &	mapping,
		const SmallPtrSetImpl< BasicBlock * > &	guaranteedUnreachable )

Definition at line 1823 of file EnzymeLogic.cpp.

◆ selectByWidth()

Value * selectByWidth	(	IRBuilder<> &	B,
		DiffeGradientUtils *	gutils,
		Value *	cond,
		Value *	tval,
		Value *	fval )

Definition at line 3286 of file EnzymeLogic.cpp.

◆ shouldAugmentCall()

bool shouldAugmentCall	(	CallInst *	op,
		const GradientUtils *	gutils )

Definition at line 1315 of file EnzymeLogic.cpp.

Referenced by AdjointGenerator::recursivelyHandleSubfunction().

◆ to_string()

std::string to_string	(	Function &	F,
		const std::vector< bool > &	us )

Definition at line 1225 of file EnzymeLogic.cpp.

Variable Documentation

◆ EnzymeFixupReturn

LLVMValueRef(* EnzymeFixupReturn) (LLVMBuilderRef, LLVMValueRef)	(	LLVMBuilderRef	,
		LLVMValueRef	) = nullptr

Definition at line 125 of file EnzymeLogic.cpp.

Classes

Macros

Functions

Variables

Macro Definition Documentation

◆ addAttribute

◆ getAttribute

◆ removeAttribute

Function Documentation

◆ calculateUnusedStoresInFunction()

◆ calculateUnusedValuesInFunction()

◆ cleanupInversionAllocs()

◆ clearFunctionAttributes()

◆ createInvertedTerminator()

◆ createTerminator() [1/2]

◆ createTerminator() [2/2]

◆ EnzymeAssumeUnknownNoFree()

◆ EnzymeJuliaAddrLoad()

◆ EnzymePrint()

◆ EnzymePrintUnnecessary()

◆ floatMemExpand()

◆ floatMemTruncate()

◆ floatValExpand()

◆ floatValTruncate()

◆ getDefaultFunctionTypeForAugmentation()

◆ getDefaultFunctionTypeForGradient() [1/2]

◆ getDefaultFunctionTypeForGradient() [2/2]

◆ legalCombinedForwardReverse()

◆ looseTypeAnalysis()

◆ nonmarkedglobals_inactiveloads()

◆ restoreCache()

◆ selectByWidth()

◆ shouldAugmentCall()

◆ to_string()

Variable Documentation

◆ EnzymeFixupReturn