Enzyme main
Loading...
Searching...
No Matches
Namespaces | Classes | Typedefs | Enumerations | Functions | Variables
mlir::enzyme Namespace Reference

Namespaces

namespace  batchutils
 
namespace  detail
 
namespace  oputils
 
namespace  transforms
 

Classes

class  ActivityAnalyzer
 Helper class to analyze the differential activity. More...
 
class  ActivityPrinterConfig
 
class  AliasAnalysis
 This analysis implements interprocedural alias analysis. More...
 
class  AliasClassLattice
 
class  AutoDiffCallFwd
 
class  AutoDiffCallRev
 
class  BackwardActivityAnnotationAnalysis
 
class  BackwardMemoryActivity
 
class  BackwardOriginsLattice
 
class  BackwardOriginsMap
 
class  BackwardValueActivity
 
struct  CacheInfo
 Information about a cache, each cache init should have one corresponding push and pop. More...
 
class  DataFlowActivityAnalyzer
 
class  DenseActivityAnnotationAnalysis
 
class  DenseBackwardActivityAnalysis
 
class  DenseBackwardActivityAnnotationAnalysis
 
class  DenseForwardActivityAnalysis
 
struct  ForLikeEnzymeOpsRemover
 
class  ForwardActivityAnnotationAnalysis
 
class  ForwardMemoryActivity
 
class  ForwardOriginsLattice
 
class  ForwardOriginsMap
 
class  ForwardValueActivity
 
struct  IfLikeEnzymeOpsRemover
 
struct  LoopCheckpointing
 
class  MapOfSetsLattice
 
class  MDiffeGradientUtils
 
class  MemoryActivity
 
struct  MemoryActivityState
 This needs to keep track of three things: More...
 
class  MEnzymeLogic
 
class  MFnTypeInfo
 
class  MGradientUtils
 
class  MGradientUtilsReverse
 
class  MTypeAnalysis
 
class  MTypeResults
 
class  OriginalClasses
 Alias classes for freshly created, e.g., allocated values. More...
 
class  PointsToPointerAnalysis
 
class  PointsToSets
 
class  SetLattice
 
class  SparseBackwardActivityAnalysis
 
class  SparseForwardActivityAnalysis
 Sparse activity analysis reasons about activity by traversing forward down the def-use chains starting from active function arguments. More...
 
class  SparseSetLattice
 
class  Utils
 
class  ValueActivity
 

Typedefs

using ValueOriginSet = SetLattice<OriginAttr>
 
using AliasClassSet = SetLattice<DistinctAttr>
 A set of alias class identifiers to be treated as a single union.
 
typedef void buildReturnFunction(OpBuilder &, mlir::Block *)
 

Enumerations

enum class  ActivityKind { Constant , ActiveVal , Unknown }
 From LLVM Enzyme's activity analysis, there are four activity states. More...
 
enum class  LoopCacheType { TENSOR , MEMREF }
 

Functions

LogicalResult reverseToposortCallgraph (CallableOpInterface callee, SymbolTableCollection *symbolTable, SmallVectorImpl< CallableOpInterface > &sorted)
 Starting from callee, compute a reverse (bottom-up) topological sorting of all functions transitively called from callee.
 
void traversePointsToSets (const enzyme::AliasClassSet &start, const enzyme::PointsToSets &pointsToSets, function_ref< void(DistinctAttr)> visit)
 Visit everything transitively pointed-to by any pointer in start.
 
LogicalResult runActivityAnnotationsForFunction (FunctionOpInterface funcOp, DataFlowSolver &solver)
 
void computeSummaries (FunctionOpInterface funcOp, DataFlowSolver &solver, enzyme::PointsToSets &p2sets, enzyme::ForwardOriginsMap &forwardOriginsMap, enzyme::BackwardOriginsMap &backwardOriginsMap, SmallVectorImpl< enzyme::ForwardOriginsLattice > &returnOperandOrigins, SmallVectorImpl< enzyme::AliasClassLattice > &returnAliasClasses)
 
void serializeSummaries (Operation *op, const enzyme::PointsToSets &p2sets, const enzyme::ForwardOriginsMap &forwardOriginsMap, ArrayRef< enzyme::ForwardOriginsLattice > returnOperandOrigins, ArrayRef< enzyme::AliasClassLattice > returnAliasClasses)
 
void removeSummaries (Operation *op)
 
void runActivityAnnotations (FunctionOpInterface callee, ArrayRef< enzyme::Activity > argActivities={}, const ActivityPrinterConfig &config=ActivityPrinterConfig())
 
void runDataFlowActivityAnalysis (FunctionOpInterface callee, ArrayRef< enzyme::Activity > argumentActivity, ArrayRef< enzyme::Activity > returnActivity, bool print=false, bool verbose=false, bool annotate=false)
 
void markReadOnlyLoads (CallableOpInterface funcOp, function_ref< void(Operation *)> annotate)
 
bool sortAttributes (Attribute a, Attribute b)
 Used when serializing to ensure a consistent order.
 
bool sortArraysLexicographic (ArrayAttr a, ArrayAttr b)
 
template<typename OpTy >
void registerAutoDiffUsingCallInterface (MLIRContext &context)
 
template<typename OpTy >
void registerAutoDiffUsingControlFlowInterface (MLIRContext &context)
 
template<typename OpTy >
void registerAutoDiffUsingBranchInterface (MLIRContext &context)
 
template<typename OpTy >
void registerAutoDiffUsingRegionTerminatorInterface (MLIRContext &context)
 
template<typename OpTy >
void registerAutoDiffUsingReturnInterface (MLIRContext &context)
 
template<typename OpTy , int... storedvals>
void registerAutoDiffUsingMemoryIdentityInterface (MLIRContext &context)
 
template<typename OpTy >
void registerAutoDiffUsingAllocationInterface (MLIRContext &context)
 
void registerAffineDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerArithDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerBuiltinDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerLLVMDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerLLVMExtDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerNVVMDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerMemRefDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerComplexDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerSCFDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerCFDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerLinalgDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerMathDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerFuncDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerTensorDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerGPUDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerEnzymeDialectAutoDiffInterface (DialectRegistry &registry)
 
void registerCoreDialectAutodiffInterfaces (DialectRegistry &registry)
 
mlir::TypedAttr getConstantAttr (mlir::Type type, llvm::StringRef value)
 
bool allResultsActive (Operation *op, const MGradientUtils *gutils)
 
void setDerivativeFastMath (Operation *op)
 Give an operation built for a derivative the fast-math flags the LLVM path gives everything it builds (Utils.cpp's getFast()).
 
Value setDerivativeFastMath (Value v)
 
template<typename OpTy >
std::enable_if_t< std::is_base_of< OpState, OpTy >::value, OpTy > setDerivativeFastMath (OpTy op)
 
bool isFunctionReturnLike (Operation *op)
 
bool inlineAutodiffOp (enzyme::AutoDiffOp &op, RewriterBase &rewriter, SymbolTableCollection &symbolTable)
 
bool hoistLoopAllocations (LoopLikeOpInterface loop, uint64_t maxHoistedBytes=0)
 Moves allocation/deallocation pairs that are private to one iteration of loop out of it, so the buffer is allocated once for the whole loop.
 
void minCutCache (Block *forward, Block *reverse, SmallVector< CacheInfo > &caches, PatternRewriter &rewriter, const IRMapping &fwdrevmap, Operation *lastFwd=nullptr)
 
void loopInvariantCache (SmallVectorImpl< CacheInfo > &caches, LoopLikeOpInterface fwdLoop, LoopLikeOpInterface revLoop, const IRMapping &mapping, int64_t threshold=16)
 
static LoopCacheType getCacheType (Operation *op)
 
static bool hasMinCut (Operation *op)
 
static bool hasLICM (Operation *op)
 
static Value ensureIndexType (Value value, OpBuilder &builder)
 
static Value traceToNonBlockArgSource (Value value)
 
static ShapedType refineCacheShapeFromAlloc (ShapedType ST, MultidimensionalAllocInterface allocOp)
 The pushed value may reach its allocation through casts that erase static extents: canonicalization rewrites memref.alloc(c) : memref<?xT> into memref.alloc() : memref<CxT> + memref.cast.
 
static MultidimensionalAllocInterface getMultiDimCacheAllocOp (Value pushedValue, LoopCacheType cacheType, ShapedType shapeTy, Operation *forOp)
 
void localizeGradients (OpBuilder &builder, MGradientUtilsReverse *gutils, Block *fwd)
 
void removalBlockExplore (Block *block, IRMapping &mapping, PatternRewriter &rewriter, llvm::SetVector< Value > &gradients, llvm::MapVector< Value, CacheInfo > &caches)
 
bool blockCmp (mlir::Block *a, mlir::Block *b)
 
bool opCmp (mlir::Operation *a, mlir::Operation *b)
 
bool regionCmp (mlir::Region *a, mlir::Region *b)
 
bool valueCmp (mlir::Value a, mlir::Value b)
 
Type getConcatType (Value val, int64_t width)
 
Value getConcatValue (OpBuilder &builder, Location loc, ArrayRef< Value > argList)
 
Value getExtractValue (OpBuilder &builder, Location loc, Type argTy, Value val, int64_t index)
 
void computeAffineIndices (OpBuilder &builder, Location loc, AffineMap map, ValueRange operands, SmallVectorImpl< Value > &indices)
 

Variables

constexpr llvm::StringLiteral undefinedSetString = "<undefined>"
 
constexpr llvm::StringLiteral unknownSetString = "<unknown>"
 
static constexpr llvm::StringLiteral kPreserveCacheAttrName = "preserve_cache"
 

Typedef Documentation

◆ AliasClassSet

using mlir::enzyme::AliasClassSet = SetLattice<DistinctAttr>

A set of alias class identifiers to be treated as a single union.

May be marked as "unknown", which is a conservative pessimistic state, or as "undefined", which is a "not-yet-analyzed" initial state. Undefined state is different from an empty alias set.

Definition at line 47 of file DataFlowAliasAnalysis.h.

◆ buildReturnFunction

typedef void mlir::enzyme::buildReturnFunction(OpBuilder &, mlir::Block *)

Definition at line 14 of file EnzymeLogic.h.

◆ ValueOriginSet

using mlir::enzyme::ValueOriginSet = SetLattice<OriginAttr>

Definition at line 17 of file ActivityAnnotations.h.

Enumeration Type Documentation

◆ ActivityKind

enum class mlir::enzyme::ActivityKind
strong

From LLVM Enzyme's activity analysis, there are four activity states.

Enumerator
Constant 
ActiveVal 
Unknown 

Definition at line 58 of file DataFlowActivityAnalysis.h.

◆ LoopCacheType

enum class mlir::enzyme::LoopCacheType
strong
Enumerator
TENSOR 
MEMREF 

Definition at line 109 of file RemovalUtils.h.

Function Documentation

◆ allResultsActive()

bool mlir::enzyme::allResultsActive ( Operation * op,
const MGradientUtils * gutils )

Definition at line 89 of file CoreDialectsAutoDiffImplementations.cpp.

References mlir::enzyme::MGradientUtils::isConstantValue().

◆ blockCmp()

bool mlir::enzyme::blockCmp ( mlir::Block * a,
mlir::Block * b )

Referenced by valueCmp().

◆ computeAffineIndices()

void mlir::enzyme::computeAffineIndices ( OpBuilder & builder,
Location loc,
AffineMap map,
ValueRange operands,
SmallVectorImpl< Value > & indices )

Definition at line 171 of file Utils.cpp.

◆ computeSummaries()

void mlir::enzyme::computeSummaries ( FunctionOpInterface funcOp,
DataFlowSolver & solver,
enzyme::PointsToSets & p2sets,
enzyme::ForwardOriginsMap & forwardOriginsMap,
enzyme::BackwardOriginsMap & backwardOriginsMap,
SmallVectorImpl< enzyme::ForwardOriginsLattice > & returnOperandOrigins,
SmallVectorImpl< enzyme::AliasClassLattice > & returnAliasClasses )

Definition at line 1062 of file ActivityAnnotations.cpp.

References isFunctionReturnLike(), mlir::enzyme::MapOfSetsLattice< KeyT, ElementT >::join(), and mlir::enzyme::BackwardOriginsMap::meet().

Referenced by mlir::enzyme::DataFlowActivityAnalyzer::DataFlowActivityAnalyzer(), and runActivityAnnotations().

◆ ensureIndexType()

static Value mlir::enzyme::ensureIndexType ( Value value,
OpBuilder & builder )
static

Definition at line 134 of file RemovalUtils.h.

Referenced by mlir::enzyme::ForLikeEnzymeOpsRemover< FinalClass, OpName >::removeEnzymeOps().

◆ getCacheType()

static LoopCacheType mlir::enzyme::getCacheType ( Operation * op)
static

Definition at line 111 of file RemovalUtils.h.

References MEMREF, and TENSOR.

Referenced by mlir::enzyme::MGradientUtilsReverse::getIndexCacheType(), and mlir::enzyme::ForLikeEnzymeOpsRemover< FinalClass, OpName >::removeEnzymeOps().

◆ getConcatType()

Type mlir::enzyme::getConcatType ( Value val,
int64_t width )

Definition at line 133 of file Utils.cpp.

Referenced by getConcatValue().

◆ getConcatValue()

Value mlir::enzyme::getConcatValue ( OpBuilder & builder,
Location loc,
ArrayRef< Value > argList )

Definition at line 155 of file Utils.cpp.

References getConcatType().

Referenced by mlir::enzyme::oputils::inactiveStoredValueShadow(), and mlir::enzyme::detail::memoryIdentityForwardHandler().

◆ getConstantAttr()

mlir::TypedAttr mlir::enzyme::getConstantAttr ( mlir::Type type,
llvm::StringRef value )

Definition at line 49 of file CoreDialectsAutoDiffImplementations.cpp.

◆ getExtractValue()

Value mlir::enzyme::getExtractValue ( OpBuilder & builder,
Location loc,
Type argTy,
Value val,
int64_t index )

Definition at line 163 of file Utils.cpp.

Referenced by mlir::enzyme::detail::memoryIdentityForwardHandler().

◆ getMultiDimCacheAllocOp()

static MultidimensionalAllocInterface mlir::enzyme::getMultiDimCacheAllocOp ( Value pushedValue,
LoopCacheType cacheType,
ShapedType shapeTy,
Operation * forOp )
static

Definition at line 182 of file RemovalUtils.h.

References MEMREF, and traceToNonBlockArgSource().

Referenced by mlir::enzyme::ForLikeEnzymeOpsRemover< FinalClass, OpName >::removeEnzymeOps().

◆ hasLICM()

static bool mlir::enzyme::hasLICM ( Operation * op)
static

Definition at line 130 of file RemovalUtils.h.

Referenced by mlir::enzyme::ForLikeEnzymeOpsRemover< FinalClass, OpName >::removeEnzymeOps().

◆ hasMinCut()

static bool mlir::enzyme::hasMinCut ( Operation * op)
static

Definition at line 119 of file RemovalUtils.h.

Referenced by mlir::enzyme::ForLikeEnzymeOpsRemover< FinalClass, OpName >::removeEnzymeOps(), and mlir::enzyme::IfLikeEnzymeOpsRemover< FinalClass, OpName, ReverseOpName >::removeEnzymeOps().

◆ hoistLoopAllocations()

bool mlir::enzyme::hoistLoopAllocations ( LoopLikeOpInterface loop,
uint64_t maxHoistedBytes = 0 )

Moves allocation/deallocation pairs that are private to one iteration of loop out of it, so the buffer is allocated once for the whole loop.

maxHoistedBytes bounds the size of a hoisted buffer; 0 means no bound. Returns whether anything moved.

Definition at line 156 of file HoistLoopAllocations.cpp.

◆ inlineAutodiffOp()

bool mlir::enzyme::inlineAutodiffOp ( enzyme::AutoDiffOp & op,
RewriterBase & rewriter,
SymbolTableCollection & symbolTable )

Definition at line 471 of file InlineEnzymeRegions.cpp.

◆ isFunctionReturnLike()

bool mlir::enzyme::isFunctionReturnLike ( Operation * op)
inline

Definition at line 39 of file AutoDiffOpInterface.h.

Referenced by computeSummaries(), createTerminator(), mlir::enzyme::ActivityAnalyzer::isConstantOperation(), isFunctionReturn(), runDataFlowActivityAnalysis(), and mlir::enzyme::DenseBackwardActivityAnalysis::visitOperation().

◆ localizeGradients()

void mlir::enzyme::localizeGradients ( OpBuilder & builder,
MGradientUtilsReverse * gutils,
Block * fwd )

Definition at line 38 of file RemovalUtils.cpp.

References mlir::enzyme::MDiffeGradientUtils::getDifferential(), and mlir::enzyme::MGradientUtils::isConstantValue().

Referenced by mlir::enzyme::LoopCheckpointing< FinalClass, OpName >::primeSegmentGradients(), and mlir::enzyme::LoopCheckpointing< FinalClass, OpName >::primeStepGradients().

◆ loopInvariantCache()

void mlir::enzyme::loopInvariantCache ( SmallVectorImpl< CacheInfo > & caches,
LoopLikeOpInterface fwdLoop,
LoopLikeOpInterface revLoop,
const IRMapping & mapping,
int64_t threshold = 16 )

Definition at line 1393 of file RemovalUtils.cpp.

References loopInvariantCacheImpl().

Referenced by mlir::enzyme::ForLikeEnzymeOpsRemover< FinalClass, OpName >::removeEnzymeOps().

◆ markReadOnlyLoads()

void mlir::enzyme::markReadOnlyLoads ( CallableOpInterface funcOp,
function_ref< void(Operation *)> annotate )

Referenced by mlir::enzyme::MEnzymeLogic::CreateReverseDiff().

◆ minCutCache()

void mlir::enzyme::minCutCache ( Block * forward,
Block * reverse,
SmallVector< CacheInfo > & caches,
PatternRewriter & rewriter,
const IRMapping & fwdrevmap,
Operation * lastFwd = nullptr )

Definition at line 760 of file RemovalUtils.cpp.

References DebugGraphviz(), dump(), mlir::enzyme::CacheInfo::initOp, mlir::enzyme::CacheInfo::popOp, and mlir::enzyme::CacheInfo::pushOp.

Referenced by mlir::enzyme::ForLikeEnzymeOpsRemover< FinalClass, OpName >::removeEnzymeOps(), and mlir::enzyme::IfLikeEnzymeOpsRemover< FinalClass, OpName, ReverseOpName >::removeEnzymeOps().

◆ opCmp()

bool mlir::enzyme::opCmp ( mlir::Operation * a,
mlir::Operation * b )

Referenced by valueCmp().

◆ refineCacheShapeFromAlloc()

static ShapedType mlir::enzyme::refineCacheShapeFromAlloc ( ShapedType ST,
MultidimensionalAllocInterface allocOp )
static

The pushed value may reach its allocation through casts that erase static extents: canonicalization rewrites memref.alloc(c) : memref<?xT> into memref.alloc() : memref<CxT> + memref.cast.

When the allocation itself becomes the multi-dimensional cache storage, its type is the accurate one – sizing the cache from the cast's dynamic type instead leaves dynamic dimensions that appendDynamicDims has no operand for, yielding an invalid cache allocation and row slices whose sizes don't match their operands.

Definition at line 166 of file RemovalUtils.h.

Referenced by mlir::enzyme::ForLikeEnzymeOpsRemover< FinalClass, OpName >::removeEnzymeOps().

◆ regionCmp()

bool mlir::enzyme::regionCmp ( mlir::Region * a,
mlir::Region * b )

◆ registerAffineDialectAutoDiffInterface()

void mlir::enzyme::registerAffineDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 1513 of file AffineAutoDiffOpInterfaceImpl.cpp.

◆ registerArithDialectAutoDiffInterface()

void mlir::enzyme::registerArithDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 239 of file ArithAutoDiffOpInterfaceImpl.cpp.

◆ registerAutoDiffUsingAllocationInterface()

template<typename OpTy >
void mlir::enzyme::registerAutoDiffUsingAllocationInterface ( MLIRContext & context)

Definition at line 305 of file CoreDialectsAutoDiffImplementations.h.

◆ registerAutoDiffUsingBranchInterface()

template<typename OpTy >
void mlir::enzyme::registerAutoDiffUsingBranchInterface ( MLIRContext & context)

Definition at line 276 of file CoreDialectsAutoDiffImplementations.h.

◆ registerAutoDiffUsingCallInterface()

template<typename OpTy >
void mlir::enzyme::registerAutoDiffUsingCallInterface ( MLIRContext & context)

Definition at line 263 of file CoreDialectsAutoDiffImplementations.h.

◆ registerAutoDiffUsingControlFlowInterface()

template<typename OpTy >
void mlir::enzyme::registerAutoDiffUsingControlFlowInterface ( MLIRContext & context)

Definition at line 270 of file CoreDialectsAutoDiffImplementations.h.

◆ registerAutoDiffUsingMemoryIdentityInterface()

template<typename OpTy , int... storedvals>
void mlir::enzyme::registerAutoDiffUsingMemoryIdentityInterface ( MLIRContext & context)

Definition at line 299 of file CoreDialectsAutoDiffImplementations.h.

◆ registerAutoDiffUsingRegionTerminatorInterface()

template<typename OpTy >
void mlir::enzyme::registerAutoDiffUsingRegionTerminatorInterface ( MLIRContext & context)

Definition at line 283 of file CoreDialectsAutoDiffImplementations.h.

Referenced by registerGPUDialectAutoDiffInterface().

◆ registerAutoDiffUsingReturnInterface()

template<typename OpTy >
void mlir::enzyme::registerAutoDiffUsingReturnInterface ( MLIRContext & context)

Definition at line 291 of file CoreDialectsAutoDiffImplementations.h.

◆ registerBuiltinDialectAutoDiffInterface()

void mlir::enzyme::registerBuiltinDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 337 of file BuiltinAutoDiffTypeInterfaceImpl.cpp.

◆ registerCFDialectAutoDiffInterface()

void mlir::enzyme::registerCFDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 36 of file CFAutoDiffOpInterfaceImpl.cpp.

◆ registerComplexDialectAutoDiffInterface()

void mlir::enzyme::registerComplexDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 78 of file ComplexAutoDiffOpInterfaceImpl.cpp.

◆ registerCoreDialectAutodiffInterfaces()

void mlir::enzyme::registerCoreDialectAutodiffInterfaces ( DialectRegistry & registry)

Definition at line 21 of file CoreDialectsAutoDiffRegistration.cpp.

Referenced by main().

◆ registerEnzymeDialectAutoDiffInterface()

void mlir::enzyme::registerEnzymeDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 173 of file EnzymeAutoDiffOpInterfaceImpl.cpp.

◆ registerFuncDialectAutoDiffInterface()

void mlir::enzyme::registerFuncDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 61 of file FuncAutoDiffOpInterfaceImpl.cpp.

◆ registerGPUDialectAutoDiffInterface()

void mlir::enzyme::registerGPUDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 68 of file GPUAutoDiffOpInterfaceImpl.cpp.

References registerAutoDiffUsingRegionTerminatorInterface().

◆ registerLinalgDialectAutoDiffInterface()

void mlir::enzyme::registerLinalgDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 414 of file LinalgAutoDiffOpInterfaceImpl.cpp.

References attachAllInterfaces().

◆ registerLLVMDialectAutoDiffInterface()

void mlir::enzyme::registerLLVMDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 1094 of file LLVMAutoDiffOpInterfaceImpl.cpp.

◆ registerLLVMExtDialectAutoDiffInterface()

void mlir::enzyme::registerLLVMExtDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 31 of file LLVMExtAutoDiffOpInterfaceImpl.cpp.

◆ registerMathDialectAutoDiffInterface()

void mlir::enzyme::registerMathDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 34 of file MathAutoDiffOpInterfaceImpl.cpp.

◆ registerMemRefDialectAutoDiffInterface()

void mlir::enzyme::registerMemRefDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 487 of file MemRefAutoDiffOpInterfaceImpl.cpp.

◆ registerNVVMDialectAutoDiffInterface()

void mlir::enzyme::registerNVVMDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 29 of file NVVMAutoDiffOpInterfaceImpl.cpp.

◆ registerSCFDialectAutoDiffInterface()

void mlir::enzyme::registerSCFDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 1212 of file SCFAutoDiffOpInterfaceImpl.cpp.

◆ registerTensorDialectAutoDiffInterface()

void mlir::enzyme::registerTensorDialectAutoDiffInterface ( DialectRegistry & registry)

Definition at line 252 of file ArithAutoDiffOpInterfaceImpl.cpp.

◆ removalBlockExplore()

void mlir::enzyme::removalBlockExplore ( Block * block,
IRMapping & mapping,
PatternRewriter & rewriter,
llvm::SetVector< Value > & gradients,
llvm::MapVector< Value, CacheInfo > & caches )

Definition at line 88 of file RemovalUtils.cpp.

References getOp, mlir::enzyme::CacheInfo::merge(), mlir::enzyme::CacheInfo::popOp, and mlir::enzyme::CacheInfo::pushedValue().

Referenced by mlir::enzyme::IfLikeEnzymeOpsRemover< FinalClass, OpName, ReverseOpName >::removeEnzymeOps().

◆ removeSummaries()

void mlir::enzyme::removeSummaries ( Operation * op)

Definition at line 1130 of file ActivityAnnotations.cpp.

Referenced by mlir::enzyme::DataFlowActivityAnalyzer::DataFlowActivityAnalyzer().

◆ reverseToposortCallgraph()

LogicalResult mlir::enzyme::reverseToposortCallgraph ( CallableOpInterface callee,
SymbolTableCollection * symbolTable,
SmallVectorImpl< CallableOpInterface > & sorted )

Starting from callee, compute a reverse (bottom-up) topological sorting of all functions transitively called from callee.

Definition at line 1029 of file ActivityAnnotations.cpp.

Referenced by mlir::enzyme::DataFlowActivityAnalyzer::DataFlowActivityAnalyzer(), and runActivityAnnotations().

◆ runActivityAnnotations()

void mlir::enzyme::runActivityAnnotations ( FunctionOpInterface callee,
ArrayRef< enzyme::Activity > argActivities = {},
const ActivityPrinterConfig & config = ActivityPrinterConfig() )

Definition at line 1157 of file ActivityAnnotations.cpp.

References mlir::enzyme::ActivityPrinterConfig::annotate, computeSummaries(), mlir::enzyme::BackwardOriginsMap::getOrigins(), mlir::enzyme::ForwardOriginsMap::getOrigins(), isPossiblyActive(), mlir::enzyme::BackwardOriginsLattice::meet(), reverseToposortCallgraph(), runActivityAnnotationsForFunction(), mlir::enzyme::MapOfSetsLattice< KeyT, ElementT >::serialize(), serializeSummaries(), traversePointsToSets(), and mlir::enzyme::ActivityPrinterConfig::verbose.

◆ runActivityAnnotationsForFunction()

LogicalResult mlir::enzyme::runActivityAnnotationsForFunction ( FunctionOpInterface funcOp,
DataFlowSolver & solver )

Definition at line 1138 of file ActivityAnnotations.cpp.

Referenced by mlir::enzyme::DataFlowActivityAnalyzer::DataFlowActivityAnalyzer(), and runActivityAnnotations().

◆ runDataFlowActivityAnalysis()

void mlir::enzyme::runDataFlowActivityAnalysis ( FunctionOpInterface callee,
ArrayRef< enzyme::Activity > argumentActivity,
ArrayRef< enzyme::Activity > returnActivity,
bool print = false,
bool verbose = false,
bool annotate = false )

Definition at line 922 of file DataFlowActivityAnalysis.cpp.

References mlir::enzyme::ValueActivity::getActiveVal(), isFunctionReturnLike(), and printActivityAnalysisResults().

◆ serializeSummaries()

void mlir::enzyme::serializeSummaries ( Operation * op,
const enzyme::PointsToSets & p2sets,
const enzyme::ForwardOriginsMap & forwardOriginsMap,
ArrayRef< enzyme::ForwardOriginsLattice > returnOperandOrigins,
ArrayRef< enzyme::AliasClassLattice > returnAliasClasses )

Definition at line 1096 of file ActivityAnnotations.cpp.

References mlir::enzyme::MapOfSetsLattice< KeyT, ElementT >::serialize().

Referenced by mlir::enzyme::DataFlowActivityAnalyzer::DataFlowActivityAnalyzer(), and runActivityAnnotations().

◆ setDerivativeFastMath() [1/3]

void mlir::enzyme::setDerivativeFastMath ( Operation * op)

Give an operation built for a derivative the fast-math flags the LLVM path gives everything it builds (Utils.cpp's getFast()).

A gradient is only as fast as the arithmetic it is allowed to reassociate, and one built without the flags its primal was compiled with leaves work on the table: much of what the reverse pass computes is a product with a seed, and only under fast-math does a zero seed fold away the expression it feeds. An op with no fast-math flags to give is left alone.

Definition at line 36 of file CoreDialectsAutoDiffImplementations.cpp.

References EnzymeMLIRFastMath().

Referenced by setDerivativeFastMath(), and setDerivativeFastMath().

◆ setDerivativeFastMath() [2/3]

template<typename OpTy >
std::enable_if_t< std::is_base_of< OpState, OpTy >::value, OpTy > mlir::enzyme::setDerivativeFastMath ( OpTy op)

Definition at line 353 of file CoreDialectsAutoDiffImplementations.h.

References setDerivativeFastMath().

◆ setDerivativeFastMath() [3/3]

Value mlir::enzyme::setDerivativeFastMath ( Value v)
inline

Definition at line 345 of file CoreDialectsAutoDiffImplementations.h.

References setDerivativeFastMath().

◆ sortArraysLexicographic()

bool mlir::enzyme::sortArraysLexicographic ( ArrayAttr a,
ArrayAttr b )

Definition at line 43 of file DataFlowLattice.cpp.

References sortAttributes().

◆ sortAttributes()

bool mlir::enzyme::sortAttributes ( Attribute a,
Attribute b )

Used when serializing to ensure a consistent order.

Definition at line 35 of file DataFlowLattice.cpp.

Referenced by sortArraysLexicographic().

◆ traceToNonBlockArgSource()

static Value mlir::enzyme::traceToNonBlockArgSource ( Value value)
static

Definition at line 144 of file RemovalUtils.h.

Referenced by getMultiDimCacheAllocOp().

◆ traversePointsToSets()

void mlir::enzyme::traversePointsToSets ( const enzyme::AliasClassSet & start,
const enzyme::PointsToSets & pointsToSets,
function_ref< void(DistinctAttr)> visit )

Visit everything transitively pointed-to by any pointer in start.

Definition at line 188 of file ActivityAnnotations.cpp.

References mlir::enzyme::SetLattice< ValueT >::getElements(), mlir::enzyme::PointsToSets::getPointsTo(), mlir::enzyme::SetLattice< ValueT >::isUndefined(), mlir::enzyme::SetLattice< ValueT >::isUnknown(), and mlir::enzyme::SetLattice< ValueT >::join().

Referenced by runActivityAnnotations(), and mlir::enzyme::DenseBackwardActivityAnnotationAnalysis::setToExitState().

◆ valueCmp()

bool mlir::enzyme::valueCmp ( mlir::Value a,
mlir::Value b )

Definition at line 104 of file Utils.cpp.

References blockCmp(), and opCmp().

Variable Documentation

◆ kPreserveCacheAttrName

llvm::StringLiteral mlir::enzyme::kPreserveCacheAttrName = "preserve_cache"
staticconstexpr

Definition at line 30 of file RemovalUtils.h.

Referenced by mlir::enzyme::IfLikeEnzymeOpsRemover< FinalClass, OpName, ReverseOpName >::removeEnzymeOps().

◆ undefinedSetString

llvm::StringLiteral mlir::enzyme::undefinedSetString = "<undefined>"
constexpr

Definition at line 38 of file DataFlowLattice.h.

Referenced by deserializeAliasSummary(), deserializePointsTo(), deserializePointsTo(), mlir::enzyme::PointsToSets::print(), and mlir::enzyme::SetLattice< ValueT >::print().

◆ unknownSetString

llvm::StringLiteral mlir::enzyme::unknownSetString = "<unknown>"
constexpr

Definition at line 39 of file DataFlowLattice.h.

Referenced by deserializeAliasSummary(), deserializePointsTo(), deserializePointsTo(), mlir::enzyme::PointsToSets::print(), and mlir::enzyme::SetLattice< ValueT >::print().

Generated on Tue Oct 13 2026 01:33:29 for Enzyme by doxygen 1.12.0