CGOpenMPRuntimeNVPTX.h source code [clang_source_code/lib/CodeGen/CGOpenMPRuntimeNVPTX.h]

1	//===----- CGOpenMPRuntimeNVPTX.h - Interface to OpenMP NVPTX Runtimes ----===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This provides a class for OpenMP runtime code generation specialized to NVPTX
10	// targets.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMENVPTX_H
15	#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMENVPTX_H
16
17	#include "CGOpenMPRuntime.h"
18	#include "CodeGenFunction.h"
19	#include "clang/AST/StmtOpenMP.h"
20
21	namespace clang {
22	namespace CodeGen {
23
24	class CGOpenMPRuntimeNVPTX : public CGOpenMPRuntime {
25	public:
26	/// Defines the execution mode.
27	enum ExecutionMode {
28	/// SPMD execution mode (all threads are worker threads).
29	EM_SPMD,
30	/// Non-SPMD execution mode (1 master thread, others are workers).
31	EM_NonSPMD,
32	/// Unknown execution mode (orphaned directive).
33	EM_Unknown,
34	};
35	private:
36	/// Parallel outlined function work for workers to execute.
37	llvm::SmallVector<llvm::Function *, 16> Work;
38
39	struct EntryFunctionState {
40	llvm::BasicBlock *ExitBB = nullptr;
41	};
42
43	class WorkerFunctionState {
44	public:
45	llvm::Function *WorkerFn;
46	const CGFunctionInfo &CGFI;
47	SourceLocation Loc;
48
49	WorkerFunctionState(CodeGenModule &CGM, SourceLocation Loc);
50
51	private:
52	void createWorkerFunction(CodeGenModule &CGM);
53	};
54
55	ExecutionMode getExecutionMode() const;
56
57	bool requiresFullRuntime() const { return RequiresFullRuntime; }
58
59	/// Get barrier to synchronize all threads in a block.
60	void syncCTAThreads(CodeGenFunction &CGF);
61
62	/// Emit the worker function for the current target region.
63	void emitWorkerFunction(WorkerFunctionState &WST);
64
65	/// Helper for worker function. Emit body of worker loop.
66	void emitWorkerLoop(CodeGenFunction &CGF, WorkerFunctionState &WST);
67
68	/// Helper for non-SPMD target entry function. Guide the master and
69	/// worker threads to their respective locations.
70	void emitNonSPMDEntryHeader(CodeGenFunction &CGF, EntryFunctionState &EST,
71	WorkerFunctionState &WST);
72
73	/// Signal termination of OMP execution for non-SPMD target entry
74	/// function.
75	void emitNonSPMDEntryFooter(CodeGenFunction &CGF, EntryFunctionState &EST);
76
77	/// Helper for generic variables globalization prolog.
78	void emitGenericVarsProlog(CodeGenFunction &CGF, SourceLocation Loc,
79	bool WithSPMDCheck = false);
80
81	/// Helper for generic variables globalization epilog.
82	void emitGenericVarsEpilog(CodeGenFunction &CGF, bool WithSPMDCheck = false);
83
84	/// Helper for SPMD mode target directive's entry function.
85	void emitSPMDEntryHeader(CodeGenFunction &CGF, EntryFunctionState &EST,
86	const OMPExecutableDirective &D);
87
88	/// Signal termination of SPMD mode execution.
89	void emitSPMDEntryFooter(CodeGenFunction &CGF, EntryFunctionState &EST);
90
91	//
92	// Base class overrides.
93	//
94
95	/// Creates offloading entry for the provided entry ID \a ID,
96	/// address \a Addr, size \a Size, and flags \a Flags.
97	void createOffloadEntry(llvm::Constant ID, llvm::Constant Addr,
98	uint64_t Size, int32_t Flags,
99	llvm::GlobalValue::LinkageTypes Linkage) override;
100
101	/// Emit outlined function specialized for the Fork-Join
102	/// programming model for applicable target directives on the NVPTX device.
103	/// \param D Directive to emit.
104	/// \param ParentName Name of the function that encloses the target region.
105	/// \param OutlinedFn Outlined function value to be defined by this call.
106	/// \param OutlinedFnID Outlined function ID value to be defined by this call.
107	/// \param IsOffloadEntry True if the outlined function is an offload entry.
108	/// An outlined function may not be an entry if, e.g. the if clause always
109	/// evaluates to false.
110	void emitNonSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,
111	llvm::Function *&OutlinedFn,
112	llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,
113	const RegionCodeGenTy &CodeGen);
114
115	/// Emit outlined function specialized for the Single Program
116	/// Multiple Data programming model for applicable target directives on the
117	/// NVPTX device.
118	/// \param D Directive to emit.
119	/// \param ParentName Name of the function that encloses the target region.
120	/// \param OutlinedFn Outlined function value to be defined by this call.
121	/// \param OutlinedFnID Outlined function ID value to be defined by this call.
122	/// \param IsOffloadEntry True if the outlined function is an offload entry.
123	/// \param CodeGen Object containing the target statements.
124	/// An outlined function may not be an entry if, e.g. the if clause always
125	/// evaluates to false.
126	void emitSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,
127	llvm::Function *&OutlinedFn,
128	llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,
129	const RegionCodeGenTy &CodeGen);
130
131	/// Emit outlined function for 'target' directive on the NVPTX
132	/// device.
133	/// \param D Directive to emit.
134	/// \param ParentName Name of the function that encloses the target region.
135	/// \param OutlinedFn Outlined function value to be defined by this call.
136	/// \param OutlinedFnID Outlined function ID value to be defined by this call.
137	/// \param IsOffloadEntry True if the outlined function is an offload entry.
138	/// An outlined function may not be an entry if, e.g. the if clause always
139	/// evaluates to false.
140	void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
141	StringRef ParentName,
142	llvm::Function *&OutlinedFn,
143	llvm::Constant *&OutlinedFnID,
144	bool IsOffloadEntry,
145	const RegionCodeGenTy &CodeGen) override;
146
147	/// Emits code for parallel or serial call of the \a OutlinedFn with
148	/// variables captured in a record which address is stored in \a
149	/// CapturedStruct.
150	/// This call is for the Non-SPMD Execution Mode.
151	/// \param OutlinedFn Outlined function to be run in parallel threads. Type of
152	/// this function is void()(kmp_int32 , kmp_int32, struct context_vars*).
153	/// \param CapturedVars A pointer to the record with the references to
154	/// variables used in \a OutlinedFn function.
155	/// \param IfCond Condition in the associated 'if' clause, if it was
156	/// specified, nullptr otherwise.
157	void emitNonSPMDParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
158	llvm::Value *OutlinedFn,
159	ArrayRef<llvm::Value *> CapturedVars,
160	const Expr *IfCond);
161
162	/// Emits code for parallel or serial call of the \a OutlinedFn with
163	/// variables captured in a record which address is stored in \a
164	/// CapturedStruct.
165	/// This call is for a parallel directive within an SPMD target directive.
166	/// \param OutlinedFn Outlined function to be run in parallel threads. Type of
167	/// this function is void()(kmp_int32 , kmp_int32, struct context_vars*).
168	/// \param CapturedVars A pointer to the record with the references to
169	/// variables used in \a OutlinedFn function.
170	/// \param IfCond Condition in the associated 'if' clause, if it was
171	/// specified, nullptr otherwise.
172	///
173	void emitSPMDParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
174	llvm::Function *OutlinedFn,
175	ArrayRef<llvm::Value *> CapturedVars,
176	const Expr *IfCond);
177
178	protected:
179	/// Get the function name of an outlined region.
180	// The name can be customized depending on the target.
181	//
182	StringRef getOutlinedHelperName() const override {
183	return "__omp_outlined__";
184	}
185
186	/// Check if the default location must be constant.
187	/// Constant for NVPTX for better optimization.
188	bool isDefaultLocationConstant() const override { return true; }
189
190	/// Returns additional flags that can be stored in reserved_2 field of the
191	/// default location.
192	/// For NVPTX target contains data about SPMD/Non-SPMD execution mode +
193	/// Full/Lightweight runtime mode. Used for better optimization.
194	unsigned getDefaultLocationReserved2Flags() const override;
195
196	public:
197	explicit CGOpenMPRuntimeNVPTX(CodeGenModule &CGM);
198	void clear() override;
199
200	/// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
201	/// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
202	virtual void emitProcBindClause(CodeGenFunction &CGF,
203	OpenMPProcBindClauseKind ProcBind,
204	SourceLocation Loc) override;
205
206	/// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
207	/// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
208	/// clause.
209	/// \param NumThreads An integer value of threads.
210	virtual void emitNumThreadsClause(CodeGenFunction &CGF,
211	llvm::Value *NumThreads,
212	SourceLocation Loc) override;
213
214	/// This function ought to emit, in the general case, a call to
215	// the openmp runtime kmpc_push_num_teams. In NVPTX backend it is not needed
216	// as these numbers are obtained through the PTX grid and block configuration.
217	/// \param NumTeams An integer expression of teams.
218	/// \param ThreadLimit An integer expression of threads.
219	void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
220	const Expr *ThreadLimit, SourceLocation Loc) override;
221
222	/// Emits inlined function for the specified OpenMP parallel
223	// directive.
224	/// \a D. This outlined function has type void()(kmp_int32 ThreadID,
225	/// kmp_int32 BoundID, struct context_vars*).
226	/// \param D OpenMP directive.
227	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
228	/// \param InnermostKind Kind of innermost directive (for simple directives it
229	/// is a directive itself, for combined - its innermost directive).
230	/// \param CodeGen Code generation sequence for the \a D directive.
231	llvm::Function *
232	emitParallelOutlinedFunction(const OMPExecutableDirective &D,
233	const VarDecl *ThreadIDVar,
234	OpenMPDirectiveKind InnermostKind,
235	const RegionCodeGenTy &CodeGen) override;
236
237	/// Emits inlined function for the specified OpenMP teams
238	// directive.
239	/// \a D. This outlined function has type void()(kmp_int32 ThreadID,
240	/// kmp_int32 BoundID, struct context_vars*).
241	/// \param D OpenMP directive.
242	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
243	/// \param InnermostKind Kind of innermost directive (for simple directives it
244	/// is a directive itself, for combined - its innermost directive).
245	/// \param CodeGen Code generation sequence for the \a D directive.
246	llvm::Function *
247	emitTeamsOutlinedFunction(const OMPExecutableDirective &D,
248	const VarDecl *ThreadIDVar,
249	OpenMPDirectiveKind InnermostKind,
250	const RegionCodeGenTy &CodeGen) override;
251
252	/// Emits code for teams call of the \a OutlinedFn with
253	/// variables captured in a record which address is stored in \a
254	/// CapturedStruct.
255	/// \param OutlinedFn Outlined function to be run by team masters. Type of
256	/// this function is void()(kmp_int32 , kmp_int32, struct context_vars*).
257	/// \param CapturedVars A pointer to the record with the references to
258	/// variables used in \a OutlinedFn function.
259	///
260	void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
261	SourceLocation Loc, llvm::Function *OutlinedFn,
262	ArrayRef<llvm::Value *> CapturedVars) override;
263
264	/// Emits code for parallel or serial call of the \a OutlinedFn with
265	/// variables captured in a record which address is stored in \a
266	/// CapturedStruct.
267	/// \param OutlinedFn Outlined function to be run in parallel threads. Type of
268	/// this function is void()(kmp_int32 , kmp_int32, struct context_vars*).
269	/// \param CapturedVars A pointer to the record with the references to
270	/// variables used in \a OutlinedFn function.
271	/// \param IfCond Condition in the associated 'if' clause, if it was
272	/// specified, nullptr otherwise.
273	void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
274	llvm::Function *OutlinedFn,
275	ArrayRef<llvm::Value *> CapturedVars,
276	const Expr *IfCond) override;
277
278	/// Emit an implicit/explicit barrier for OpenMP threads.
279	/// \param Kind Directive for which this implicit barrier call must be
280	/// generated. Must be OMPD_barrier for explicit barrier generation.
281	/// \param EmitChecks true if need to emit checks for cancellation barriers.
282	/// \param ForceSimpleCall true simple barrier call must be emitted, false if
283	/// runtime class decides which one to emit (simple or with cancellation
284	/// checks).
285	///
286	void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
287	OpenMPDirectiveKind Kind, bool EmitChecks = true,
288	bool ForceSimpleCall = false) override;
289
290	/// Emits a critical region.
291	/// \param CriticalName Name of the critical region.
292	/// \param CriticalOpGen Generator for the statement associated with the given
293	/// critical region.
294	/// \param Hint Value of the 'hint' clause (optional).
295	void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
296	const RegionCodeGenTy &CriticalOpGen,
297	SourceLocation Loc,
298	const Expr *Hint = nullptr) override;
299
300	/// Emit a code for reduction clause.
301	///
302	/// \param Privates List of private copies for original reduction arguments.
303	/// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
304	/// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
305	/// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
306	/// or 'operator binop(LHS, RHS)'.
307	/// \param Options List of options for reduction codegen:
308	/// WithNowait true if parent directive has also nowait clause, false
309	/// otherwise.
310	/// SimpleReduction Emit reduction operation only. Used for omp simd
311	/// directive on the host.
312	/// ReductionKind The kind of reduction to perform.
313	virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
314	ArrayRef<const Expr *> Privates,
315	ArrayRef<const Expr *> LHSExprs,
316	ArrayRef<const Expr *> RHSExprs,
317	ArrayRef<const Expr *> ReductionOps,
318	ReductionOptionsTy Options) override;
319
320	/// Returns specified OpenMP runtime function for the current OpenMP
321	/// implementation. Specialized for the NVPTX device.
322	/// \param Function OpenMP runtime function.
323	/// \return Specified function.
324	llvm::FunctionCallee createNVPTXRuntimeFunction(unsigned Function);
325
326	/// Translates the native parameter of outlined function if this is required
327	/// for target.
328	/// \param FD Field decl from captured record for the parameter.
329	/// \param NativeParam Parameter itself.
330	const VarDecl translateParameter(const FieldDecl FD,
331	const VarDecl *NativeParam) const override;
332
333	/// Gets the address of the native argument basing on the address of the
334	/// target-specific parameter.
335	/// \param NativeParam Parameter itself.
336	/// \param TargetParam Corresponding target-specific parameter.
337	Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
338	const VarDecl *TargetParam) const override;
339
340	/// Emits call of the outlined function with the provided arguments,
341	/// translating these arguments to correct target-specific arguments.
342	void emitOutlinedFunctionCall(
343	CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
344	ArrayRef<llvm::Value *> Args = llvm::None) const override;
345
346	/// Emits OpenMP-specific function prolog.
347	/// Required for device constructs.
348	void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) override;
349
350	/// Gets the OpenMP-specific address of the local variable.
351	Address getAddressOfLocalVariable(CodeGenFunction &CGF,
352	const VarDecl *VD) override;
353
354	/// Target codegen is specialized based on two data-sharing modes: CUDA, in
355	/// which the local variables are actually global threadlocal, and Generic, in
356	/// which the local variables are placed in global memory if they may escape
357	/// their declaration context.
358	enum DataSharingMode {
359	/// CUDA data sharing mode.
360	CUDA,
361	/// Generic data-sharing mode.
362	Generic,
363	};
364
365	/// Cleans up references to the objects in finished function.
366	///
367	void functionFinished(CodeGenFunction &CGF) override;
368
369	/// Choose a default value for the dist_schedule clause.
370	void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
371	const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
372	llvm::Value *&Chunk) const override;
373
374	/// Choose a default value for the schedule clause.
375	void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
376	const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
377	const Expr *&ChunkExpr) const override;
378
379	/// Adjust some parameters for the target-based directives, like addresses of
380	/// the variables captured by reference in lambdas.
381	void adjustTargetSpecificDataForLambdas(
382	CodeGenFunction &CGF, const OMPExecutableDirective &D) const override;
383
384	/// Perform check on requires decl to ensure that target architecture
385	/// supports unified addressing
386	void checkArchForUnifiedAddressing(const OMPRequiresDecl *D) const override;
387
388	/// Returns default address space for the constant firstprivates, __constant__
389	/// address space by default.
390	unsigned getDefaultFirstprivateAddressSpace() const override;
391
392	/// Checks if the variable has associated OMPAllocateDeclAttr attribute with
393	/// the predefined allocator and translates it into the corresponding address
394	/// space.
395	bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS) override;
396
397	private:
398	/// Track the execution mode when codegening directives within a target
399	/// region. The appropriate mode (SPMD/NON-SPMD) is set on entry to the
400	/// target region and used by containing directives such as 'parallel'
401	/// to emit optimized code.
402	ExecutionMode CurrentExecutionMode = EM_Unknown;
403
404	/// Check if the full runtime is required (default - yes).
405	bool RequiresFullRuntime = true;
406
407	/// true if we're emitting the code for the target region and next parallel
408	/// region is L0 for sure.
409	bool IsInTargetMasterThreadRegion = false;
410	/// true if currently emitting code for target/teams/distribute region, false
411	/// - otherwise.
412	bool IsInTTDRegion = false;
413	/// true if we're definitely in the parallel region.
414	bool IsInParallelRegion = false;
415
416	/// Map between an outlined function and its wrapper.
417	llvm::DenseMap<llvm::Function , llvm::Function > WrapperFunctionsMap;
418
419	/// Emit function which wraps the outline parallel region
420	/// and controls the parameters which are passed to this function.
421	/// The wrapper ensures that the outlined function is called
422	/// with the correct arguments when data is shared.
423	llvm::Function *createParallelDataSharingWrapper(
424	llvm::Function *OutlinedParallelFn, const OMPExecutableDirective &D);
425
426	/// The data for the single globalized variable.
427	struct MappedVarData {
428	/// Corresponding field in the global record.
429	const FieldDecl *FD = nullptr;
430	/// Corresponding address.
431	Address PrivateAddr = Address::invalid();
432	/// true, if only one element is required (for latprivates in SPMD mode),
433	/// false, if need to create based on the warp-size.
434	bool IsOnePerTeam = false;
435	MappedVarData() = delete;
436	MappedVarData(const FieldDecl *FD, bool IsOnePerTeam = false)
437	: FD(FD), IsOnePerTeam(IsOnePerTeam) {}
438	};
439	/// The map of local variables to their addresses in the global memory.
440	using DeclToAddrMapTy = llvm::MapVector<const Decl *, MappedVarData>;
441	/// Set of the parameters passed by value escaping OpenMP context.
442	using EscapedParamsTy = llvm::SmallPtrSet<const Decl *, 4>;
443	struct FunctionData {
444	DeclToAddrMapTy LocalVarData;
445	llvm::Optional<DeclToAddrMapTy> SecondaryLocalVarData = llvm::None;
446	EscapedParamsTy EscapedParameters;
447	llvm::SmallVector<const ValueDecl*, 4> EscapedVariableLengthDecls;
448	llvm::SmallVector<llvm::Value *, 4> EscapedVariableLengthDeclsAddrs;
449	const RecordDecl *GlobalRecord = nullptr;
450	llvm::Optional<const RecordDecl *> SecondaryGlobalRecord = llvm::None;
451	llvm::Value *GlobalRecordAddr = nullptr;
452	llvm::Value *IsInSPMDModeFlag = nullptr;
453	std::unique_ptr<CodeGenFunction::OMPMapVars> MappedParams;
454	};
455	/// Maps the function to the list of the globalized variables with their
456	/// addresses.
457	llvm::SmallDenseMap<llvm::Function *, FunctionData> FunctionGlobalizedDecls;
458	/// List of records for the globalized variables in target/teams/distribute
459	/// contexts. Inner records are going to be joined into the single record,
460	/// while those resulting records are going to be joined into the single
461	/// union. This resulting union (one per CU) is the entry point for the static
462	/// memory management runtime functions.
463	struct GlobalPtrSizeRecsTy {
464	llvm::GlobalVariable *UseSharedMemory = nullptr;
465	llvm::GlobalVariable *RecSize = nullptr;
466	llvm::GlobalVariable *Buffer = nullptr;
467	SourceLocation Loc;
468	llvm::SmallVector<const RecordDecl *, 2> Records;
469	unsigned RegionCounter = 0;
470	};
471	llvm::SmallVector<GlobalPtrSizeRecsTy, 8> GlobalizedRecords;
472	llvm::GlobalVariable *KernelTeamsReductionPtr = nullptr;
473	/// List of the records with the list of fields for the reductions across the
474	/// teams. Used to build the intermediate buffer for the fast teams
475	/// reductions.
476	/// All the records are gathered into a union `union.type` is created.
477	llvm::SmallVector<const RecordDecl *, 4> TeamsReductions;
478	/// Shared pointer for the global memory in the global memory buffer used for
479	/// the given kernel.
480	llvm::GlobalVariable *KernelStaticGlobalized = nullptr;
481	/// Pair of the Non-SPMD team and all reductions variables in this team
482	/// region.
483	std::pair<const Decl , llvm::SmallVector<const ValueDecl , 4>>
484	TeamAndReductions;
485	};
486
487	} // CodeGen namespace.
488	} // clang namespace.
489
490	#endif // LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMENVPTX_H
491