Clang Project

clang_source_code/lib/CodeGen/CGOpenMPRuntime.cpp
1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCXXABI.h"
14#include "CGCleanup.h"
15#include "CGOpenMPRuntime.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "clang/CodeGen/ConstantInitBuilder.h"
19#include "clang/AST/Decl.h"
20#include "clang/AST/StmtOpenMP.h"
21#include "clang/Basic/BitmaskEnum.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/Bitcode/BitcodeReader.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/GlobalValue.h"
26#include "llvm/IR/Value.h"
27#include "llvm/Support/Format.h"
28#include "llvm/Support/raw_ostream.h"
29#include <cassert>
30
31using namespace clang;
32using namespace CodeGen;
33
34namespace {
35/// Base class for handling code generation inside OpenMP regions.
36class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
37public:
38  /// Kinds of OpenMP regions used in codegen.
39  enum CGOpenMPRegionKind {
40    /// Region with outlined function for standalone 'parallel'
41    /// directive.
42    ParallelOutlinedRegion,
43    /// Region with outlined function for standalone 'task' directive.
44    TaskOutlinedRegion,
45    /// Region for constructs that do not require function outlining,
46    /// like 'for', 'sections', 'atomic' etc. directives.
47    InlinedRegion,
48    /// Region with outlined function for standalone 'target' directive.
49    TargetRegion,
50  };
51
52  CGOpenMPRegionInfo(const CapturedStmt &CS,
53                     const CGOpenMPRegionKind RegionKind,
54                     const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
55                     bool HasCancel)
56      : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
57        CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
58
59  CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
60                     const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
61                     bool HasCancel)
62      : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
63        Kind(Kind), HasCancel(HasCancel) {}
64
65  /// Get a variable or parameter for storing global thread id
66  /// inside OpenMP construct.
67  virtual const VarDecl *getThreadIDVariable() const = 0;
68
69  /// Emit the captured statement body.
70  void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
71
72  /// Get an LValue for the current ThreadID variable.
73  /// \return LValue for thread id variable. This LValue always has type int32*.
74  virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
75
76  virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
77
78  CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
79
80  OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
81
82  bool hasCancel() const { return HasCancel; }
83
84  static bool classof(const CGCapturedStmtInfo *Info) {
85    return Info->getKind() == CR_OpenMP;
86  }
87
88  ~CGOpenMPRegionInfo() override = default;
89
90protected:
91  CGOpenMPRegionKind RegionKind;
92  RegionCodeGenTy CodeGen;
93  OpenMPDirectiveKind Kind;
94  bool HasCancel;
95};
96
97/// API for captured statement code generation in OpenMP constructs.
98class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
99public:
100  CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
101                             const RegionCodeGenTy &CodeGen,
102                             OpenMPDirectiveKind Kind, bool HasCancel,
103                             StringRef HelperName)
104      : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
105                           HasCancel),
106        ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
107     (0) . __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 107, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
108  }
109
110  /// Get a variable or parameter for storing global thread id
111  /// inside OpenMP construct.
112  const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
113
114  /// Get the name of the capture helper.
115  StringRef getHelperName() const override { return HelperName; }
116
117  static bool classof(const CGCapturedStmtInfo *Info) {
118    return CGOpenMPRegionInfo::classof(Info) &&
119           cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
120               ParallelOutlinedRegion;
121  }
122
123private:
124  /// A variable or parameter storing global thread id for OpenMP
125  /// constructs.
126  const VarDecl *ThreadIDVar;
127  StringRef HelperName;
128};
129
130/// API for captured statement code generation in OpenMP constructs.
131class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
132public:
133  class UntiedTaskActionTy final : public PrePostActionTy {
134    bool Untied;
135    const VarDecl *PartIDVar;
136    const RegionCodeGenTy UntiedCodeGen;
137    llvm::SwitchInst *UntiedSwitch = nullptr;
138
139  public:
140    UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
141                       const RegionCodeGenTy &UntiedCodeGen)
142        : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
143    void Enter(CodeGenFunction &CGF) override {
144      if (Untied) {
145        // Emit task switching point.
146        LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
147            CGF.GetAddrOfLocalVar(PartIDVar),
148            PartIDVar->getType()->castAs<PointerType>());
149        llvm::Value *Res =
150            CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
151        llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
152        UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
153        CGF.EmitBlock(DoneBB);
154        CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
155        CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
156        UntiedSwitch->addCase(CGF.Builder.getInt32(0),
157                              CGF.Builder.GetInsertBlock());
158        emitUntiedSwitch(CGF);
159      }
160    }
161    void emitUntiedSwitch(CodeGenFunction &CGF) const {
162      if (Untied) {
163        LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
164            CGF.GetAddrOfLocalVar(PartIDVar),
165            PartIDVar->getType()->castAs<PointerType>());
166        CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
167                              PartIdLVal);
168        UntiedCodeGen(CGF);
169        CodeGenFunction::JumpDest CurPoint =
170            CGF.getJumpDestInCurrentScope(".untied.next.");
171        CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
172        CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
173        UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
174                              CGF.Builder.GetInsertBlock());
175        CGF.EmitBranchThroughCleanup(CurPoint);
176        CGF.EmitBlock(CurPoint.getBlock());
177      }
178    }
179    unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
180  };
181  CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
182                                 const VarDecl *ThreadIDVar,
183                                 const RegionCodeGenTy &CodeGen,
184                                 OpenMPDirectiveKind Kind, bool HasCancel,
185                                 const UntiedTaskActionTy &Action)
186      : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
187        ThreadIDVar(ThreadIDVar), Action(Action) {
188     (0) . __assert_fail ("ThreadIDVar != nullptr && \"No ThreadID in OpenMP region.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 188, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
189  }
190
191  /// Get a variable or parameter for storing global thread id
192  /// inside OpenMP construct.
193  const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
194
195  /// Get an LValue for the current ThreadID variable.
196  LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
197
198  /// Get the name of the capture helper.
199  StringRef getHelperName() const override { return ".omp_outlined."; }
200
201  void emitUntiedSwitch(CodeGenFunction &CGF) override {
202    Action.emitUntiedSwitch(CGF);
203  }
204
205  static bool classof(const CGCapturedStmtInfo *Info) {
206    return CGOpenMPRegionInfo::classof(Info) &&
207           cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
208               TaskOutlinedRegion;
209  }
210
211private:
212  /// A variable or parameter storing global thread id for OpenMP
213  /// constructs.
214  const VarDecl *ThreadIDVar;
215  /// Action for emitting code for untied tasks.
216  const UntiedTaskActionTy &Action;
217};
218
219/// API for inlined captured statement code generation in OpenMP
220/// constructs.
221class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
222public:
223  CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
224                            const RegionCodeGenTy &CodeGen,
225                            OpenMPDirectiveKind Kind, bool HasCancel)
226      : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
227        OldCSI(OldCSI),
228        OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
229
230  // Retrieve the value of the context parameter.
231  llvm::Value *getContextValue() const override {
232    if (OuterRegionInfo)
233      return OuterRegionInfo->getContextValue();
234    llvm_unreachable("No context value for inlined OpenMP region");
235  }
236
237  void setContextValue(llvm::Value *V) override {
238    if (OuterRegionInfo) {
239      OuterRegionInfo->setContextValue(V);
240      return;
241    }
242    llvm_unreachable("No context value for inlined OpenMP region");
243  }
244
245  /// Lookup the captured field decl for a variable.
246  const FieldDecl *lookup(const VarDecl *VD) const override {
247    if (OuterRegionInfo)
248      return OuterRegionInfo->lookup(VD);
249    // If there is no outer outlined region,no need to lookup in a list of
250    // captured variables, we can use the original one.
251    return nullptr;
252  }
253
254  FieldDecl *getThisFieldDecl() const override {
255    if (OuterRegionInfo)
256      return OuterRegionInfo->getThisFieldDecl();
257    return nullptr;
258  }
259
260  /// Get a variable or parameter for storing global thread id
261  /// inside OpenMP construct.
262  const VarDecl *getThreadIDVariable() const override {
263    if (OuterRegionInfo)
264      return OuterRegionInfo->getThreadIDVariable();
265    return nullptr;
266  }
267
268  /// Get an LValue for the current ThreadID variable.
269  LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
270    if (OuterRegionInfo)
271      return OuterRegionInfo->getThreadIDVariableLValue(CGF);
272    llvm_unreachable("No LValue for inlined OpenMP construct");
273  }
274
275  /// Get the name of the capture helper.
276  StringRef getHelperName() const override {
277    if (auto *OuterRegionInfo = getOldCSI())
278      return OuterRegionInfo->getHelperName();
279    llvm_unreachable("No helper name for inlined OpenMP construct");
280  }
281
282  void emitUntiedSwitch(CodeGenFunction &CGF) override {
283    if (OuterRegionInfo)
284      OuterRegionInfo->emitUntiedSwitch(CGF);
285  }
286
287  CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
288
289  static bool classof(const CGCapturedStmtInfo *Info) {
290    return CGOpenMPRegionInfo::classof(Info) &&
291           cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
292  }
293
294  ~CGOpenMPInlinedRegionInfo() override = default;
295
296private:
297  /// CodeGen info about outer OpenMP region.
298  CodeGenFunction::CGCapturedStmtInfo *OldCSI;
299  CGOpenMPRegionInfo *OuterRegionInfo;
300};
301
302/// API for captured statement code generation in OpenMP target
303/// constructs. For this captures, implicit parameters are used instead of the
304/// captured fields. The name of the target region has to be unique in a given
305/// application so it is provided by the client, because only the client has
306/// the information to generate that.
307class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
308public:
309  CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
310                           const RegionCodeGenTy &CodeGen, StringRef HelperName)
311      : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
312                           /*HasCancel=*/false),
313        HelperName(HelperName) {}
314
315  /// This is unused for target regions because each starts executing
316  /// with a single thread.
317  const VarDecl *getThreadIDVariable() const override { return nullptr; }
318
319  /// Get the name of the capture helper.
320  StringRef getHelperName() const override { return HelperName; }
321
322  static bool classof(const CGCapturedStmtInfo *Info) {
323    return CGOpenMPRegionInfo::classof(Info) &&
324           cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
325  }
326
327private:
328  StringRef HelperName;
329};
330
331static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
332  llvm_unreachable("No codegen for expressions");
333}
334/// API for generation of expressions captured in a innermost OpenMP
335/// region.
336class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
337public:
338  CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
339      : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
340                                  OMPD_unknown,
341                                  /*HasCancel=*/false),
342        PrivScope(CGF) {
343    // Make sure the globals captured in the provided statement are local by
344    // using the privatization logic. We assume the same variable is not
345    // captured more than once.
346    for (const auto &C : CS.captures()) {
347      if (!C.capturesVariable() && !C.capturesVariableByCopy())
348        continue;
349
350      const VarDecl *VD = C.getCapturedVar();
351      if (VD->isLocalVarDeclOrParm())
352        continue;
353
354      DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
355                      /*RefersToEnclosingVariableOrCapture=*/false,
356                      VD->getType().getNonReferenceType(), VK_LValue,
357                      C.getLocation());
358      PrivScope.addPrivate(
359          VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(); });
360    }
361    (void)PrivScope.Privatize();
362  }
363
364  /// Lookup the captured field decl for a variable.
365  const FieldDecl *lookup(const VarDecl *VD) const override {
366    if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
367      return FD;
368    return nullptr;
369  }
370
371  /// Emit the captured statement body.
372  void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
373    llvm_unreachable("No body for expressions");
374  }
375
376  /// Get a variable or parameter for storing global thread id
377  /// inside OpenMP construct.
378  const VarDecl *getThreadIDVariable() const override {
379    llvm_unreachable("No thread id for expressions");
380  }
381
382  /// Get the name of the capture helper.
383  StringRef getHelperName() const override {
384    llvm_unreachable("No helper name for expressions");
385  }
386
387  static bool classof(const CGCapturedStmtInfo *Info) { return false; }
388
389private:
390  /// Private scope to capture global variables.
391  CodeGenFunction::OMPPrivateScope PrivScope;
392};
393
394/// RAII for emitting code of OpenMP constructs.
395class InlinedOpenMPRegionRAII {
396  CodeGenFunction &CGF;
397  llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
398  FieldDecl *LambdaThisCaptureField = nullptr;
399  const CodeGen::CGBlockInfo *BlockInfo = nullptr;
400
401public:
402  /// Constructs region for combined constructs.
403  /// \param CodeGen Code generation sequence for combined directives. Includes
404  /// a list of functions used for code generation of implicitly inlined
405  /// regions.
406  InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
407                          OpenMPDirectiveKind Kind, bool HasCancel)
408      : CGF(CGF) {
409    // Start emission for the construct.
410    CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
411        CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
412    std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
413    LambdaThisCaptureField = CGF.LambdaThisCaptureField;
414    CGF.LambdaThisCaptureField = nullptr;
415    BlockInfo = CGF.BlockInfo;
416    CGF.BlockInfo = nullptr;
417  }
418
419  ~InlinedOpenMPRegionRAII() {
420    // Restore original CapturedStmtInfo only if we're done with code emission.
421    auto *OldCSI =
422        cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
423    delete CGF.CapturedStmtInfo;
424    CGF.CapturedStmtInfo = OldCSI;
425    std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
426    CGF.LambdaThisCaptureField = LambdaThisCaptureField;
427    CGF.BlockInfo = BlockInfo;
428  }
429};
430
431/// Values for bit flags used in the ident_t to describe the fields.
432/// All enumeric elements are named and described in accordance with the code
433/// from https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
434enum OpenMPLocationFlags : unsigned {
435  /// Use trampoline for internal microtask.
436  OMP_IDENT_IMD = 0x01,
437  /// Use c-style ident structure.
438  OMP_IDENT_KMPC = 0x02,
439  /// Atomic reduction option for kmpc_reduce.
440  OMP_ATOMIC_REDUCE = 0x10,
441  /// Explicit 'barrier' directive.
442  OMP_IDENT_BARRIER_EXPL = 0x20,
443  /// Implicit barrier in code.
444  OMP_IDENT_BARRIER_IMPL = 0x40,
445  /// Implicit barrier in 'for' directive.
446  OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
447  /// Implicit barrier in 'sections' directive.
448  OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
449  /// Implicit barrier in 'single' directive.
450  OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
451  /// Call of __kmp_for_static_init for static loop.
452  OMP_IDENT_WORK_LOOP = 0x200,
453  /// Call of __kmp_for_static_init for sections.
454  OMP_IDENT_WORK_SECTIONS = 0x400,
455  /// Call of __kmp_for_static_init for distribute.
456  OMP_IDENT_WORK_DISTRIBUTE = 0x800,
457  LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)
458};
459
460/// Describes ident structure that describes a source location.
461/// All descriptions are taken from
462/// https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
463/// Original structure:
464/// typedef struct ident {
465///    kmp_int32 reserved_1;   /**<  might be used in Fortran;
466///                                  see above  */
467///    kmp_int32 flags;        /**<  also f.flags; KMP_IDENT_xxx flags;
468///                                  KMP_IDENT_KMPC identifies this union
469///                                  member  */
470///    kmp_int32 reserved_2;   /**<  not really used in Fortran any more;
471///                                  see above */
472///#if USE_ITT_BUILD
473///                            /*  but currently used for storing
474///                                region-specific ITT */
475///                            /*  contextual information. */
476///#endif /* USE_ITT_BUILD */
477///    kmp_int32 reserved_3;   /**< source[4] in Fortran, do not use for
478///                                 C++  */
479///    char const *psource;    /**< String describing the source location.
480///                            The string is composed of semi-colon separated
481//                             fields which describe the source file,
482///                            the function and a pair of line numbers that
483///                            delimit the construct.
484///                             */
485/// } ident_t;
486enum IdentFieldIndex {
487  /// might be used in Fortran
488  IdentField_Reserved_1,
489  /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
490  IdentField_Flags,
491  /// Not really used in Fortran any more
492  IdentField_Reserved_2,
493  /// Source[4] in Fortran, do not use for C++
494  IdentField_Reserved_3,
495  /// String describing the source location. The string is composed of
496  /// semi-colon separated fields which describe the source file, the function
497  /// and a pair of line numbers that delimit the construct.
498  IdentField_PSource
499};
500
501/// Schedule types for 'omp for' loops (these enumerators are taken from
502/// the enum sched_type in kmp.h).
503enum OpenMPSchedType {
504  /// Lower bound for default (unordered) versions.
505  OMP_sch_lower = 32,
506  OMP_sch_static_chunked = 33,
507  OMP_sch_static = 34,
508  OMP_sch_dynamic_chunked = 35,
509  OMP_sch_guided_chunked = 36,
510  OMP_sch_runtime = 37,
511  OMP_sch_auto = 38,
512  /// static with chunk adjustment (e.g., simd)
513  OMP_sch_static_balanced_chunked = 45,
514  /// Lower bound for 'ordered' versions.
515  OMP_ord_lower = 64,
516  OMP_ord_static_chunked = 65,
517  OMP_ord_static = 66,
518  OMP_ord_dynamic_chunked = 67,
519  OMP_ord_guided_chunked = 68,
520  OMP_ord_runtime = 69,
521  OMP_ord_auto = 70,
522  OMP_sch_default = OMP_sch_static,
523  /// dist_schedule types
524  OMP_dist_sch_static_chunked = 91,
525  OMP_dist_sch_static = 92,
526  /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
527  /// Set if the monotonic schedule modifier was present.
528  OMP_sch_modifier_monotonic = (1 << 29),
529  /// Set if the nonmonotonic schedule modifier was present.
530  OMP_sch_modifier_nonmonotonic = (1 << 30),
531};
532
533enum OpenMPRTLFunction {
534  /// Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc,
535  /// kmpc_micro microtask, ...);
536  OMPRTL__kmpc_fork_call,
537  /// Call to void *__kmpc_threadprivate_cached(ident_t *loc,
538  /// kmp_int32 global_tid, void *data, size_t size, void ***cache);
539  OMPRTL__kmpc_threadprivate_cached,
540  /// Call to void __kmpc_threadprivate_register( ident_t *,
541  /// void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
542  OMPRTL__kmpc_threadprivate_register,
543  // Call to __kmpc_int32 kmpc_global_thread_num(ident_t *loc);
544  OMPRTL__kmpc_global_thread_num,
545  // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
546  // kmp_critical_name *crit);
547  OMPRTL__kmpc_critical,
548  // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32
549  // global_tid, kmp_critical_name *crit, uintptr_t hint);
550  OMPRTL__kmpc_critical_with_hint,
551  // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
552  // kmp_critical_name *crit);
553  OMPRTL__kmpc_end_critical,
554  // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
555  // global_tid);
556  OMPRTL__kmpc_cancel_barrier,
557  // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
558  OMPRTL__kmpc_barrier,
559  // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
560  OMPRTL__kmpc_for_static_fini,
561  // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
562  // global_tid);
563  OMPRTL__kmpc_serialized_parallel,
564  // Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
565  // global_tid);
566  OMPRTL__kmpc_end_serialized_parallel,
567  // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
568  // kmp_int32 num_threads);
569  OMPRTL__kmpc_push_num_threads,
570  // Call to void __kmpc_flush(ident_t *loc);
571  OMPRTL__kmpc_flush,
572  // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
573  OMPRTL__kmpc_master,
574  // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
575  OMPRTL__kmpc_end_master,
576  // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
577  // int end_part);
578  OMPRTL__kmpc_omp_taskyield,
579  // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);
580  OMPRTL__kmpc_single,
581  // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
582  OMPRTL__kmpc_end_single,
583  // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
584  // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
585  // kmp_routine_entry_t *task_entry);
586  OMPRTL__kmpc_omp_task_alloc,
587  // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t *
588  // new_task);
589  OMPRTL__kmpc_omp_task,
590  // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
591  // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
592  // kmp_int32 didit);
593  OMPRTL__kmpc_copyprivate,
594  // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
595  // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
596  // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
597  OMPRTL__kmpc_reduce,
598  // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
599  // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
600  // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
601  // *lck);
602  OMPRTL__kmpc_reduce_nowait,
603  // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
604  // kmp_critical_name *lck);
605  OMPRTL__kmpc_end_reduce,
606  // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
607  // kmp_critical_name *lck);
608  OMPRTL__kmpc_end_reduce_nowait,
609  // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
610  // kmp_task_t * new_task);
611  OMPRTL__kmpc_omp_task_begin_if0,
612  // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
613  // kmp_task_t * new_task);
614  OMPRTL__kmpc_omp_task_complete_if0,
615  // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
616  OMPRTL__kmpc_ordered,
617  // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
618  OMPRTL__kmpc_end_ordered,
619  // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
620  // global_tid);
621  OMPRTL__kmpc_omp_taskwait,
622  // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
623  OMPRTL__kmpc_taskgroup,
624  // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
625  OMPRTL__kmpc_end_taskgroup,
626  // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
627  // int proc_bind);
628  OMPRTL__kmpc_push_proc_bind,
629  // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32
630  // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t
631  // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
632  OMPRTL__kmpc_omp_task_with_deps,
633  // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32
634  // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
635  // ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
636  OMPRTL__kmpc_omp_wait_deps,
637  // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
638  // global_tid, kmp_int32 cncl_kind);
639  OMPRTL__kmpc_cancellationpoint,
640  // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
641  // kmp_int32 cncl_kind);
642  OMPRTL__kmpc_cancel,
643  // Call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
644  // kmp_int32 num_teams, kmp_int32 thread_limit);
645  OMPRTL__kmpc_push_num_teams,
646  // Call to void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
647  // microtask, ...);
648  OMPRTL__kmpc_fork_teams,
649  // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
650  // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
651  // sched, kmp_uint64 grainsize, void *task_dup);
652  OMPRTL__kmpc_taskloop,
653  // Call to void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
654  // num_dims, struct kmp_dim *dims);
655  OMPRTL__kmpc_doacross_init,
656  // Call to void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
657  OMPRTL__kmpc_doacross_fini,
658  // Call to void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
659  // *vec);
660  OMPRTL__kmpc_doacross_post,
661  // Call to void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
662  // *vec);
663  OMPRTL__kmpc_doacross_wait,
664  // Call to void *__kmpc_task_reduction_init(int gtid, int num_data, void
665  // *data);
666  OMPRTL__kmpc_task_reduction_init,
667  // Call to void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
668  // *d);
669  OMPRTL__kmpc_task_reduction_get_th_data,
670  // Call to void *__kmpc_alloc(int gtid, size_t sz, const omp_allocator_t *al);
671  OMPRTL__kmpc_alloc,
672  // Call to void __kmpc_free(int gtid, void *ptr, const omp_allocator_t *al);
673  OMPRTL__kmpc_free,
674
675  //
676  // Offloading related calls
677  //
678  // Call to void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
679  // size);
680  OMPRTL__kmpc_push_target_tripcount,
681  // Call to int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
682  // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
683  // *arg_types);
684  OMPRTL__tgt_target,
685  // Call to int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
686  // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
687  // *arg_types);
688  OMPRTL__tgt_target_nowait,
689  // Call to int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
690  // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
691  // *arg_types, int32_t num_teams, int32_t thread_limit);
692  OMPRTL__tgt_target_teams,
693  // Call to int32_t __tgt_target_teams_nowait(int64_t device_id, void
694  // *host_ptr, int32_t arg_num, void** args_base, void **args, size_t
695  // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
696  OMPRTL__tgt_target_teams_nowait,
697  // Call to void __tgt_register_lib(__tgt_bin_desc *desc);
698  OMPRTL__tgt_register_lib,
699  // Call to void __tgt_unregister_lib(__tgt_bin_desc *desc);
700  OMPRTL__tgt_unregister_lib,
701  // Call to void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
702  // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
703  OMPRTL__tgt_target_data_begin,
704  // Call to void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
705  // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
706  // *arg_types);
707  OMPRTL__tgt_target_data_begin_nowait,
708  // Call to void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
709  // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
710  OMPRTL__tgt_target_data_end,
711  // Call to void __tgt_target_data_end_nowait(int64_t device_id, int32_t
712  // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
713  // *arg_types);
714  OMPRTL__tgt_target_data_end_nowait,
715  // Call to void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
716  // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
717  OMPRTL__tgt_target_data_update,
718  // Call to void __tgt_target_data_update_nowait(int64_t device_id, int32_t
719  // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
720  // *arg_types);
721  OMPRTL__tgt_target_data_update_nowait,
722};
723
724/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
725/// region.
726class CleanupTy final : public EHScopeStack::Cleanup {
727  PrePostActionTy *Action;
728
729public:
730  explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
731  void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
732    if (!CGF.HaveInsertPoint())
733      return;
734    Action->Exit(CGF);
735  }
736};
737
738} // anonymous namespace
739
740void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
741  CodeGenFunction::RunCleanupsScope Scope(CGF);
742  if (PrePostAction) {
743    CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
744    Callback(CodeGen, CGF, *PrePostAction);
745  } else {
746    PrePostActionTy Action;
747    Callback(CodeGen, CGF, Action);
748  }
749}
750
751/// Check if the combiner is a call to UDR combiner and if it is so return the
752/// UDR decl used for reduction.
753static const OMPDeclareReductionDecl *
754getReductionInit(const Expr *ReductionOp) {
755  if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
756    if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
757      if (const auto *DRE =
758              dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
759        if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
760          return DRD;
761  return nullptr;
762}
763
764static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
765                                             const OMPDeclareReductionDecl *DRD,
766                                             const Expr *InitOp,
767                                             Address Private, Address Original,
768                                             QualType Ty) {
769  if (DRD->getInitializer()) {
770    std::pair<llvm::Function *, llvm::Function *> Reduction =
771        CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
772    const auto *CE = cast<CallExpr>(InitOp);
773    const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
774    const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
775    const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
776    const auto *LHSDRE =
777        cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
778    const auto *RHSDRE =
779        cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
780    CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
781    PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
782                            [=]() { return Private; });
783    PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
784                            [=]() { return Original; });
785    (void)PrivateScope.Privatize();
786    RValue Func = RValue::get(Reduction.second);
787    CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
788    CGF.EmitIgnoredExpr(InitOp);
789  } else {
790    llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
791    std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
792    auto *GV = new llvm::GlobalVariable(
793        CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
794        llvm::GlobalValue::PrivateLinkage, Init, Name);
795    LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
796    RValue InitRVal;
797    switch (CGF.getEvaluationKind(Ty)) {
798    case TEK_Scalar:
799      InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
800      break;
801    case TEK_Complex:
802      InitRVal =
803          RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
804      break;
805    case TEK_Aggregate:
806      InitRVal = RValue::getAggregate(LV.getAddress());
807      break;
808    }
809    OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
810    CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
811    CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
812                         /*IsInitializer=*/false);
813  }
814}
815
816/// Emit initialization of arrays of complex types.
817/// \param DestAddr Address of the array.
818/// \param Type Type of array.
819/// \param Init Initial expression of array.
820/// \param SrcAddr Address of the original array.
821static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
822                                 QualType Type, bool EmitDeclareReductionInit,
823                                 const Expr *Init,
824                                 const OMPDeclareReductionDecl *DRD,
825                                 Address SrcAddr = Address::invalid()) {
826  // Perform element-by-element initialization.
827  QualType ElementTy;
828
829  // Drill down to the base element type on both arrays.
830  const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
831  llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
832  DestAddr =
833      CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
834  if (DRD)
835    SrcAddr =
836        CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
837
838  llvm::Value *SrcBegin = nullptr;
839  if (DRD)
840    SrcBegin = SrcAddr.getPointer();
841  llvm::Value *DestBegin = DestAddr.getPointer();
842  // Cast from pointer to array type to pointer to single element.
843  llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
844  // The basic structure here is a while-do loop.
845  llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
846  llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
847  llvm::Value *IsEmpty =
848      CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
849  CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
850
851  // Enter the loop body, making that address the current address.
852  llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
853  CGF.EmitBlock(BodyBB);
854
855  CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
856
857  llvm::PHINode *SrcElementPHI = nullptr;
858  Address SrcElementCurrent = Address::invalid();
859  if (DRD) {
860    SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
861                                          "omp.arraycpy.srcElementPast");
862    SrcElementPHI->addIncoming(SrcBegin, EntryBB);
863    SrcElementCurrent =
864        Address(SrcElementPHI,
865                SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
866  }
867  llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
868      DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
869  DestElementPHI->addIncoming(DestBegin, EntryBB);
870  Address DestElementCurrent =
871      Address(DestElementPHI,
872              DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
873
874  // Emit copy.
875  {
876    CodeGenFunction::RunCleanupsScope InitScope(CGF);
877    if (EmitDeclareReductionInit) {
878      emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
879                                       SrcElementCurrent, ElementTy);
880    } else
881      CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
882                           /*IsInitializer=*/false);
883  }
884
885  if (DRD) {
886    // Shift the address forward by one element.
887    llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
888        SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
889    SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
890  }
891
892  // Shift the address forward by one element.
893  llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
894      DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
895  // Check whether we've reached the end.
896  llvm::Value *Done =
897      CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
898  CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
899  DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
900
901  // Done.
902  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
903}
904
905LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
906  return CGF.EmitOMPSharedLValue(E);
907}
908
909LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
910                                            const Expr *E) {
911  if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
912    return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
913  return LValue();
914}
915
916void ReductionCodeGen::emitAggregateInitialization(
917    CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
918    const OMPDeclareReductionDecl *DRD) {
919  // Emit VarDecl with copy init for arrays.
920  // Get the address of the original variable captured in current
921  // captured region.
922  const auto *PrivateVD =
923      cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
924  bool EmitDeclareReductionInit =
925      DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
926  EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
927                       EmitDeclareReductionInit,
928                       EmitDeclareReductionInit ? ClausesData[N].ReductionOp
929                                                : PrivateVD->getInit(),
930                       DRD, SharedLVal.getAddress());
931}
932
933ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
934                                   ArrayRef<const Expr *> Privates,
935                                   ArrayRef<const Expr *> ReductionOps) {
936  ClausesData.reserve(Shareds.size());
937  SharedAddresses.reserve(Shareds.size());
938  Sizes.reserve(Shareds.size());
939  BaseDecls.reserve(Shareds.size());
940  auto IPriv = Privates.begin();
941  auto IRed = ReductionOps.begin();
942  for (const Expr *Ref : Shareds) {
943    ClausesData.emplace_back(Ref, *IPriv, *IRed);
944    std::advance(IPriv, 1);
945    std::advance(IRed, 1);
946  }
947}
948
949void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) {
950   (0) . __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 951, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(SharedAddresses.size() == N &&
951 (0) . __assert_fail ("SharedAddresses.size() == N && \"Number of generated lvalues must be exactly N.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 951, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">         "Number of generated lvalues must be exactly N.");
952  LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
953  LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
954  SharedAddresses.emplace_back(First, Second);
955}
956
957void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
958  const auto *PrivateVD =
959      cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
960  QualType PrivateType = PrivateVD->getType();
961  bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
962  if (!PrivateType->isVariablyModifiedType()) {
963    Sizes.emplace_back(
964        CGF.getTypeSize(
965            SharedAddresses[N].first.getType().getNonReferenceType()),
966        nullptr);
967    return;
968  }
969  llvm::Value *Size;
970  llvm::Value *SizeInChars;
971  auto *ElemType =
972      cast<llvm::PointerType>(SharedAddresses[N].first.getPointer()->getType())
973          ->getElementType();
974  auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
975  if (AsArraySection) {
976    Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(),
977                                     SharedAddresses[N].first.getPointer());
978    Size = CGF.Builder.CreateNUWAdd(
979        Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
980    SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
981  } else {
982    SizeInChars = CGF.getTypeSize(
983        SharedAddresses[N].first.getType().getNonReferenceType());
984    Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
985  }
986  Sizes.emplace_back(SizeInChars, Size);
987  CodeGenFunction::OpaqueValueMapping OpaqueMap(
988      CGF,
989      cast<OpaqueValueExpr>(
990          CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
991      RValue::get(Size));
992  CGF.EmitVariablyModifiedType(PrivateType);
993}
994
995void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
996                                         llvm::Value *Size) {
997  const auto *PrivateVD =
998      cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
999  QualType PrivateType = PrivateVD->getType();
1000  if (!PrivateType->isVariablyModifiedType()) {
1001     (0) . __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 1003, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!Size && !Sizes[N].second &&
1002 (0) . __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 1003, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">           "Size should be nullptr for non-variably modified reduction "
1003 (0) . __assert_fail ("!Size && !Sizes[N].second && \"Size should be nullptr for non-variably modified reduction \" \"items.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 1003, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">           "items.");
1004    return;
1005  }
1006  CodeGenFunction::OpaqueValueMapping OpaqueMap(
1007      CGF,
1008      cast<OpaqueValueExpr>(
1009          CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1010      RValue::get(Size));
1011  CGF.EmitVariablyModifiedType(PrivateType);
1012}
1013
1014void ReductionCodeGen::emitInitialization(
1015    CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
1016    llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
1017   (0) . __assert_fail ("SharedAddresses.size() > N && \"No variable was generated\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 1017, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(SharedAddresses.size() > N && "No variable was generated");
1018  const auto *PrivateVD =
1019      cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1020  const OMPDeclareReductionDecl *DRD =
1021      getReductionInit(ClausesData[N].ReductionOp);
1022  QualType PrivateType = PrivateVD->getType();
1023  PrivateAddr = CGF.Builder.CreateElementBitCast(
1024      PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1025  QualType SharedType = SharedAddresses[N].first.getType();
1026  SharedLVal = CGF.MakeAddrLValue(
1027      CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(),
1028                                       CGF.ConvertTypeForMem(SharedType)),
1029      SharedType, SharedAddresses[N].first.getBaseInfo(),
1030      CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
1031  if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
1032    emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
1033  } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1034    emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
1035                                     PrivateAddr, SharedLVal.getAddress(),
1036                                     SharedLVal.getType());
1037  } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
1038             !CGF.isTrivialInitializer(PrivateVD->getInit())) {
1039    CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
1040                         PrivateVD->getType().getQualifiers(),
1041                         /*IsInitializer=*/false);
1042  }
1043}
1044
1045bool ReductionCodeGen::needCleanups(unsigned N) {
1046  const auto *PrivateVD =
1047      cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1048  QualType PrivateType = PrivateVD->getType();
1049  QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1050  return DTorKind != QualType::DK_none;
1051}
1052
1053void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
1054                                    Address PrivateAddr) {
1055  const auto *PrivateVD =
1056      cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1057  QualType PrivateType = PrivateVD->getType();
1058  QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1059  if (needCleanups(N)) {
1060    PrivateAddr = CGF.Builder.CreateElementBitCast(
1061        PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1062    CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
1063  }
1064}
1065
1066static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1067                          LValue BaseLV) {
1068  BaseTy = BaseTy.getNonReferenceType();
1069  while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1070         !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1071    if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
1072      BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy);
1073    } else {
1074      LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy);
1075      BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
1076    }
1077    BaseTy = BaseTy->getPointeeType();
1078  }
1079  return CGF.MakeAddrLValue(
1080      CGF.Builder.CreateElementBitCast(BaseLV.getAddress(),
1081                                       CGF.ConvertTypeForMem(ElTy)),
1082      BaseLV.getType(), BaseLV.getBaseInfo(),
1083      CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
1084}
1085
1086static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1087                          llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
1088                          llvm::Value *Addr) {
1089  Address Tmp = Address::invalid();
1090  Address TopTmp = Address::invalid();
1091  Address MostTopTmp = Address::invalid();
1092  BaseTy = BaseTy.getNonReferenceType();
1093  while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1094         !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1095    Tmp = CGF.CreateMemTemp(BaseTy);
1096    if (TopTmp.isValid())
1097      CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
1098    else
1099      MostTopTmp = Tmp;
1100    TopTmp = Tmp;
1101    BaseTy = BaseTy->getPointeeType();
1102  }
1103  llvm::Type *Ty = BaseLVType;
1104  if (Tmp.isValid())
1105    Ty = Tmp.getElementType();
1106  Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
1107  if (Tmp.isValid()) {
1108    CGF.Builder.CreateStore(Addr, Tmp);
1109    return MostTopTmp;
1110  }
1111  return Address(Addr, BaseLVAlignment);
1112}
1113
1114static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
1115  const VarDecl *OrigVD = nullptr;
1116  if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
1117    const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
1118    while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1119      Base = TempOASE->getBase()->IgnoreParenImpCasts();
1120    while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1121      Base = TempASE->getBase()->IgnoreParenImpCasts();
1122    DE = cast<DeclRefExpr>(Base);
1123    OrigVD = cast<VarDecl>(DE->getDecl());
1124  } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
1125    const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
1126    while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1127      Base = TempASE->getBase()->IgnoreParenImpCasts();
1128    DE = cast<DeclRefExpr>(Base);
1129    OrigVD = cast<VarDecl>(DE->getDecl());
1130  }
1131  return OrigVD;
1132}
1133
1134Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
1135                                               Address PrivateAddr) {
1136  const DeclRefExpr *DE;
1137  if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
1138    BaseDecls.emplace_back(OrigVD);
1139    LValue OriginalBaseLValue = CGF.EmitLValue(DE);
1140    LValue BaseLValue =
1141        loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1142                    OriginalBaseLValue);
1143    llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1144        BaseLValue.getPointer(), SharedAddresses[N].first.getPointer());
1145    llvm::Value *PrivatePointer =
1146        CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1147            PrivateAddr.getPointer(),
1148            SharedAddresses[N].first.getAddress().getType());
1149    llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1150    return castToBase(CGF, OrigVD->getType(),
1151                      SharedAddresses[N].first.getType(),
1152                      OriginalBaseLValue.getAddress().getType(),
1153                      OriginalBaseLValue.getAlignment(), Ptr);
1154  }
1155  BaseDecls.emplace_back(
1156      cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1157  return PrivateAddr;
1158}
1159
1160bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1161  const OMPDeclareReductionDecl *DRD =
1162      getReductionInit(ClausesData[N].ReductionOp);
1163  return DRD && DRD->getInitializer();
1164}
1165
1166LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1167  return CGF.EmitLoadOfPointerLValue(
1168      CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1169      getThreadIDVariable()->getType()->castAs<PointerType>());
1170}
1171
1172void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1173  if (!CGF.HaveInsertPoint())
1174    return;
1175  // 1.2.2 OpenMP Language Terminology
1176  // Structured block - An executable statement with a single entry at the
1177  // top and a single exit at the bottom.
1178  // The point of exit cannot be a branch out of the structured block.
1179  // longjmp() and throw() must not violate the entry/exit criteria.
1180  CGF.EHStack.pushTerminate();
1181  CodeGen(CGF);
1182  CGF.EHStack.popTerminate();
1183}
1184
1185LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1186    CodeGenFunction &CGF) {
1187  return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1188                            getThreadIDVariable()->getType(),
1189                            AlignmentSource::Decl);
1190}
1191
1192static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1193                                       QualType FieldTy) {
1194  auto *Field = FieldDecl::Create(
1195      C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1196      C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1197      /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1198  Field->setAccess(AS_public);
1199  DC->addDecl(Field);
1200  return Field;
1201}
1202
1203CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1204                                 StringRef Separator)
1205    : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1206      OffloadEntriesInfoManager(CGM) {
1207  ASTContext &C = CGM.getContext();
1208  RecordDecl *RD = C.buildImplicitRecord("ident_t");
1209  QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
1210  RD->startDefinition();
1211  // reserved_1
1212  addFieldToRecordDecl(C, RD, KmpInt32Ty);
1213  // flags
1214  addFieldToRecordDecl(C, RD, KmpInt32Ty);
1215  // reserved_2
1216  addFieldToRecordDecl(C, RD, KmpInt32Ty);
1217  // reserved_3
1218  addFieldToRecordDecl(C, RD, KmpInt32Ty);
1219  // psource
1220  addFieldToRecordDecl(C, RD, C.VoidPtrTy);
1221  RD->completeDefinition();
1222  IdentQTy = C.getRecordType(RD);
1223  IdentTy = CGM.getTypes().ConvertRecordDeclType(RD);
1224  KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1225
1226  loadOffloadInfoMetadata();
1227}
1228
1229void CGOpenMPRuntime::clear() {
1230  InternalVars.clear();
1231  // Clean non-target variable declarations possibly used only in debug info.
1232  for (const auto &Data : EmittedNonTargetVariables) {
1233    if (!Data.getValue().pointsToAliveValue())
1234      continue;
1235    auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1236    if (!GV)
1237      continue;
1238    if (!GV->isDeclaration() || GV->getNumUses() > 0)
1239      continue;
1240    GV->eraseFromParent();
1241  }
1242}
1243
1244std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1245  SmallString<128> Buffer;
1246  llvm::raw_svector_ostream OS(Buffer);
1247  StringRef Sep = FirstSeparator;
1248  for (StringRef Part : Parts) {
1249    OS << Sep << Part;
1250    Sep = Separator;
1251  }
1252  return OS.str();
1253}
1254
1255static llvm::Function *
1256emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1257                          const Expr *CombinerInitializer, const VarDecl *In,
1258                          const VarDecl *Out, bool IsCombiner) {
1259  // void .omp_combiner.(Ty *in, Ty *out);
1260  ASTContext &C = CGM.getContext();
1261  QualType PtrTy = C.getPointerType(Ty).withRestrict();
1262  FunctionArgList Args;
1263  ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1264                               /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1265  ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1266                              /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1267  Args.push_back(&OmpOutParm);
1268  Args.push_back(&OmpInParm);
1269  const CGFunctionInfo &FnInfo =
1270      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1271  llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1272  std::string Name = CGM.getOpenMPRuntime().getName(
1273      {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1274  auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1275                                    Name, &CGM.getModule());
1276  CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1277  Fn->removeFnAttr(llvm::Attribute::NoInline);
1278  Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1279  Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1280  CodeGenFunction CGF(CGM);
1281  // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1282  // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1283  CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1284                    Out->getLocation());
1285  CodeGenFunction::OMPPrivateScope Scope(CGF);
1286  Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1287  Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1288    return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1289        .getAddress();
1290  });
1291  Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1292  Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1293    return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1294        .getAddress();
1295  });
1296  (void)Scope.Privatize();
1297  if (!IsCombiner && Out->hasInit() &&
1298      !CGF.isTrivialInitializer(Out->getInit())) {
1299    CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1300                         Out->getType().getQualifiers(),
1301                         /*IsInitializer=*/true);
1302  }
1303  if (CombinerInitializer)
1304    CGF.EmitIgnoredExpr(CombinerInitializer);
1305  Scope.ForceCleanup();
1306  CGF.FinishFunction();
1307  return Fn;
1308}
1309
1310void CGOpenMPRuntime::emitUserDefinedReduction(
1311    CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1312  if (UDRMap.count(D) > 0)
1313    return;
1314  llvm::Function *Combiner = emitCombinerOrInitializer(
1315      CGM, D->getType(), D->getCombiner(),
1316      cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1317      cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1318      /*IsCombiner=*/true);
1319  llvm::Function *Initializer = nullptr;
1320  if (const Expr *Init = D->getInitializer()) {
1321    Initializer = emitCombinerOrInitializer(
1322        CGM, D->getType(),
1323        D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1324                                                                     : nullptr,
1325        cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1326        cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1327        /*IsCombiner=*/false);
1328  }
1329  UDRMap.try_emplace(D, Combiner, Initializer);
1330  if (CGF) {
1331    auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1332    Decls.second.push_back(D);
1333  }
1334}
1335
1336std::pair<llvm::Function *, llvm::Function *>
1337CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1338  auto I = UDRMap.find(D);
1339  if (I != UDRMap.end())
1340    return I->second;
1341  emitUserDefinedReduction(/*CGF=*/nullptr, D);
1342  return UDRMap.lookup(D);
1343}
1344
1345static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1346    CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1347    const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1348    const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1349   (0) . __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 1350, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(ThreadIDVar->getType()->isPointerType() &&
1350 (0) . __assert_fail ("ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 *\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 1350, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">         "thread id variable must be of type kmp_int32 *");
1351  CodeGenFunction CGF(CGM, true);
1352  bool HasCancel = false;
1353  if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1354    HasCancel = OPD->hasCancel();
1355  else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1356    HasCancel = OPSD->hasCancel();
1357  else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1358    HasCancel = OPFD->hasCancel();
1359  else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1360    HasCancel = OPFD->hasCancel();
1361  else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1362    HasCancel = OPFD->hasCancel();
1363  else if (const auto *OPFD =
1364               dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1365    HasCancel = OPFD->hasCancel();
1366  else if (const auto *OPFD =
1367               dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1368    HasCancel = OPFD->hasCancel();
1369  CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1370                                    HasCancel, OutlinedHelperName);
1371  CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1372  return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
1373}
1374
1375llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1376    const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1377    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1378  const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1379  return emitParallelOrTeamsOutlinedFunction(
1380      CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1381}
1382
1383llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1384    const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1385    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1386  const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1387  return emitParallelOrTeamsOutlinedFunction(
1388      CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1389}
1390
1391llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1392    const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1393    const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1394    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1395    bool Tied, unsigned &NumberOfParts) {
1396  auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1397                                              PrePostActionTy &) {
1398    llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1399    llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1400    llvm::Value *TaskArgs[] = {
1401        UpLoc, ThreadID,
1402        CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1403                                    TaskTVar->getType()->castAs<PointerType>())
1404            .getPointer()};
1405    CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
1406  };
1407  CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1408                                                            UntiedCodeGen);
1409  CodeGen.setAction(Action);
1410   (0) . __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 1411, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!ThreadIDVar->getType()->isPointerType() &&
1411 (0) . __assert_fail ("!ThreadIDVar->getType()->isPointerType() && \"thread id variable must be of type kmp_int32 for tasks\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 1411, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">         "thread id variable must be of type kmp_int32 for tasks");
1412  const OpenMPDirectiveKind Region =
1413      isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1414                                                      : OMPD_task;
1415  const CapturedStmt *CS = D.getCapturedStmt(Region);
1416  const auto *TD = dyn_cast<OMPTaskDirective>(&D);
1417  CodeGenFunction CGF(CGM, true);
1418  CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1419                                        InnermostKind,
1420                                        TD ? TD->hasCancel() : false, Action);
1421  CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1422  llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1423  if (!Tied)
1424    NumberOfParts = Action.getNumberOfParts();
1425  return Res;
1426}
1427
1428static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1429                             const RecordDecl *RD, const CGRecordLayout &RL,
1430                             ArrayRef<llvm::Constant *> Data) {
1431  llvm::StructType *StructTy = RL.getLLVMType();
1432  unsigned PrevIdx = 0;
1433  ConstantInitBuilder CIBuilder(CGM);
1434  auto DI = Data.begin();
1435  for (const FieldDecl *FD : RD->fields()) {
1436    unsigned Idx = RL.getLLVMFieldNo(FD);
1437    // Fill the alignment.
1438    for (unsigned I = PrevIdx; I < Idx; ++I)
1439      Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1440    PrevIdx = Idx + 1;
1441    Fields.add(*DI);
1442    ++DI;
1443  }
1444}
1445
1446template <class... As>
1447static llvm::GlobalVariable *
1448createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1449                   ArrayRef<llvm::Constant *> Data, const Twine &Name,
1450                   As &&... Args) {
1451  const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1452  const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1453  ConstantInitBuilder CIBuilder(CGM);
1454  ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1455  buildStructValue(Fields, CGM, RD, RL, Data);
1456  return Fields.finishAndCreateGlobal(
1457      Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1458      std::forward<As>(Args)...);
1459}
1460
1461template <typename T>
1462static void
1463createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1464                                         ArrayRef<llvm::Constant *> Data,
1465                                         T &Parent) {
1466  const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1467  const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1468  ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1469  buildStructValue(Fields, CGM, RD, RL, Data);
1470  Fields.finishAndAddTo(Parent);
1471}
1472
1473Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) {
1474  CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1475  unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1476  FlagsTy FlagsKey(Flags, Reserved2Flags);
1477  llvm::Value *Entry = OpenMPDefaultLocMap.lookup(FlagsKey);
1478  if (!Entry) {
1479    if (!DefaultOpenMPPSource) {
1480      // Initialize default location for psource field of ident_t structure of
1481      // all ident_t objects. Format is ";file;function;line;column;;".
1482      // Taken from
1483      // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp_str.cpp
1484      DefaultOpenMPPSource =
1485          CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer();
1486      DefaultOpenMPPSource =
1487          llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
1488    }
1489
1490    llvm::Constant *Data[] = {
1491        llvm::ConstantInt::getNullValue(CGM.Int32Ty),
1492        llvm::ConstantInt::get(CGM.Int32Ty, Flags),
1493        llvm::ConstantInt::get(CGM.Int32Ty, Reserved2Flags),
1494        llvm::ConstantInt::getNullValue(CGM.Int32Ty), DefaultOpenMPPSource};
1495    llvm::GlobalValue *DefaultOpenMPLocation =
1496        createGlobalStruct(CGM, IdentQTy, isDefaultLocationConstant(), Data, "",
1497                           llvm::GlobalValue::PrivateLinkage);
1498    DefaultOpenMPLocation->setUnnamedAddr(
1499        llvm::GlobalValue::UnnamedAddr::Global);
1500
1501    OpenMPDefaultLocMap[FlagsKey] = Entry = DefaultOpenMPLocation;
1502  }
1503  return Address(Entry, Align);
1504}
1505
1506void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1507                                             bool AtCurrentPoint) {
1508  auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1509   (0) . __assert_fail ("!Elem.second.ServiceInsertPt && \"Insert point is set already.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 1509, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!Elem.second.ServiceInsertPt && "Insert point is set already.");
1510
1511  llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1512  if (AtCurrentPoint) {
1513    Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1514        Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1515  } else {
1516    Elem.second.ServiceInsertPt =
1517        new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1518    Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1519  }
1520}
1521
1522void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1523  auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1524  if (Elem.second.ServiceInsertPt) {
1525    llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1526    Elem.second.ServiceInsertPt = nullptr;
1527    Ptr->eraseFromParent();
1528  }
1529}
1530
1531llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1532                                                 SourceLocation Loc,
1533                                                 unsigned Flags) {
1534  Flags |= OMP_IDENT_KMPC;
1535  // If no debug info is generated - return global default location.
1536  if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1537      Loc.isInvalid())
1538    return getOrCreateDefaultLocation(Flags).getPointer();
1539
1540   (0) . __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 1540, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(CGF.CurFn && "No function in current CodeGenFunction.");
1541
1542  CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1543  Address LocValue = Address::invalid();
1544  auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1545  if (I != OpenMPLocThreadIDMap.end())
1546    LocValue = Address(I->second.DebugLoc, Align);
1547
1548  // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
1549  // GetOpenMPThreadID was called before this routine.
1550  if (!LocValue.isValid()) {
1551    // Generate "ident_t .kmpc_loc.addr;"
1552    Address AI = CGF.CreateMemTemp(IdentQTy, ".kmpc_loc.addr");
1553    auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1554    Elem.second.DebugLoc = AI.getPointer();
1555    LocValue = AI;
1556
1557    if (!Elem.second.ServiceInsertPt)
1558      setLocThreadIdInsertPt(CGF);
1559    CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1560    CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1561    CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
1562                             CGF.getTypeSize(IdentQTy));
1563  }
1564
1565  // char **psource = &.kmpc_loc_<flags>.addr.psource;
1566  LValue Base = CGF.MakeAddrLValue(LocValue, IdentQTy);
1567  auto Fields = cast<RecordDecl>(IdentQTy->getAsTagDecl())->field_begin();
1568  LValue PSource =
1569      CGF.EmitLValueForField(Base, *std::next(Fields, IdentField_PSource));
1570
1571  llvm::Value *OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
1572  if (OMPDebugLoc == nullptr) {
1573    SmallString<128> Buffer2;
1574    llvm::raw_svector_ostream OS2(Buffer2);
1575    // Build debug location
1576    PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1577    OS2 << ";" << PLoc.getFilename() << ";";
1578    if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1579      OS2 << FD->getQualifiedNameAsString();
1580    OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1581    OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
1582    OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
1583  }
1584  // *psource = ";<File>;<Function>;<Line>;<Column>;;";
1585  CGF.EmitStoreOfScalar(OMPDebugLoc, PSource);
1586
1587  // Our callers always pass this to a runtime function, so for
1588  // convenience, go ahead and return a naked pointer.
1589  return LocValue.getPointer();
1590}
1591
1592llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1593                                          SourceLocation Loc) {
1594   (0) . __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 1594, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(CGF.CurFn && "No function in current CodeGenFunction.");
1595
1596  llvm::Value *ThreadID = nullptr;
1597  // Check whether we've already cached a load of the thread id in this
1598  // function.
1599  auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1600  if (I != OpenMPLocThreadIDMap.end()) {
1601    ThreadID = I->second.ThreadID;
1602    if (ThreadID != nullptr)
1603      return ThreadID;
1604  }
1605  // If exceptions are enabled, do not use parameter to avoid possible crash.
1606  if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1607      !CGF.getLangOpts().CXXExceptions ||
1608      CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1609    if (auto *OMPRegionInfo =
1610            dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1611      if (OMPRegionInfo->getThreadIDVariable()) {
1612        // Check if this an outlined function with thread id passed as argument.
1613        LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1614        ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1615        // If value loaded in entry block, cache it and use it everywhere in
1616        // function.
1617        if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1618          auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1619          Elem.second.ThreadID = ThreadID;
1620        }
1621        return ThreadID;
1622      }
1623    }
1624  }
1625
1626  // This is not an outlined function region - need to call __kmpc_int32
1627  // kmpc_global_thread_num(ident_t *loc).
1628  // Generate thread id value and cache this value for use across the
1629  // function.
1630  auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1631  if (!Elem.second.ServiceInsertPt)
1632    setLocThreadIdInsertPt(CGF);
1633  CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1634  CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1635  llvm::CallInst *Call = CGF.Builder.CreateCall(
1636      createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1637      emitUpdateLocation(CGF, Loc));
1638  Call->setCallingConv(CGF.getRuntimeCC());
1639  Elem.second.ThreadID = Call;
1640  return Call;
1641}
1642
1643void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1644   (0) . __assert_fail ("CGF.CurFn && \"No function in current CodeGenFunction.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 1644, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(CGF.CurFn && "No function in current CodeGenFunction.");
1645  if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1646    clearLocThreadIdInsertPt(CGF);
1647    OpenMPLocThreadIDMap.erase(CGF.CurFn);
1648  }
1649  if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1650    for(auto *D : FunctionUDRMap[CGF.CurFn])
1651      UDRMap.erase(D);
1652    FunctionUDRMap.erase(CGF.CurFn);
1653  }
1654}
1655
1656llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1657  return IdentTy->getPointerTo();
1658}
1659
1660llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1661  if (!Kmpc_MicroTy) {
1662    // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1663    llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1664                                 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1665    Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1666  }
1667  return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1668}
1669
1670llvm::FunctionCallee CGOpenMPRuntime::createRuntimeFunction(unsigned Function) {
1671  llvm::FunctionCallee RTLFn = nullptr;
1672  switch (static_cast<OpenMPRTLFunction>(Function)) {
1673  case OMPRTL__kmpc_fork_call: {
1674    // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
1675    // microtask, ...);
1676    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1677                                getKmpc_MicroPointerTy()};
1678    auto *FnTy =
1679        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
1680    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
1681    if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
1682      if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
1683        llvm::LLVMContext &Ctx = F->getContext();
1684        llvm::MDBuilder MDB(Ctx);
1685        // Annotate the callback behavior of the __kmpc_fork_call:
1686        //  - The callback callee is argument number 2 (microtask).
1687        //  - The first two arguments of the callback callee are unknown (-1).
1688        //  - All variadic arguments to the __kmpc_fork_call are passed to the
1689        //    callback callee.
1690        F->addMetadata(
1691            llvm::LLVMContext::MD_callback,
1692            *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
1693                                        2, {-1, -1},
1694                                        /* VarArgsArePassed */ true)}));
1695      }
1696    }
1697    break;
1698  }
1699  case OMPRTL__kmpc_global_thread_num: {
1700    // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
1701    llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1702    auto *FnTy =
1703        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1704    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
1705    break;
1706  }
1707  case OMPRTL__kmpc_threadprivate_cached: {
1708    // Build void *__kmpc_threadprivate_cached(ident_t *loc,
1709    // kmp_int32 global_tid, void *data, size_t size, void ***cache);
1710    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1711                                CGM.VoidPtrTy, CGM.SizeTy,
1712                                CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1713    auto *FnTy =
1714        llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
1715    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
1716    break;
1717  }
1718  case OMPRTL__kmpc_critical: {
1719    // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1720    // kmp_critical_name *crit);
1721    llvm::Type *TypeParams[] = {
1722        getIdentTyPointerTy(), CGM.Int32Ty,
1723        llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1724    auto *FnTy =
1725        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1726    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
1727    break;
1728  }
1729  case OMPRTL__kmpc_critical_with_hint: {
1730    // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1731    // kmp_critical_name *crit, uintptr_t hint);
1732    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1733                                llvm::PointerType::getUnqual(KmpCriticalNameTy),
1734                                CGM.IntPtrTy};
1735    auto *FnTy =
1736        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1737    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint");
1738    break;
1739  }
1740  case OMPRTL__kmpc_threadprivate_register: {
1741    // Build void __kmpc_threadprivate_register(ident_t *, void *data,
1742    // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
1743    // typedef void *(*kmpc_ctor)(void *);
1744    auto *KmpcCtorTy =
1745        llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1746                                /*isVarArg*/ false)->getPointerTo();
1747    // typedef void *(*kmpc_cctor)(void *, void *);
1748    llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1749    auto *KmpcCopyCtorTy =
1750        llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1751                                /*isVarArg*/ false)
1752            ->getPointerTo();
1753    // typedef void (*kmpc_dtor)(void *);
1754    auto *KmpcDtorTy =
1755        llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
1756            ->getPointerTo();
1757    llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
1758                              KmpcCopyCtorTy, KmpcDtorTy};
1759    auto *FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1760                                        /*isVarArg*/ false);
1761    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
1762    break;
1763  }
1764  case OMPRTL__kmpc_end_critical: {
1765    // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1766    // kmp_critical_name *crit);
1767    llvm::Type *TypeParams[] = {
1768        getIdentTyPointerTy(), CGM.Int32Ty,
1769        llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1770    auto *FnTy =
1771        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1772    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
1773    break;
1774  }
1775  case OMPRTL__kmpc_cancel_barrier: {
1776    // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
1777    // global_tid);
1778    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1779    auto *FnTy =
1780        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1781    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
1782    break;
1783  }
1784  case OMPRTL__kmpc_barrier: {
1785    // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
1786    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1787    auto *FnTy =
1788        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1789    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
1790    break;
1791  }
1792  case OMPRTL__kmpc_for_static_fini: {
1793    // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
1794    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1795    auto *FnTy =
1796        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1797    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini");
1798    break;
1799  }
1800  case OMPRTL__kmpc_push_num_threads: {
1801    // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
1802    // kmp_int32 num_threads)
1803    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1804                                CGM.Int32Ty};
1805    auto *FnTy =
1806        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1807    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
1808    break;
1809  }
1810  case OMPRTL__kmpc_serialized_parallel: {
1811    // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
1812    // global_tid);
1813    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1814    auto *FnTy =
1815        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1816    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
1817    break;
1818  }
1819  case OMPRTL__kmpc_end_serialized_parallel: {
1820    // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
1821    // global_tid);
1822    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1823    auto *FnTy =
1824        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1825    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
1826    break;
1827  }
1828  case OMPRTL__kmpc_flush: {
1829    // Build void __kmpc_flush(ident_t *loc);
1830    llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1831    auto *FnTy =
1832        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1833    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
1834    break;
1835  }
1836  case OMPRTL__kmpc_master: {
1837    // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
1838    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1839    auto *FnTy =
1840        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1841    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
1842    break;
1843  }
1844  case OMPRTL__kmpc_end_master: {
1845    // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
1846    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1847    auto *FnTy =
1848        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1849    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
1850    break;
1851  }
1852  case OMPRTL__kmpc_omp_taskyield: {
1853    // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
1854    // int end_part);
1855    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1856    auto *FnTy =
1857        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1858    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
1859    break;
1860  }
1861  case OMPRTL__kmpc_single: {
1862    // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
1863    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1864    auto *FnTy =
1865        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1866    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
1867    break;
1868  }
1869  case OMPRTL__kmpc_end_single: {
1870    // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
1871    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1872    auto *FnTy =
1873        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1874    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
1875    break;
1876  }
1877  case OMPRTL__kmpc_omp_task_alloc: {
1878    // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
1879    // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1880    // kmp_routine_entry_t *task_entry);
1881     (0) . __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 1882, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(KmpRoutineEntryPtrTy != nullptr &&
1882 (0) . __assert_fail ("KmpRoutineEntryPtrTy != nullptr && \"Type kmp_routine_entry_t must be created.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 1882, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">           "Type kmp_routine_entry_t must be created.");
1883    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1884                                CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
1885    // Return void * and then cast to particular kmp_task_t type.
1886    auto *FnTy =
1887        llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
1888    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
1889    break;
1890  }
1891  case OMPRTL__kmpc_omp_task: {
1892    // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
1893    // *new_task);
1894    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1895                                CGM.VoidPtrTy};
1896    auto *FnTy =
1897        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1898    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
1899    break;
1900  }
1901  case OMPRTL__kmpc_copyprivate: {
1902    // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
1903    // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
1904    // kmp_int32 didit);
1905    llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1906    auto *CpyFnTy =
1907        llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
1908    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
1909                                CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
1910                                CGM.Int32Ty};
1911    auto *FnTy =
1912        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1913    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
1914    break;
1915  }
1916  case OMPRTL__kmpc_reduce: {
1917    // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
1918    // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
1919    // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
1920    llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1921    auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
1922                                               /*isVarArg=*/false);
1923    llvm::Type *TypeParams[] = {
1924        getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
1925        CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
1926        llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1927    auto *FnTy =
1928        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1929    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
1930    break;
1931  }
1932  case OMPRTL__kmpc_reduce_nowait: {
1933    // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
1934    // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
1935    // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
1936    // *lck);
1937    llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1938    auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
1939                                               /*isVarArg=*/false);
1940    llvm::Type *TypeParams[] = {
1941        getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
1942        CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
1943        llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1944    auto *FnTy =
1945        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1946    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
1947    break;
1948  }
1949  case OMPRTL__kmpc_end_reduce: {
1950    // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
1951    // kmp_critical_name *lck);
1952    llvm::Type *TypeParams[] = {
1953        getIdentTyPointerTy(), CGM.Int32Ty,
1954        llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1955    auto *FnTy =
1956        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1957    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
1958    break;
1959  }
1960  case OMPRTL__kmpc_end_reduce_nowait: {
1961    // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
1962    // kmp_critical_name *lck);
1963    llvm::Type *TypeParams[] = {
1964        getIdentTyPointerTy(), CGM.Int32Ty,
1965        llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1966    auto *FnTy =
1967        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1968    RTLFn =
1969        CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
1970    break;
1971  }
1972  case OMPRTL__kmpc_omp_task_begin_if0: {
1973    // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
1974    // *new_task);
1975    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1976                                CGM.VoidPtrTy};
1977    auto *FnTy =
1978        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1979    RTLFn =
1980        CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
1981    break;
1982  }
1983  case OMPRTL__kmpc_omp_task_complete_if0: {
1984    // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
1985    // *new_task);
1986    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1987                                CGM.VoidPtrTy};
1988    auto *FnTy =
1989        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1990    RTLFn = CGM.CreateRuntimeFunction(FnTy,
1991                                      /*Name=*/"__kmpc_omp_task_complete_if0");
1992    break;
1993  }
1994  case OMPRTL__kmpc_ordered: {
1995    // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
1996    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1997    auto *FnTy =
1998        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1999    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
2000    break;
2001  }
2002  case OMPRTL__kmpc_end_ordered: {
2003    // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
2004    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2005    auto *FnTy =
2006        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2007    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
2008    break;
2009  }
2010  case OMPRTL__kmpc_omp_taskwait: {
2011    // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
2012    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2013    auto *FnTy =
2014        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2015    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
2016    break;
2017  }
2018  case OMPRTL__kmpc_taskgroup: {
2019    // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
2020    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2021    auto *FnTy =
2022        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2023    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
2024    break;
2025  }
2026  case OMPRTL__kmpc_end_taskgroup: {
2027    // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
2028    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2029    auto *FnTy =
2030        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2031    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
2032    break;
2033  }
2034  case OMPRTL__kmpc_push_proc_bind: {
2035    // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
2036    // int proc_bind)
2037    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2038    auto *FnTy =
2039        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2040    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
2041    break;
2042  }
2043  case OMPRTL__kmpc_omp_task_with_deps: {
2044    // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
2045    // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
2046    // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
2047    llvm::Type *TypeParams[] = {
2048        getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
2049        CGM.VoidPtrTy,         CGM.Int32Ty, CGM.VoidPtrTy};
2050    auto *FnTy =
2051        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2052    RTLFn =
2053        CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
2054    break;
2055  }
2056  case OMPRTL__kmpc_omp_wait_deps: {
2057    // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
2058    // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
2059    // kmp_depend_info_t *noalias_dep_list);
2060    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2061                                CGM.Int32Ty,           CGM.VoidPtrTy,
2062                                CGM.Int32Ty,           CGM.VoidPtrTy};
2063    auto *FnTy =
2064        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2065    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
2066    break;
2067  }
2068  case OMPRTL__kmpc_cancellationpoint: {
2069    // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
2070    // global_tid, kmp_int32 cncl_kind)
2071    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2072    auto *FnTy =
2073        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2074    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
2075    break;
2076  }
2077  case OMPRTL__kmpc_cancel: {
2078    // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
2079    // kmp_int32 cncl_kind)
2080    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2081    auto *FnTy =
2082        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2083    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
2084    break;
2085  }
2086  case OMPRTL__kmpc_push_num_teams: {
2087    // Build void kmpc_push_num_teams (ident_t loc, kmp_int32 global_tid,
2088    // kmp_int32 num_teams, kmp_int32 num_threads)
2089    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2090        CGM.Int32Ty};
2091    auto *FnTy =
2092        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2093    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_teams");
2094    break;
2095  }
2096  case OMPRTL__kmpc_fork_teams: {
2097    // Build void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
2098    // microtask, ...);
2099    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2100                                getKmpc_MicroPointerTy()};
2101    auto *FnTy =
2102        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
2103    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_teams");
2104    if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
2105      if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
2106        llvm::LLVMContext &Ctx = F->getContext();
2107        llvm::MDBuilder MDB(Ctx);
2108        // Annotate the callback behavior of the __kmpc_fork_teams:
2109        //  - The callback callee is argument number 2 (microtask).
2110        //  - The first two arguments of the callback callee are unknown (-1).
2111        //  - All variadic arguments to the __kmpc_fork_teams are passed to the
2112        //    callback callee.
2113        F->addMetadata(
2114            llvm::LLVMContext::MD_callback,
2115            *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
2116                                        2, {-1, -1},
2117                                        /* VarArgsArePassed */ true)}));
2118      }
2119    }
2120    break;
2121  }
2122  case OMPRTL__kmpc_taskloop: {
2123    // Build void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
2124    // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
2125    // sched, kmp_uint64 grainsize, void *task_dup);
2126    llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2127                                CGM.IntTy,
2128                                CGM.VoidPtrTy,
2129                                CGM.IntTy,
2130                                CGM.Int64Ty->getPointerTo(),
2131                                CGM.Int64Ty->getPointerTo(),
2132                                CGM.Int64Ty,
2133                                CGM.IntTy,
2134                                CGM.IntTy,
2135                                CGM.Int64Ty,
2136                                CGM.VoidPtrTy};
2137    auto *FnTy =
2138        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2139    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_taskloop");
2140    break;
2141  }
2142  case OMPRTL__kmpc_doacross_init: {
2143    // Build void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
2144    // num_dims, struct kmp_dim *dims);
2145    llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2146                                CGM.Int32Ty,
2147                                CGM.Int32Ty,
2148                                CGM.VoidPtrTy};
2149    auto *FnTy =
2150        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2151    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_init");
2152    break;
2153  }
2154  case OMPRTL__kmpc_doacross_fini: {
2155    // Build void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
2156    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2157    auto *FnTy =
2158        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2159    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_fini");
2160    break;
2161  }
2162  case OMPRTL__kmpc_doacross_post: {
2163    // Build void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
2164    // *vec);
2165    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2166                                CGM.Int64Ty->getPointerTo()};
2167    auto *FnTy =
2168        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2169    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_post");
2170    break;
2171  }
2172  case OMPRTL__kmpc_doacross_wait: {
2173    // Build void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
2174    // *vec);
2175    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2176                                CGM.Int64Ty->getPointerTo()};
2177    auto *FnTy =
2178        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2179    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_wait");
2180    break;
2181  }
2182  case OMPRTL__kmpc_task_reduction_init: {
2183    // Build void *__kmpc_task_reduction_init(int gtid, int num_data, void
2184    // *data);
2185    llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy};
2186    auto *FnTy =
2187        llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2188    RTLFn =
2189        CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_task_reduction_init");
2190    break;
2191  }
2192  case OMPRTL__kmpc_task_reduction_get_th_data: {
2193    // Build void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
2194    // *d);
2195    llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2196    auto *FnTy =
2197        llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2198    RTLFn = CGM.CreateRuntimeFunction(
2199        FnTy, /*Name=*/"__kmpc_task_reduction_get_th_data");
2200    break;
2201  }
2202  case OMPRTL__kmpc_alloc: {
2203    // Build to void *__kmpc_alloc(int gtid, size_t sz, const omp_allocator_t
2204    // *al);
2205    // omp_allocator_t type is void *.
2206    llvm::Type *TypeParams[] = {CGM.IntTy, CGM.SizeTy, CGM.VoidPtrPtrTy};
2207    auto *FnTy =
2208        llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2209    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_alloc");
2210    break;
2211  }
2212  case OMPRTL__kmpc_free: {
2213    // Build to void __kmpc_free(int gtid, void *ptr, const omp_allocator_t
2214    // *al);
2215    // omp_allocator_t type is void *.
2216    llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrPtrTy};
2217    auto *FnTy =
2218        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2219    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_free");
2220    break;
2221  }
2222  case OMPRTL__kmpc_push_target_tripcount: {
2223    // Build void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
2224    // size);
2225    llvm::Type *TypeParams[] = {CGM.Int64Ty, CGM.Int64Ty};
2226    llvm::FunctionType *FnTy =
2227        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2228    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_target_tripcount");
2229    break;
2230  }
2231  case OMPRTL__tgt_target: {
2232    // Build int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
2233    // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2234    // *arg_types);
2235    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2236                                CGM.VoidPtrTy,
2237                                CGM.Int32Ty,
2238                                CGM.VoidPtrPtrTy,
2239                                CGM.VoidPtrPtrTy,
2240                                CGM.SizeTy->getPointerTo(),
2241                                CGM.Int64Ty->getPointerTo()};
2242    auto *FnTy =
2243        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2244    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target");
2245    break;
2246  }
2247  case OMPRTL__tgt_target_nowait: {
2248    // Build int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
2249    // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes,
2250    // int64_t *arg_types);
2251    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2252                                CGM.VoidPtrTy,
2253                                CGM.Int32Ty,
2254                                CGM.VoidPtrPtrTy,
2255                                CGM.VoidPtrPtrTy,
2256                                CGM.SizeTy->getPointerTo(),
2257                                CGM.Int64Ty->getPointerTo()};
2258    auto *FnTy =
2259        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2260    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_nowait");
2261    break;
2262  }
2263  case OMPRTL__tgt_target_teams: {
2264    // Build int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
2265    // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes,
2266    // int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2267    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2268                                CGM.VoidPtrTy,
2269                                CGM.Int32Ty,
2270                                CGM.VoidPtrPtrTy,
2271                                CGM.VoidPtrPtrTy,
2272                                CGM.SizeTy->getPointerTo(),
2273                                CGM.Int64Ty->getPointerTo(),
2274                                CGM.Int32Ty,
2275                                CGM.Int32Ty};
2276    auto *FnTy =
2277        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2278    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams");
2279    break;
2280  }
2281  case OMPRTL__tgt_target_teams_nowait: {
2282    // Build int32_t __tgt_target_teams_nowait(int64_t device_id, void
2283    // *host_ptr, int32_t arg_num, void** args_base, void **args, size_t
2284    // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2285    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2286                                CGM.VoidPtrTy,
2287                                CGM.Int32Ty,
2288                                CGM.VoidPtrPtrTy,
2289                                CGM.VoidPtrPtrTy,
2290                                CGM.SizeTy->getPointerTo(),
2291                                CGM.Int64Ty->getPointerTo(),
2292                                CGM.Int32Ty,
2293                                CGM.Int32Ty};
2294    auto *FnTy =
2295        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2296    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams_nowait");
2297    break;
2298  }
2299  case OMPRTL__tgt_register_lib: {
2300    // Build void __tgt_register_lib(__tgt_bin_desc *desc);
2301    QualType ParamTy =
2302        CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2303    llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2304    auto *FnTy =
2305        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2306    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_lib");
2307    break;
2308  }
2309  case OMPRTL__tgt_unregister_lib: {
2310    // Build void __tgt_unregister_lib(__tgt_bin_desc *desc);
2311    QualType ParamTy =
2312        CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2313    llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2314    auto *FnTy =
2315        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2316    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_unregister_lib");
2317    break;
2318  }
2319  case OMPRTL__tgt_target_data_begin: {
2320    // Build void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
2321    // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2322    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2323                                CGM.Int32Ty,
2324                                CGM.VoidPtrPtrTy,
2325                                CGM.VoidPtrPtrTy,
2326                                CGM.SizeTy->getPointerTo(),
2327                                CGM.Int64Ty->getPointerTo()};
2328    auto *FnTy =
2329        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2330    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin");
2331    break;
2332  }
2333  case OMPRTL__tgt_target_data_begin_nowait: {
2334    // Build void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
2335    // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2336    // *arg_types);
2337    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2338                                CGM.Int32Ty,
2339                                CGM.VoidPtrPtrTy,
2340                                CGM.VoidPtrPtrTy,
2341                                CGM.SizeTy->getPointerTo(),
2342                                CGM.Int64Ty->getPointerTo()};
2343    auto *FnTy =
2344        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2345    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin_nowait");
2346    break;
2347  }
2348  case OMPRTL__tgt_target_data_end: {
2349    // Build void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
2350    // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2351    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2352                                CGM.Int32Ty,
2353                                CGM.VoidPtrPtrTy,
2354                                CGM.VoidPtrPtrTy,
2355                                CGM.SizeTy->getPointerTo(),
2356                                CGM.Int64Ty->getPointerTo()};
2357    auto *FnTy =
2358        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2359    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end");
2360    break;
2361  }
2362  case OMPRTL__tgt_target_data_end_nowait: {
2363    // Build void __tgt_target_data_end_nowait(int64_t device_id, int32_t
2364    // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2365    // *arg_types);
2366    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2367                                CGM.Int32Ty,
2368                                CGM.VoidPtrPtrTy,
2369                                CGM.VoidPtrPtrTy,
2370                                CGM.SizeTy->getPointerTo(),
2371                                CGM.Int64Ty->getPointerTo()};
2372    auto *FnTy =
2373        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2374    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end_nowait");
2375    break;
2376  }
2377  case OMPRTL__tgt_target_data_update: {
2378    // Build void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
2379    // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2380    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2381                                CGM.Int32Ty,
2382                                CGM.VoidPtrPtrTy,
2383                                CGM.VoidPtrPtrTy,
2384                                CGM.SizeTy->getPointerTo(),
2385                                CGM.Int64Ty->getPointerTo()};
2386    auto *FnTy =
2387        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2388    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update");
2389    break;
2390  }
2391  case OMPRTL__tgt_target_data_update_nowait: {
2392    // Build void __tgt_target_data_update_nowait(int64_t device_id, int32_t
2393    // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2394    // *arg_types);
2395    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2396                                CGM.Int32Ty,
2397                                CGM.VoidPtrPtrTy,
2398                                CGM.VoidPtrPtrTy,
2399                                CGM.SizeTy->getPointerTo(),
2400                                CGM.Int64Ty->getPointerTo()};
2401    auto *FnTy =
2402        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2403    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update_nowait");
2404    break;
2405  }
2406  }
2407   (0) . __assert_fail ("RTLFn && \"Unable to find OpenMP runtime function\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 2407, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(RTLFn && "Unable to find OpenMP runtime function");
2408  return RTLFn;
2409}
2410
2411llvm::FunctionCallee
2412CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) {
2413   (0) . __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 2414, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert((IVSize == 32 || IVSize == 64) &&
2414 (0) . __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 2414, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">         "IV size is not compatible with the omp runtime");
2415  StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
2416                                            : "__kmpc_for_static_init_4u")
2417                                : (IVSigned ? "__kmpc_for_static_init_8"
2418                                            : "__kmpc_for_static_init_8u");
2419  llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2420  auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2421  llvm::Type *TypeParams[] = {
2422    getIdentTyPointerTy(),                     // loc
2423    CGM.Int32Ty,                               // tid
2424    CGM.Int32Ty,                               // schedtype
2425    llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2426    PtrTy,                                     // p_lower
2427    PtrTy,                                     // p_upper
2428    PtrTy,                                     // p_stride
2429    ITy,                                       // incr
2430    ITy                                        // chunk
2431  };
2432  auto *FnTy =
2433      llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2434  return CGM.CreateRuntimeFunction(FnTy, Name);
2435}
2436
2437llvm::FunctionCallee
2438CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) {
2439   (0) . __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 2440, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert((IVSize == 32 || IVSize == 64) &&
2440 (0) . __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 2440, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">         "IV size is not compatible with the omp runtime");
2441  StringRef Name =
2442      IVSize == 32
2443          ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
2444          : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
2445  llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2446  llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
2447                               CGM.Int32Ty,           // tid
2448                               CGM.Int32Ty,           // schedtype
2449                               ITy,                   // lower
2450                               ITy,                   // upper
2451                               ITy,                   // stride
2452                               ITy                    // chunk
2453  };
2454  auto *FnTy =
2455      llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2456  return CGM.CreateRuntimeFunction(FnTy, Name);
2457}
2458
2459llvm::FunctionCallee
2460CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) {
2461   (0) . __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 2462, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert((IVSize == 32 || IVSize == 64) &&
2462 (0) . __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 2462, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">         "IV size is not compatible with the omp runtime");
2463  StringRef Name =
2464      IVSize == 32
2465          ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
2466          : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
2467  llvm::Type *TypeParams[] = {
2468      getIdentTyPointerTy(), // loc
2469      CGM.Int32Ty,           // tid
2470  };
2471  auto *FnTy =
2472      llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2473  return CGM.CreateRuntimeFunction(FnTy, Name);
2474}
2475
2476llvm::FunctionCallee
2477CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) {
2478   (0) . __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 2479, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert((IVSize == 32 || IVSize == 64) &&
2479 (0) . __assert_fail ("(IVSize == 32 || IVSize == 64) && \"IV size is not compatible with the omp runtime\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 2479, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">         "IV size is not compatible with the omp runtime");
2480  StringRef Name =
2481      IVSize == 32
2482          ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
2483          : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
2484  llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2485  auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2486  llvm::Type *TypeParams[] = {
2487    getIdentTyPointerTy(),                     // loc
2488    CGM.Int32Ty,                               // tid
2489    llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2490    PtrTy,                                     // p_lower
2491    PtrTy,                                     // p_upper
2492    PtrTy                                      // p_stride
2493  };
2494  auto *FnTy =
2495      llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2496  return CGM.CreateRuntimeFunction(FnTy, Name);
2497}
2498
2499Address CGOpenMPRuntime::getAddrOfDeclareTargetLink(const VarDecl *VD) {
2500  if (CGM.getLangOpts().OpenMPSimd)
2501    return Address::invalid();
2502  llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2503      OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2504  if (Res && *Res == OMPDeclareTargetDeclAttr::MT_Link) {
2505    SmallString<64> PtrName;
2506    {
2507      llvm::raw_svector_ostream OS(PtrName);
2508      OS << CGM.getMangledName(GlobalDecl(VD)) << "_decl_tgt_link_ptr";
2509    }
2510    llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
2511    if (!Ptr) {
2512      QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
2513      Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
2514                                        PtrName);
2515      if (!CGM.getLangOpts().OpenMPIsDevice) {
2516        auto *GV = cast<llvm::GlobalVariable>(Ptr);
2517        GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2518        GV->setInitializer(CGM.GetAddrOfGlobal(VD));
2519      }
2520      CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ptr));
2521      registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
2522    }
2523    return Address(Ptr, CGM.getContext().getDeclAlign(VD));
2524  }
2525  return Address::invalid();
2526}
2527
2528llvm::Constant *
2529CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
2530  assert(!CGM.getLangOpts().OpenMPUseTLS ||
2531         !CGM.getContext().getTargetInfo().isTLSSupported());
2532  // Lookup the entry, lazily creating it if necessary.
2533  std::string Suffix = getName({"cache", ""});
2534  return getOrCreateInternalVariable(
2535      CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
2536}
2537
2538Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
2539                                                const VarDecl *VD,
2540                                                Address VDAddr,
2541                                                SourceLocation Loc) {
2542  if (CGM.getLangOpts().OpenMPUseTLS &&
2543      CGM.getContext().getTargetInfo().isTLSSupported())
2544    return VDAddr;
2545
2546  llvm::Type *VarTy = VDAddr.getElementType();
2547  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2548                         CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
2549                                                       CGM.Int8PtrTy),
2550                         CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
2551                         getOrCreateThreadPrivateCache(VD)};
2552  return Address(CGF.EmitRuntimeCall(
2553      createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2554                 VDAddr.getAlignment());
2555}
2556
2557void CGOpenMPRuntime::emitThreadPrivateVarInit(
2558    CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
2559    llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
2560  // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
2561  // library.
2562  llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
2563  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
2564                      OMPLoc);
2565  // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
2566  // to register constructor/destructor for variable.
2567  llvm::Value *Args[] = {
2568      OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
2569      Ctor, CopyCtor, Dtor};
2570  CGF.EmitRuntimeCall(
2571      createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
2572}
2573
2574llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
2575    const VarDecl *VD, Address VDAddr, SourceLocation Loc,
2576    bool PerformInit, CodeGenFunction *CGF) {
2577  if (CGM.getLangOpts().OpenMPUseTLS &&
2578      CGM.getContext().getTargetInfo().isTLSSupported())
2579    return nullptr;
2580
2581  VD = VD->getDefinition(CGM.getContext());
2582  if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
2583    QualType ASTTy = VD->getType();
2584
2585    llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
2586    const Expr *Init = VD->getAnyInitializer();
2587    if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2588      // Generate function that re-emits the declaration's initializer into the
2589      // threadprivate copy of the variable VD
2590      CodeGenFunction CtorCGF(CGM);
2591      FunctionArgList Args;
2592      ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2593                            /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2594                            ImplicitParamDecl::Other);
2595      Args.push_back(&Dst);
2596
2597      const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2598          CGM.getContext().VoidPtrTy, Args);
2599      llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2600      std::string Name = getName({"__kmpc_global_ctor_", ""});
2601      llvm::Function *Fn =
2602          CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2603      CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
2604                            Args, Loc, Loc);
2605      llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
2606          CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2607          CGM.getContext().VoidPtrTy, Dst.getLocation());
2608      Address Arg = Address(ArgVal, VDAddr.getAlignment());
2609      Arg = CtorCGF.Builder.CreateElementBitCast(
2610          Arg, CtorCGF.ConvertTypeForMem(ASTTy));
2611      CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
2612                               /*IsInitializer=*/true);
2613      ArgVal = CtorCGF.EmitLoadOfScalar(
2614          CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2615          CGM.getContext().VoidPtrTy, Dst.getLocation());
2616      CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
2617      CtorCGF.FinishFunction();
2618      Ctor = Fn;
2619    }
2620    if (VD->getType().isDestructedType() != QualType::DK_none) {
2621      // Generate function that emits destructor call for the threadprivate copy
2622      // of the variable VD
2623      CodeGenFunction DtorCGF(CGM);
2624      FunctionArgList Args;
2625      ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2626                            /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2627                            ImplicitParamDecl::Other);
2628      Args.push_back(&Dst);
2629
2630      const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2631          CGM.getContext().VoidTy, Args);
2632      llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2633      std::string Name = getName({"__kmpc_global_dtor_", ""});
2634      llvm::Function *Fn =
2635          CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2636      auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2637      DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
2638                            Loc, Loc);
2639      // Create a scope with an artificial location for the body of this function.
2640      auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2641      llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
2642          DtorCGF.GetAddrOfLocalVar(&Dst),
2643          /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
2644      DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
2645                          DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2646                          DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2647      DtorCGF.FinishFunction();
2648      Dtor = Fn;
2649    }
2650    // Do not emit init function if it is not required.
2651    if (!Ctor && !Dtor)
2652      return nullptr;
2653
2654    llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2655    auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
2656                                               /*isVarArg=*/false)
2657                           ->getPointerTo();
2658    // Copying constructor for the threadprivate variable.
2659    // Must be NULL - reserved by runtime, but currently it requires that this
2660    // parameter is always NULL. Otherwise it fires assertion.
2661    CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2662    if (Ctor == nullptr) {
2663      auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
2664                                             /*isVarArg=*/false)
2665                         ->getPointerTo();
2666      Ctor = llvm::Constant::getNullValue(CtorTy);
2667    }
2668    if (Dtor == nullptr) {
2669      auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
2670                                             /*isVarArg=*/false)
2671                         ->getPointerTo();
2672      Dtor = llvm::Constant::getNullValue(DtorTy);
2673    }
2674    if (!CGF) {
2675      auto *InitFunctionTy =
2676          llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
2677      std::string Name = getName({"__omp_threadprivate_init_", ""});
2678      llvm::Function *InitFunction = CGM.CreateGlobalInitOrDestructFunction(
2679          InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
2680      CodeGenFunction InitCGF(CGM);
2681      FunctionArgList ArgList;
2682      InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
2683                            CGM.getTypes().arrangeNullaryFunction(), ArgList,
2684                            Loc, Loc);
2685      emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2686      InitCGF.FinishFunction();
2687      return InitFunction;
2688    }
2689    emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2690  }
2691  return nullptr;
2692}
2693
2694/// Obtain information that uniquely identifies a target entry. This
2695/// consists of the file and device IDs as well as line number associated with
2696/// the relevant entry source location.
2697static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
2698                                     unsigned &DeviceID, unsigned &FileID,
2699                                     unsigned &LineNum) {
2700  SourceManager &SM = C.getSourceManager();
2701
2702  // The loc should be always valid and have a file ID (the user cannot use
2703  // #pragma directives in macros)
2704
2705   (0) . __assert_fail ("Loc.isValid() && \"Source location is expected to be always valid.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 2705, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Loc.isValid() && "Source location is expected to be always valid.");
2706
2707  PresumedLoc PLoc = SM.getPresumedLoc(Loc);
2708   (0) . __assert_fail ("PLoc.isValid() && \"Source location is expected to be always valid.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 2708, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(PLoc.isValid() && "Source location is expected to be always valid.");
2709
2710  llvm::sys::fs::UniqueID ID;
2711  if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
2712    SM.getDiagnostics().Report(diag::err_cannot_open_file)
2713        << PLoc.getFilename() << EC.message();
2714
2715  DeviceID = ID.getDevice();
2716  FileID = ID.getFile();
2717  LineNum = PLoc.getLine();
2718}
2719
2720bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
2721                                                     llvm::GlobalVariable *Addr,
2722                                                     bool PerformInit) {
2723  Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2724      OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2725  if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link)
2726    return CGM.getLangOpts().OpenMPIsDevice;
2727  VD = VD->getDefinition(CGM.getContext());
2728  if (VD && !DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
2729    return CGM.getLangOpts().OpenMPIsDevice;
2730
2731  QualType ASTTy = VD->getType();
2732
2733  SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
2734  // Produce the unique prefix to identify the new target regions. We use
2735  // the source location of the variable declaration which we know to not
2736  // conflict with any target region.
2737  unsigned DeviceID;
2738  unsigned FileID;
2739  unsigned Line;
2740  getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
2741  SmallString<128> Buffer, Out;
2742  {
2743    llvm::raw_svector_ostream OS(Buffer);
2744    OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
2745       << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
2746  }
2747
2748  const Expr *Init = VD->getAnyInitializer();
2749  if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2750    llvm::Constant *Ctor;
2751    llvm::Constant *ID;
2752    if (CGM.getLangOpts().OpenMPIsDevice) {
2753      // Generate function that re-emits the declaration's initializer into
2754      // the threadprivate copy of the variable VD
2755      CodeGenFunction CtorCGF(CGM);
2756
2757      const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2758      llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2759      llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2760          FTy, Twine(Buffer, "_ctor"), FI, Loc);
2761      auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
2762      CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2763                            FunctionArgList(), Loc, Loc);
2764      auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
2765      CtorCGF.EmitAnyExprToMem(Init,
2766                               Address(Addr, CGM.getContext().getDeclAlign(VD)),
2767                               Init->getType().getQualifiers(),
2768                               /*IsInitializer=*/true);
2769      CtorCGF.FinishFunction();
2770      Ctor = Fn;
2771      ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2772      CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
2773    } else {
2774      Ctor = new llvm::GlobalVariable(
2775          CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2776          llvm::GlobalValue::PrivateLinkage,
2777          llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
2778      ID = Ctor;
2779    }
2780
2781    // Register the information for the entry associated with the constructor.
2782    Out.clear();
2783    OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2784        DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
2785        ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
2786  }
2787  if (VD->getType().isDestructedType() != QualType::DK_none) {
2788    llvm::Constant *Dtor;
2789    llvm::Constant *ID;
2790    if (CGM.getLangOpts().OpenMPIsDevice) {
2791      // Generate function that emits destructor call for the threadprivate
2792      // copy of the variable VD
2793      CodeGenFunction DtorCGF(CGM);
2794
2795      const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2796      llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2797      llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2798          FTy, Twine(Buffer, "_dtor"), FI, Loc);
2799      auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2800      DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2801                            FunctionArgList(), Loc, Loc);
2802      // Create a scope with an artificial location for the body of this
2803      // function.
2804      auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2805      DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
2806                          ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2807                          DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2808      DtorCGF.FinishFunction();
2809      Dtor = Fn;
2810      ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2811      CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
2812    } else {
2813      Dtor = new llvm::GlobalVariable(
2814          CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2815          llvm::GlobalValue::PrivateLinkage,
2816          llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
2817      ID = Dtor;
2818    }
2819    // Register the information for the entry associated with the destructor.
2820    Out.clear();
2821    OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2822        DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
2823        ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
2824  }
2825  return CGM.getLangOpts().OpenMPIsDevice;
2826}
2827
2828Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
2829                                                          QualType VarType,
2830                                                          StringRef Name) {
2831  std::string Suffix = getName({"artificial", ""});
2832  std::string CacheSuffix = getName({"cache", ""});
2833  llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
2834  llvm::Value *GAddr =
2835      getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
2836  llvm::Value *Args[] = {
2837      emitUpdateLocation(CGF, SourceLocation()),
2838      getThreadID(CGF, SourceLocation()),
2839      CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
2840      CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
2841                                /*IsSigned=*/false),
2842      getOrCreateInternalVariable(
2843          CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
2844  return Address(
2845      CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2846          CGF.EmitRuntimeCall(
2847              createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2848          VarLVType->getPointerTo(/*AddrSpace=*/0)),
2849      CGM.getPointerAlign());
2850}
2851
2852void CGOpenMPRuntime::emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond,
2853                                      const RegionCodeGenTy &ThenGen,
2854                                      const RegionCodeGenTy &ElseGen) {
2855  CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
2856
2857  // If the condition constant folds and can be elided, try to avoid emitting
2858  // the condition and the dead arm of the if/else.
2859  bool CondConstant;
2860  if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
2861    if (CondConstant)
2862      ThenGen(CGF);
2863    else
2864      ElseGen(CGF);
2865    return;
2866  }
2867
2868  // Otherwise, the condition did not fold, or we couldn't elide it.  Just
2869  // emit the conditional branch.
2870  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
2871  llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
2872  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
2873  CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
2874
2875  // Emit the 'then' code.
2876  CGF.EmitBlock(ThenBlock);
2877  ThenGen(CGF);
2878  CGF.EmitBranch(ContBlock);
2879  // Emit the 'else' code if present.
2880  // There is no need to emit line number for unconditional branch.
2881  (void)ApplyDebugLocation::CreateEmpty(CGF);
2882  CGF.EmitBlock(ElseBlock);
2883  ElseGen(CGF);
2884  // There is no need to emit line number for unconditional branch.
2885  (void)ApplyDebugLocation::CreateEmpty(CGF);
2886  CGF.EmitBranch(ContBlock);
2887  // Emit the continuation block for code after the if.
2888  CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
2889}
2890
2891void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
2892                                       llvm::Function *OutlinedFn,
2893                                       ArrayRef<llvm::Value *> CapturedVars,
2894                                       const Expr *IfCond) {
2895  if (!CGF.HaveInsertPoint())
2896    return;
2897  llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
2898  auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
2899                                                     PrePostActionTy &) {
2900    // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
2901    CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2902    llvm::Value *Args[] = {
2903        RTLoc,
2904        CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
2905        CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
2906    llvm::SmallVector<llvm::Value *, 16> RealArgs;
2907    RealArgs.append(std::begin(Args), std::end(Args));
2908    RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2909
2910    llvm::FunctionCallee RTLFn =
2911        RT.createRuntimeFunction(OMPRTL__kmpc_fork_call);
2912    CGF.EmitRuntimeCall(RTLFn, RealArgs);
2913  };
2914  auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF,
2915                                                          PrePostActionTy &) {
2916    CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2917    llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
2918    // Build calls:
2919    // __kmpc_serialized_parallel(&Loc, GTid);
2920    llvm::Value *Args[] = {RTLoc, ThreadID};
2921    CGF.EmitRuntimeCall(
2922        RT.createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), Args);
2923
2924    // OutlinedFn(&GTid, &zero, CapturedStruct);
2925    Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
2926                                                        /*Name*/ ".zero.addr");
2927    CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
2928    llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2929    // ThreadId for serialized parallels is 0.
2930    OutlinedFnArgs.push_back(ZeroAddr.getPointer());
2931    OutlinedFnArgs.push_back(ZeroAddr.getPointer());
2932    OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2933    RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
2934
2935    // __kmpc_end_serialized_parallel(&Loc, GTid);
2936    llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2937    CGF.EmitRuntimeCall(
2938        RT.createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel),
2939        EndArgs);
2940  };
2941  if (IfCond) {
2942    emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
2943  } else {
2944    RegionCodeGenTy ThenRCG(ThenGen);
2945    ThenRCG(CGF);
2946  }
2947}
2948
2949// If we're inside an (outlined) parallel region, use the region info's
2950// thread-ID variable (it is passed in a first argument of the outlined function
2951// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2952// regular serial code region, get thread ID by calling kmp_int32
2953// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2954// return the address of that temp.
2955Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
2956                                             SourceLocation Loc) {
2957  if (auto *OMPRegionInfo =
2958          dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2959    if (OMPRegionInfo->getThreadIDVariable())
2960      return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
2961
2962  llvm::Value *ThreadID = getThreadID(CGF, Loc);
2963  QualType Int32Ty =
2964      CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2965  Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
2966  CGF.EmitStoreOfScalar(ThreadID,
2967                        CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
2968
2969  return ThreadIDTemp;
2970}
2971
2972llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
2973    llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
2974  SmallString<256> Buffer;
2975  llvm::raw_svector_ostream Out(Buffer);
2976  Out << Name;
2977  StringRef RuntimeName = Out.str();
2978  auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
2979  if (Elem.second) {
2980     (0) . __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 2981, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Elem.second->getType()->getPointerElementType() == Ty &&
2981 (0) . __assert_fail ("Elem.second->getType()->getPointerElementType() == Ty && \"OMP internal variable has different type than requested\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 2981, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">           "OMP internal variable has different type than requested");
2982    return &*Elem.second;
2983  }
2984
2985  return Elem.second = new llvm::GlobalVariable(
2986             CGM.getModule(), Ty, /*IsConstant*/ false,
2987             llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
2988             Elem.first(), /*InsertBefore=*/nullptr,
2989             llvm::GlobalValue::NotThreadLocal, AddressSpace);
2990}
2991
2992llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
2993  std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
2994  std::string Name = getName({Prefix, "var"});
2995  return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
2996}
2997
2998namespace {
2999/// Common pre(post)-action for different OpenMP constructs.
3000class CommonActionTy final : public PrePostActionTy {
3001  llvm::FunctionCallee EnterCallee;
3002  ArrayRef<llvm::Value *> EnterArgs;
3003  llvm::FunctionCallee ExitCallee;
3004  ArrayRef<llvm::Value *> ExitArgs;
3005  bool Conditional;
3006  llvm::BasicBlock *ContBlock = nullptr;
3007
3008public:
3009  CommonActionTy(llvm::FunctionCallee EnterCallee,
3010                 ArrayRef<llvm::Value *> EnterArgs,
3011                 llvm::FunctionCallee ExitCallee,
3012                 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
3013      : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
3014        ExitArgs(ExitArgs), Conditional(Conditional) {}
3015  void Enter(CodeGenFunction &CGF) override {
3016    llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
3017    if (Conditional) {
3018      llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
3019      auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
3020      ContBlock = CGF.createBasicBlock("omp_if.end");
3021      // Generate the branch (If-stmt)
3022      CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
3023      CGF.EmitBlock(ThenBlock);
3024    }
3025  }
3026  void Done(CodeGenFunction &CGF) {
3027    // Emit the rest of blocks/branches
3028    CGF.EmitBranch(ContBlock);
3029    CGF.EmitBlock(ContBlock, true);
3030  }
3031  void Exit(CodeGenFunction &CGF) override {
3032    CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
3033  }
3034};
3035} // anonymous namespace
3036
3037void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
3038                                         StringRef CriticalName,
3039                                         const RegionCodeGenTy &CriticalOpGen,
3040                                         SourceLocation Loc, const Expr *Hint) {
3041  // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
3042  // CriticalOpGen();
3043  // __kmpc_end_critical(ident_t *, gtid, Lock);
3044  // Prepare arguments and build a call to __kmpc_critical
3045  if (!CGF.HaveInsertPoint())
3046    return;
3047  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3048                         getCriticalRegionLock(CriticalName)};
3049  llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
3050                                                std::end(Args));
3051  if (Hint) {
3052    EnterArgs.push_back(CGF.Builder.CreateIntCast(
3053        CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
3054  }
3055  CommonActionTy Action(
3056      createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint
3057                                 : OMPRTL__kmpc_critical),
3058      EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
3059  CriticalOpGen.setAction(Action);
3060  emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
3061}
3062
3063void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
3064                                       const RegionCodeGenTy &MasterOpGen,
3065                                       SourceLocation Loc) {
3066  if (!CGF.HaveInsertPoint())
3067    return;
3068  // if(__kmpc_master(ident_t *, gtid)) {
3069  //   MasterOpGen();
3070  //   __kmpc_end_master(ident_t *, gtid);
3071  // }
3072  // Prepare arguments and build a call to __kmpc_master
3073  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3074  CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args,
3075                        createRuntimeFunction(OMPRTL__kmpc_end_master), Args,
3076                        /*Conditional=*/true);
3077  MasterOpGen.setAction(Action);
3078  emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
3079  Action.Done(CGF);
3080}
3081
3082void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
3083                                        SourceLocation Loc) {
3084  if (!CGF.HaveInsertPoint())
3085    return;
3086  // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
3087  llvm::Value *Args[] = {
3088      emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3089      llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
3090  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args);
3091  if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3092    Region->emitUntiedSwitch(CGF);
3093}
3094
3095void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
3096                                          const RegionCodeGenTy &TaskgroupOpGen,
3097                                          SourceLocation Loc) {
3098  if (!CGF.HaveInsertPoint())
3099    return;
3100  // __kmpc_taskgroup(ident_t *, gtid);
3101  // TaskgroupOpGen();
3102  // __kmpc_end_taskgroup(ident_t *, gtid);
3103  // Prepare arguments and build a call to __kmpc_taskgroup
3104  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3105  CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args,
3106                        createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
3107                        Args);
3108  TaskgroupOpGen.setAction(Action);
3109  emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
3110}
3111
3112/// Given an array of pointers to variables, project the address of a
3113/// given variable.
3114static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
3115                                      unsigned Index, const VarDecl *Var) {
3116  // Pull out the pointer to the variable.
3117  Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
3118  llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
3119
3120  Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
3121  Addr = CGF.Builder.CreateElementBitCast(
3122      Addr, CGF.ConvertTypeForMem(Var->getType()));
3123  return Addr;
3124}
3125
3126static llvm::Value *emitCopyprivateCopyFunction(
3127    CodeGenModule &CGM, llvm::Type *ArgsType,
3128    ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
3129    ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
3130    SourceLocation Loc) {
3131  ASTContext &C = CGM.getContext();
3132  // void copy_func(void *LHSArg, void *RHSArg);
3133  FunctionArgList Args;
3134  ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3135                           ImplicitParamDecl::Other);
3136  ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3137                           ImplicitParamDecl::Other);
3138  Args.push_back(&LHSArg);
3139  Args.push_back(&RHSArg);
3140  const auto &CGFI =
3141      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3142  std::string Name =
3143      CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
3144  auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
3145                                    llvm::GlobalValue::InternalLinkage, Name,
3146                                    &CGM.getModule());
3147  CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
3148  Fn->setDoesNotRecurse();
3149  CodeGenFunction CGF(CGM);
3150  CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3151  // Dest = (void*[n])(LHSArg);
3152  // Src = (void*[n])(RHSArg);
3153  Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3154      CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
3155      ArgsType), CGF.getPointerAlign());
3156  Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3157      CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
3158      ArgsType), CGF.getPointerAlign());
3159  // *(Type0*)Dst[0] = *(Type0*)Src[0];
3160  // *(Type1*)Dst[1] = *(Type1*)Src[1];
3161  // ...
3162  // *(Typen*)Dst[n] = *(Typen*)Src[n];
3163  for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
3164    const auto *DestVar =
3165        cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
3166    Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
3167
3168    const auto *SrcVar =
3169        cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
3170    Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
3171
3172    const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
3173    QualType Type = VD->getType();
3174    CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
3175  }
3176  CGF.FinishFunction();
3177  return Fn;
3178}
3179
3180void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
3181                                       const RegionCodeGenTy &SingleOpGen,
3182                                       SourceLocation Loc,
3183                                       ArrayRef<const Expr *> CopyprivateVars,
3184                                       ArrayRef<const Expr *> SrcExprs,
3185                                       ArrayRef<const Expr *> DstExprs,
3186                                       ArrayRef<const Expr *> AssignmentOps) {
3187  if (!CGF.HaveInsertPoint())
3188    return;
3189  assert(CopyprivateVars.size() == SrcExprs.size() &&
3190         CopyprivateVars.size() == DstExprs.size() &&
3191         CopyprivateVars.size() == AssignmentOps.size());
3192  ASTContext &C = CGM.getContext();
3193  // int32 did_it = 0;
3194  // if(__kmpc_single(ident_t *, gtid)) {
3195  //   SingleOpGen();
3196  //   __kmpc_end_single(ident_t *, gtid);
3197  //   did_it = 1;
3198  // }
3199  // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3200  // <copy_func>, did_it);
3201
3202  Address DidIt = Address::invalid();
3203  if (!CopyprivateVars.empty()) {
3204    // int32 did_it = 0;
3205    QualType KmpInt32Ty =
3206        C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3207    DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
3208    CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
3209  }
3210  // Prepare arguments and build a call to __kmpc_single
3211  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3212  CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args,
3213                        createRuntimeFunction(OMPRTL__kmpc_end_single), Args,
3214                        /*Conditional=*/true);
3215  SingleOpGen.setAction(Action);
3216  emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
3217  if (DidIt.isValid()) {
3218    // did_it = 1;
3219    CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
3220  }
3221  Action.Done(CGF);
3222  // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3223  // <copy_func>, did_it);
3224  if (DidIt.isValid()) {
3225    llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
3226    QualType CopyprivateArrayTy =
3227        C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
3228                               /*IndexTypeQuals=*/0);
3229    // Create a list of all private variables for copyprivate.
3230    Address CopyprivateList =
3231        CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
3232    for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
3233      Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
3234      CGF.Builder.CreateStore(
3235          CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3236              CGF.EmitLValue(CopyprivateVars[I]).getPointer(), CGF.VoidPtrTy),
3237          Elem);
3238    }
3239    // Build function that copies private values from single region to all other
3240    // threads in the corresponding parallel region.
3241    llvm::Value *CpyFn = emitCopyprivateCopyFunction(
3242        CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
3243        CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
3244    llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
3245    Address CL =
3246      CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
3247                                                      CGF.VoidPtrTy);
3248    llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
3249    llvm::Value *Args[] = {
3250        emitUpdateLocation(CGF, Loc), // ident_t *<loc>
3251        getThreadID(CGF, Loc),        // i32 <gtid>
3252        BufSize,                      // size_t <buf_size>
3253        CL.getPointer(),              // void *<copyprivate list>
3254        CpyFn,                        // void (*) (void *, void *) <copy_func>
3255        DidItVal                      // i32 did_it
3256    };
3257    CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
3258  }
3259}
3260
3261void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
3262                                        const RegionCodeGenTy &OrderedOpGen,
3263                                        SourceLocation Loc, bool IsThreads) {
3264  if (!CGF.HaveInsertPoint())
3265    return;
3266  // __kmpc_ordered(ident_t *, gtid);
3267  // OrderedOpGen();
3268  // __kmpc_end_ordered(ident_t *, gtid);
3269  // Prepare arguments and build a call to __kmpc_ordered
3270  if (IsThreads) {
3271    llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3272    CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args,
3273                          createRuntimeFunction(OMPRTL__kmpc_end_ordered),
3274                          Args);
3275    OrderedOpGen.setAction(Action);
3276    emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3277    return;
3278  }
3279  emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3280}
3281
3282unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
3283  unsigned Flags;
3284  if (Kind == OMPD_for)
3285    Flags = OMP_IDENT_BARRIER_IMPL_FOR;
3286  else if (Kind == OMPD_sections)
3287    Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
3288  else if (Kind == OMPD_single)
3289    Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
3290  else if (Kind == OMPD_barrier)
3291    Flags = OMP_IDENT_BARRIER_EXPL;
3292  else
3293    Flags = OMP_IDENT_BARRIER_IMPL;
3294  return Flags;
3295}
3296
3297void CGOpenMPRuntime::getDefaultScheduleAndChunk(
3298    CodeGenFunction &CGF, const OMPLoopDirective &S,
3299    OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
3300  // Check if the loop directive is actually a doacross loop directive. In this
3301  // case choose static, 1 schedule.
3302  if (llvm::any_of(
3303          S.getClausesOfKind<OMPOrderedClause>(),
3304          [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
3305    ScheduleKind = OMPC_SCHEDULE_static;
3306    // Chunk size is 1 in this case.
3307    llvm::APInt ChunkSize(32, 1);
3308    ChunkExpr = IntegerLiteral::Create(
3309        CGF.getContext(), ChunkSize,
3310        CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3311        SourceLocation());
3312  }
3313}
3314
3315void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
3316                                      OpenMPDirectiveKind Kind, bool EmitChecks,
3317                                      bool ForceSimpleCall) {
3318  if (!CGF.HaveInsertPoint())
3319    return;
3320  // Build call __kmpc_cancel_barrier(loc, thread_id);
3321  // Build call __kmpc_barrier(loc, thread_id);
3322  unsigned Flags = getDefaultFlagsForBarriers(Kind);
3323  // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
3324  // thread_id);
3325  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
3326                         getThreadID(CGF, Loc)};
3327  if (auto *OMPRegionInfo =
3328          dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
3329    if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
3330      llvm::Value *Result = CGF.EmitRuntimeCall(
3331          createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
3332      if (EmitChecks) {
3333        // if (__kmpc_cancel_barrier()) {
3334        //   exit from construct;
3335        // }
3336        llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
3337        llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
3338        llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
3339        CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3340        CGF.EmitBlock(ExitBB);
3341        //   exit from construct;
3342        CodeGenFunction::JumpDest CancelDestination =
3343            CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
3344        CGF.EmitBranchThroughCleanup(CancelDestination);
3345        CGF.EmitBlock(ContBB, /*IsFinished=*/true);
3346      }
3347      return;
3348    }
3349  }
3350  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
3351}
3352
3353/// Map the OpenMP loop schedule to the runtime enumeration.
3354static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
3355                                          bool Chunked, bool Ordered) {
3356  switch (ScheduleKind) {
3357  case OMPC_SCHEDULE_static:
3358    return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
3359                   : (Ordered ? OMP_ord_static : OMP_sch_static);
3360  case OMPC_SCHEDULE_dynamic:
3361    return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
3362  case OMPC_SCHEDULE_guided:
3363    return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
3364  case OMPC_SCHEDULE_runtime:
3365    return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
3366  case OMPC_SCHEDULE_auto:
3367    return Ordered ? OMP_ord_auto : OMP_sch_auto;
3368  case OMPC_SCHEDULE_unknown:
3369     (0) . __assert_fail ("!Chunked && \"chunk was specified but schedule kind not known\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3369, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!Chunked && "chunk was specified but schedule kind not known");
3370    return Ordered ? OMP_ord_static : OMP_sch_static;
3371  }
3372  llvm_unreachable("Unexpected runtime schedule");
3373}
3374
3375/// Map the OpenMP distribute schedule to the runtime enumeration.
3376static OpenMPSchedType
3377getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
3378  // only static is allowed for dist_schedule
3379  return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
3380}
3381
3382bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
3383                                         bool Chunked) const {
3384  OpenMPSchedType Schedule =
3385      getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3386  return Schedule == OMP_sch_static;
3387}
3388
3389bool CGOpenMPRuntime::isStaticNonchunked(
3390    OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3391  OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3392  return Schedule == OMP_dist_sch_static;
3393}
3394
3395bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
3396                                      bool Chunked) const {
3397  OpenMPSchedType Schedule =
3398      getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3399  return Schedule == OMP_sch_static_chunked;
3400}
3401
3402bool CGOpenMPRuntime::isStaticChunked(
3403    OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3404  OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3405  return Schedule == OMP_dist_sch_static_chunked;
3406}
3407
3408bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
3409  OpenMPSchedType Schedule =
3410      getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
3411   (0) . __assert_fail ("Schedule != OMP_sch_static_chunked && \"cannot be chunked here\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3411, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here");
3412  return Schedule != OMP_sch_static;
3413}
3414
3415static int addMonoNonMonoModifier(OpenMPSchedType Schedule,
3416                                  OpenMPScheduleClauseModifier M1,
3417                                  OpenMPScheduleClauseModifier M2) {
3418  int Modifier = 0;
3419  switch (M1) {
3420  case OMPC_SCHEDULE_MODIFIER_monotonic:
3421    Modifier = OMP_sch_modifier_monotonic;
3422    break;
3423  case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3424    Modifier = OMP_sch_modifier_nonmonotonic;
3425    break;
3426  case OMPC_SCHEDULE_MODIFIER_simd:
3427    if (Schedule == OMP_sch_static_chunked)
3428      Schedule = OMP_sch_static_balanced_chunked;
3429    break;
3430  case OMPC_SCHEDULE_MODIFIER_last:
3431  case OMPC_SCHEDULE_MODIFIER_unknown:
3432    break;
3433  }
3434  switch (M2) {
3435  case OMPC_SCHEDULE_MODIFIER_monotonic:
3436    Modifier = OMP_sch_modifier_monotonic;
3437    break;
3438  case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3439    Modifier = OMP_sch_modifier_nonmonotonic;
3440    break;
3441  case OMPC_SCHEDULE_MODIFIER_simd:
3442    if (Schedule == OMP_sch_static_chunked)
3443      Schedule = OMP_sch_static_balanced_chunked;
3444    break;
3445  case OMPC_SCHEDULE_MODIFIER_last:
3446  case OMPC_SCHEDULE_MODIFIER_unknown:
3447    break;
3448  }
3449  return Schedule | Modifier;
3450}
3451
3452void CGOpenMPRuntime::emitForDispatchInit(
3453    CodeGenFunction &CGF, SourceLocation Loc,
3454    const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
3455    bool Ordered, const DispatchRTInput &DispatchValues) {
3456  if (!CGF.HaveInsertPoint())
3457    return;
3458  OpenMPSchedType Schedule = getRuntimeSchedule(
3459      ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
3460  assert(Ordered ||
3461         (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
3462          Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
3463          Schedule != OMP_sch_static_balanced_chunked));
3464  // Call __kmpc_dispatch_init(
3465  //          ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
3466  //          kmp_int[32|64] lower, kmp_int[32|64] upper,
3467  //          kmp_int[32|64] stride, kmp_int[32|64] chunk);
3468
3469  // If the Chunk was not specified in the clause - use default value 1.
3470  llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
3471                                            : CGF.Builder.getIntN(IVSize, 1);
3472  llvm::Value *Args[] = {
3473      emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3474      CGF.Builder.getInt32(addMonoNonMonoModifier(
3475          Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
3476      DispatchValues.LB,                                // Lower
3477      DispatchValues.UB,                                // Upper
3478      CGF.Builder.getIntN(IVSize, 1),                   // Stride
3479      Chunk                                             // Chunk
3480  };
3481  CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
3482}
3483
3484static void emitForStaticInitCall(
3485    CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
3486    llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
3487    OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
3488    const CGOpenMPRuntime::StaticRTInput &Values) {
3489  if (!CGF.HaveInsertPoint())
3490    return;
3491
3492  assert(!Values.Ordered);
3493  assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
3494         Schedule == OMP_sch_static_balanced_chunked ||
3495         Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
3496         Schedule == OMP_dist_sch_static ||
3497         Schedule == OMP_dist_sch_static_chunked);
3498
3499  // Call __kmpc_for_static_init(
3500  //          ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
3501  //          kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
3502  //          kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
3503  //          kmp_int[32|64] incr, kmp_int[32|64] chunk);
3504  llvm::Value *Chunk = Values.Chunk;
3505  if (Chunk == nullptr) {
3506     (0) . __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3508, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
3507 (0) . __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3508, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">            Schedule == OMP_dist_sch_static) &&
3508 (0) . __assert_fail ("(Schedule == OMP_sch_static || Schedule == OMP_ord_static || Schedule == OMP_dist_sch_static) && \"expected static non-chunked schedule\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3508, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">           "expected static non-chunked schedule");
3509    // If the Chunk was not specified in the clause - use default value 1.
3510    Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
3511  } else {
3512     (0) . __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3516, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert((Schedule == OMP_sch_static_chunked ||
3513 (0) . __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3516, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">            Schedule == OMP_sch_static_balanced_chunked ||
3514 (0) . __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3516, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">            Schedule == OMP_ord_static_chunked ||
3515 (0) . __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3516, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">            Schedule == OMP_dist_sch_static_chunked) &&
3516 (0) . __assert_fail ("(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static_balanced_chunked || Schedule == OMP_ord_static_chunked || Schedule == OMP_dist_sch_static_chunked) && \"expected static chunked schedule\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3516, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">           "expected static chunked schedule");
3517  }
3518  llvm::Value *Args[] = {
3519      UpdateLocation,
3520      ThreadId,
3521      CGF.Builder.getInt32(addMonoNonMonoModifier(Schedule, M1,
3522                                                  M2)), // Schedule type
3523      Values.IL.getPointer(),                           // &isLastIter
3524      Values.LB.getPointer(),                           // &LB
3525      Values.UB.getPointer(),                           // &UB
3526      Values.ST.getPointer(),                           // &Stride
3527      CGF.Builder.getIntN(Values.IVSize, 1),            // Incr
3528      Chunk                                             // Chunk
3529  };
3530  CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
3531}
3532
3533void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
3534                                        SourceLocation Loc,
3535                                        OpenMPDirectiveKind DKind,
3536                                        const OpenMPScheduleTy &ScheduleKind,
3537                                        const StaticRTInput &Values) {
3538  OpenMPSchedType ScheduleNum = getRuntimeSchedule(
3539      ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
3540   (0) . __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3541, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(isOpenMPWorksharingDirective(DKind) &&
3541 (0) . __assert_fail ("isOpenMPWorksharingDirective(DKind) && \"Expected loop-based or sections-based directive.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3541, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">         "Expected loop-based or sections-based directive.");
3542  llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
3543                                             isOpenMPLoopDirective(DKind)
3544                                                 ? OMP_IDENT_WORK_LOOP
3545                                                 : OMP_IDENT_WORK_SECTIONS);
3546  llvm::Value *ThreadId = getThreadID(CGF, Loc);
3547  llvm::FunctionCallee StaticInitFunction =
3548      createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3549  emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3550                        ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
3551}
3552
3553void CGOpenMPRuntime::emitDistributeStaticInit(
3554    CodeGenFunction &CGF, SourceLocation Loc,
3555    OpenMPDistScheduleClauseKind SchedKind,
3556    const CGOpenMPRuntime::StaticRTInput &Values) {
3557  OpenMPSchedType ScheduleNum =
3558      getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
3559  llvm::Value *UpdatedLocation =
3560      emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
3561  llvm::Value *ThreadId = getThreadID(CGF, Loc);
3562  llvm::FunctionCallee StaticInitFunction =
3563      createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3564  emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3565                        ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
3566                        OMPC_SCHEDULE_MODIFIER_unknown, Values);
3567}
3568
3569void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
3570                                          SourceLocation Loc,
3571                                          OpenMPDirectiveKind DKind) {
3572  if (!CGF.HaveInsertPoint())
3573    return;
3574  // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
3575  llvm::Value *Args[] = {
3576      emitUpdateLocation(CGF, Loc,
3577                         isOpenMPDistributeDirective(DKind)
3578                             ? OMP_IDENT_WORK_DISTRIBUTE
3579                             : isOpenMPLoopDirective(DKind)
3580                                   ? OMP_IDENT_WORK_LOOP
3581                                   : OMP_IDENT_WORK_SECTIONS),
3582      getThreadID(CGF, Loc)};
3583  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
3584                      Args);
3585}
3586
3587void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
3588                                                 SourceLocation Loc,
3589                                                 unsigned IVSize,
3590                                                 bool IVSigned) {
3591  if (!CGF.HaveInsertPoint())
3592    return;
3593  // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
3594  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3595  CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
3596}
3597
3598llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
3599                                          SourceLocation Loc, unsigned IVSize,
3600                                          bool IVSigned, Address IL,
3601                                          Address LB, Address UB,
3602                                          Address ST) {
3603  // Call __kmpc_dispatch_next(
3604  //          ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
3605  //          kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
3606  //          kmp_int[32|64] *p_stride);
3607  llvm::Value *Args[] = {
3608      emitUpdateLocation(CGF, Loc),
3609      getThreadID(CGF, Loc),
3610      IL.getPointer(), // &isLastIter
3611      LB.getPointer(), // &Lower
3612      UB.getPointer(), // &Upper
3613      ST.getPointer()  // &Stride
3614  };
3615  llvm::Value *Call =
3616      CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
3617  return CGF.EmitScalarConversion(
3618      Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
3619      CGF.getContext().BoolTy, Loc);
3620}
3621
3622void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
3623                                           llvm::Value *NumThreads,
3624                                           SourceLocation Loc) {
3625  if (!CGF.HaveInsertPoint())
3626    return;
3627  // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
3628  llvm::Value *Args[] = {
3629      emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3630      CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
3631  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
3632                      Args);
3633}
3634
3635void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
3636                                         OpenMPProcBindClauseKind ProcBind,
3637                                         SourceLocation Loc) {
3638  if (!CGF.HaveInsertPoint())
3639    return;
3640  // Constants for proc bind value accepted by the runtime.
3641  enum ProcBindTy {
3642    ProcBindFalse = 0,
3643    ProcBindTrue,
3644    ProcBindMaster,
3645    ProcBindClose,
3646    ProcBindSpread,
3647    ProcBindIntel,
3648    ProcBindDefault
3649  } RuntimeProcBind;
3650  switch (ProcBind) {
3651  case OMPC_PROC_BIND_master:
3652    RuntimeProcBind = ProcBindMaster;
3653    break;
3654  case OMPC_PROC_BIND_close:
3655    RuntimeProcBind = ProcBindClose;
3656    break;
3657  case OMPC_PROC_BIND_spread:
3658    RuntimeProcBind = ProcBindSpread;
3659    break;
3660  case OMPC_PROC_BIND_unknown:
3661    llvm_unreachable("Unsupported proc_bind value.");
3662  }
3663  // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
3664  llvm::Value *Args[] = {
3665      emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3666      llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind, /*isSigned=*/true)};
3667  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
3668}
3669
3670void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
3671                                SourceLocation Loc) {
3672  if (!CGF.HaveInsertPoint())
3673    return;
3674  // Build call void __kmpc_flush(ident_t *loc)
3675  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
3676                      emitUpdateLocation(CGF, Loc));
3677}
3678
3679namespace {
3680/// Indexes of fields for type kmp_task_t.
3681enum KmpTaskTFields {
3682  /// List of shared variables.
3683  KmpTaskTShareds,
3684  /// Task routine.
3685  KmpTaskTRoutine,
3686  /// Partition id for the untied tasks.
3687  KmpTaskTPartId,
3688  /// Function with call of destructors for private variables.
3689  Data1,
3690  /// Task priority.
3691  Data2,
3692  /// (Taskloops only) Lower bound.
3693  KmpTaskTLowerBound,
3694  /// (Taskloops only) Upper bound.
3695  KmpTaskTUpperBound,
3696  /// (Taskloops only) Stride.
3697  KmpTaskTStride,
3698  /// (Taskloops only) Is last iteration flag.
3699  KmpTaskTLastIter,
3700  /// (Taskloops only) Reduction data.
3701  KmpTaskTReductions,
3702};
3703} // anonymous namespace
3704
3705bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
3706  return OffloadEntriesTargetRegion.empty() &&
3707         OffloadEntriesDeviceGlobalVar.empty();
3708}
3709
3710/// Initialize target region entry.
3711void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3712    initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3713                                    StringRef ParentName, unsigned LineNum,
3714                                    unsigned Order) {
3715   (0) . __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3717, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
3716 (0) . __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3717, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                                             "only required for the device "
3717 (0) . __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3717, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                                             "code generation.");
3718  OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3719      OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
3720                                   OMPTargetRegionEntryTargetRegion);
3721  ++OffloadingEntriesNum;
3722}
3723
3724void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3725    registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3726                                  StringRef ParentName, unsigned LineNum,
3727                                  llvm::Constant *Addr, llvm::Constant *ID,
3728                                  OMPTargetRegionEntryKind Flags) {
3729  // If we are emitting code for a target, the entry is already initialized,
3730  // only has to be registered.
3731  if (CGM.getLangOpts().OpenMPIsDevice) {
3732    if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
3733      unsigned DiagID = CGM.getDiags().getCustomDiagID(
3734          DiagnosticsEngine::Error,
3735          "Unable to find target region on line '%0' in the device code.");
3736      CGM.getDiags().Report(DiagID) << LineNum;
3737      return;
3738    }
3739    auto &Entry =
3740        OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3741     (0) . __assert_fail ("Entry.isValid() && \"Entry not initialized!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3741, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Entry.isValid() && "Entry not initialized!");
3742    Entry.setAddress(Addr);
3743    Entry.setID(ID);
3744    Entry.setFlags(Flags);
3745  } else {
3746    OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
3747    OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3748    ++OffloadingEntriesNum;
3749  }
3750}
3751
3752bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3753    unsigned DeviceID, unsigned FileID, StringRef ParentName,
3754    unsigned LineNum) const {
3755  auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3756  if (PerDevice == OffloadEntriesTargetRegion.end())
3757    return false;
3758  auto PerFile = PerDevice->second.find(FileID);
3759  if (PerFile == PerDevice->second.end())
3760    return false;
3761  auto PerParentName = PerFile->second.find(ParentName);
3762  if (PerParentName == PerFile->second.end())
3763    return false;
3764  auto PerLine = PerParentName->second.find(LineNum);
3765  if (PerLine == PerParentName->second.end())
3766    return false;
3767  // Fail if this entry is already registered.
3768  if (PerLine->second.getAddress() || PerLine->second.getID())
3769    return false;
3770  return true;
3771}
3772
3773void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3774    const OffloadTargetRegionEntryInfoActTy &Action) {
3775  // Scan all target region entries and perform the provided action.
3776  for (const auto &D : OffloadEntriesTargetRegion)
3777    for (const auto &F : D.second)
3778      for (const auto &P : F.second)
3779        for (const auto &L : P.second)
3780          Action(D.first, F.first, P.first(), L.first, L.second);
3781}
3782
3783void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3784    initializeDeviceGlobalVarEntryInfo(StringRef Name,
3785                                       OMPTargetGlobalVarEntryKind Flags,
3786                                       unsigned Order) {
3787   (0) . __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3789, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
3788 (0) . __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3789, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                                             "only required for the device "
3789 (0) . __assert_fail ("CGM.getLangOpts().OpenMPIsDevice && \"Initialization of entries is \" \"only required for the device \" \"code generation.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3789, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                                             "code generation.");
3790  OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3791  ++OffloadingEntriesNum;
3792}
3793
3794void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3795    registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3796                                     CharUnits VarSize,
3797                                     OMPTargetGlobalVarEntryKind Flags,
3798                                     llvm::GlobalValue::LinkageTypes Linkage) {
3799  if (CGM.getLangOpts().OpenMPIsDevice) {
3800    auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3801     (0) . __assert_fail ("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3802, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Entry.isValid() && Entry.getFlags() == Flags &&
3802 (0) . __assert_fail ("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3802, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">           "Entry not initialized!");
3803     (0) . __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3804, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&
3804 (0) . __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3804, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">           "Resetting with the new address.");
3805    if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
3806      if (Entry.getVarSize().isZero()) {
3807        Entry.setVarSize(VarSize);
3808        Entry.setLinkage(Linkage);
3809      }
3810      return;
3811    }
3812    Entry.setVarSize(VarSize);
3813    Entry.setLinkage(Linkage);
3814    Entry.setAddress(Addr);
3815  } else {
3816    if (hasDeviceGlobalVarEntryInfo(VarName)) {
3817      auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3818       (0) . __assert_fail ("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3819, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Entry.isValid() && Entry.getFlags() == Flags &&
3819 (0) . __assert_fail ("Entry.isValid() && Entry.getFlags() == Flags && \"Entry not initialized!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3819, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">             "Entry not initialized!");
3820       (0) . __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3821, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&
3821 (0) . __assert_fail ("(!Entry.getAddress() || Entry.getAddress() == Addr) && \"Resetting with the new address.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3821, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">             "Resetting with the new address.");
3822      if (Entry.getVarSize().isZero()) {
3823        Entry.setVarSize(VarSize);
3824        Entry.setLinkage(Linkage);
3825      }
3826      return;
3827    }
3828    OffloadEntriesDeviceGlobalVar.try_emplace(
3829        VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
3830    ++OffloadingEntriesNum;
3831  }
3832}
3833
3834void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3835    actOnDeviceGlobalVarEntriesInfo(
3836        const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
3837  // Scan all target region entries and perform the provided action.
3838  for (const auto &E : OffloadEntriesDeviceGlobalVar)
3839    Action(E.getKey(), E.getValue());
3840}
3841
3842llvm::Function *
3843CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration() {
3844  // If we don't have entries or if we are emitting code for the device, we
3845  // don't need to do anything.
3846  if (CGM.getLangOpts().OpenMPIsDevice || OffloadEntriesInfoManager.empty())
3847    return nullptr;
3848
3849  llvm::Module &M = CGM.getModule();
3850  ASTContext &C = CGM.getContext();
3851
3852  // Get list of devices we care about
3853  const std::vector<llvm::Triple> &Devices = CGM.getLangOpts().OMPTargetTriples;
3854
3855  // We should be creating an offloading descriptor only if there are devices
3856  // specified.
3857   (0) . __assert_fail ("!Devices.empty() && \"No OpenMP offloading devices??\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 3857, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!Devices.empty() && "No OpenMP offloading devices??");
3858
3859  // Create the external variables that will point to the begin and end of the
3860  // host entries section. These will be defined by the linker.
3861  llvm::Type *OffloadEntryTy =
3862      CGM.getTypes().ConvertTypeForMem(getTgtOffloadEntryQTy());
3863  std::string EntriesBeginName = getName({"omp_offloading", "entries_begin"});
3864  auto *HostEntriesBegin = new llvm::GlobalVariable(
3865      M, OffloadEntryTy, /*isConstant=*/true,
3866      llvm::GlobalValue::ExternalLinkage, /*Initializer=*/nullptr,
3867      EntriesBeginName);
3868  std::string EntriesEndName = getName({"omp_offloading", "entries_end"});
3869  auto *HostEntriesEnd =
3870      new llvm::GlobalVariable(M, OffloadEntryTy, /*isConstant=*/true,
3871                               llvm::GlobalValue::ExternalLinkage,
3872                               /*Initializer=*/nullptr, EntriesEndName);
3873
3874  // Create all device images
3875  auto *DeviceImageTy = cast<llvm::StructType>(
3876      CGM.getTypes().ConvertTypeForMem(getTgtDeviceImageQTy()));
3877  ConstantInitBuilder DeviceImagesBuilder(CGM);
3878  ConstantArrayBuilder DeviceImagesEntries =
3879      DeviceImagesBuilder.beginArray(DeviceImageTy);
3880
3881  for (const llvm::Triple &Device : Devices) {
3882    StringRef T = Device.getTriple();
3883    std::string BeginName = getName({"omp_offloading", "img_start", ""});
3884    auto *ImgBegin = new llvm::GlobalVariable(
3885        M, CGM.Int8Ty, /*isConstant=*/true,
3886        llvm::GlobalValue::ExternalWeakLinkage,
3887        /*Initializer=*/nullptr, Twine(BeginName).concat(T));
3888    std::string EndName = getName({"omp_offloading", "img_end", ""});
3889    auto *ImgEnd = new llvm::GlobalVariable(
3890        M, CGM.Int8Ty, /*isConstant=*/true,
3891        llvm::GlobalValue::ExternalWeakLinkage,
3892        /*Initializer=*/nullptr, Twine(EndName).concat(T));
3893
3894    llvm::Constant *Data[] = {ImgBegin, ImgEnd, HostEntriesBegin,
3895                              HostEntriesEnd};
3896    createConstantGlobalStructAndAddToParent(CGM, getTgtDeviceImageQTy(), Data,
3897                                             DeviceImagesEntries);
3898  }
3899
3900  // Create device images global array.
3901  std::string ImagesName = getName({"omp_offloading", "device_images"});
3902  llvm::GlobalVariable *DeviceImages =
3903      DeviceImagesEntries.finishAndCreateGlobal(ImagesName,
3904                                                CGM.getPointerAlign(),
3905                                                /*isConstant=*/true);
3906  DeviceImages->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3907
3908  // This is a Zero array to be used in the creation of the constant expressions
3909  llvm::Constant *Index[] = {llvm::Constant::getNullValue(CGM.Int32Ty),
3910                             llvm::Constant::getNullValue(CGM.Int32Ty)};
3911
3912  // Create the target region descriptor.
3913  llvm::Constant *Data[] = {
3914      llvm::ConstantInt::get(CGM.Int32Ty, Devices.size()),
3915      llvm::ConstantExpr::getGetElementPtr(DeviceImages->getValueType(),
3916                                           DeviceImages, Index),
3917      HostEntriesBegin, HostEntriesEnd};
3918  std::string Descriptor = getName({"omp_offloading", "descriptor"});
3919  llvm::GlobalVariable *Desc = createGlobalStruct(
3920      CGM, getTgtBinaryDescriptorQTy(), /*IsConstant=*/true, Data, Descriptor);
3921
3922  // Emit code to register or unregister the descriptor at execution
3923  // startup or closing, respectively.
3924
3925  llvm::Function *UnRegFn;
3926  {
3927    FunctionArgList Args;
3928    ImplicitParamDecl DummyPtr(C, C.VoidPtrTy, ImplicitParamDecl::Other);
3929    Args.push_back(&DummyPtr);
3930
3931    CodeGenFunction CGF(CGM);
3932    // Disable debug info for global (de-)initializer because they are not part
3933    // of some particular construct.
3934    CGF.disableDebugInfo();
3935    const auto &FI =
3936        CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3937    llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
3938    std::string UnregName = getName({"omp_offloading", "descriptor_unreg"});
3939    UnRegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, UnregName, FI);
3940    CGF.StartFunction(GlobalDecl(), C.VoidTy, UnRegFn, FI, Args);
3941    CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_unregister_lib),
3942                        Desc);
3943    CGF.FinishFunction();
3944  }
3945  llvm::Function *RegFn;
3946  {
3947    CodeGenFunction CGF(CGM);
3948    // Disable debug info for global (de-)initializer because they are not part
3949    // of some particular construct.
3950    CGF.disableDebugInfo();
3951    const auto &FI = CGM.getTypes().arrangeNullaryFunction();
3952    llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
3953
3954    // Encode offload target triples into the registration function name. It
3955    // will serve as a comdat key for the registration/unregistration code for
3956    // this particular combination of offloading targets.
3957    SmallVector<StringRef, 4U> RegFnNameParts(Devices.size() + 2U);
3958    RegFnNameParts[0] = "omp_offloading";
3959    RegFnNameParts[1] = "descriptor_reg";
3960    llvm::transform(Devices, std::next(RegFnNameParts.begin(), 2),
3961                    [](const llvm::Triple &T) -> const std::string& {
3962                      return T.getTriple();
3963                    });
3964    llvm::sort(std::next(RegFnNameParts.begin(), 2), RegFnNameParts.end());
3965    std::string Descriptor = getName(RegFnNameParts);
3966    RegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, Descriptor, FI);
3967    CGF.StartFunction(GlobalDecl(), C.VoidTy, RegFn, FI, FunctionArgList());
3968    CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_register_lib), Desc);
3969    // Create a variable to drive the registration and unregistration of the
3970    // descriptor, so we can reuse the logic that emits Ctors and Dtors.
3971    ImplicitParamDecl RegUnregVar(C, C.getTranslationUnitDecl(),
3972                                  SourceLocation(), nullptr, C.CharTy,
3973                                  ImplicitParamDecl::Other);
3974    CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc);
3975    CGF.FinishFunction();
3976  }
3977  if (CGM.supportsCOMDAT()) {
3978    // It is sufficient to call registration function only once, so create a
3979    // COMDAT group for registration/unregistration functions and associated
3980    // data. That would reduce startup time and code size. Registration
3981    // function serves as a COMDAT group key.
3982    llvm::Comdat *ComdatKey = M.getOrInsertComdat(RegFn->getName());
3983    RegFn->setLinkage(llvm::GlobalValue::LinkOnceAnyLinkage);
3984    RegFn->setVisibility(llvm::GlobalValue::HiddenVisibility);
3985    RegFn->setComdat(ComdatKey);
3986    UnRegFn->setComdat(ComdatKey);
3987    DeviceImages->setComdat(ComdatKey);
3988    Desc->setComdat(ComdatKey);
3989  }
3990  return RegFn;
3991}
3992
3993void CGOpenMPRuntime::createOffloadEntry(
3994    llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
3995    llvm::GlobalValue::LinkageTypes Linkage) {
3996  StringRef Name = Addr->getName();
3997  llvm::Module &M = CGM.getModule();
3998  llvm::LLVMContext &C = M.getContext();
3999
4000  // Create constant string with the name.
4001  llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
4002
4003  std::string StringName = getName({"omp_offloading", "entry_name"});
4004  auto *Str = new llvm::GlobalVariable(
4005      M, StrPtrInit->getType(), /*isConstant=*/true,
4006      llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
4007  Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4008
4009  llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy),
4010                            llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy),
4011                            llvm::ConstantInt::get(CGM.SizeTy, Size),
4012                            llvm::ConstantInt::get(CGM.Int32Ty, Flags),
4013                            llvm::ConstantInt::get(CGM.Int32Ty, 0)};
4014  std::string EntryName = getName({"omp_offloading", "entry", ""});
4015  llvm::GlobalVariable *Entry = createGlobalStruct(
4016      CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
4017      Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
4018
4019  // The entry has to be created in the section the linker expects it to be.
4020  std::string Section = getName({"omp_offloading", "entries"});
4021  Entry->setSection(Section);
4022}
4023
4024void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
4025  // Emit the offloading entries and metadata so that the device codegen side
4026  // can easily figure out what to emit. The produced metadata looks like
4027  // this:
4028  //
4029  // !omp_offload.info = !{!1, ...}
4030  //
4031  // Right now we only generate metadata for function that contain target
4032  // regions.
4033
4034  // If we do not have entries, we don't need to do anything.
4035  if (OffloadEntriesInfoManager.empty())
4036    return;
4037
4038  llvm::Module &M = CGM.getModule();
4039  llvm::LLVMContext &C = M.getContext();
4040  SmallVector<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *, 16>
4041      OrderedEntries(OffloadEntriesInfoManager.size());
4042  llvm::SmallVector<StringRef, 16> ParentFunctions(
4043      OffloadEntriesInfoManager.size());
4044
4045  // Auxiliary methods to create metadata values and strings.
4046  auto &&GetMDInt = [this](unsigned V) {
4047    return llvm::ConstantAsMetadata::get(
4048        llvm::ConstantInt::get(CGM.Int32Ty, V));
4049  };
4050
4051  auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
4052
4053  // Create the offloading info metadata node.
4054  llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
4055
4056  // Create function that emits metadata for each target region entry;
4057  auto &&TargetRegionMetadataEmitter =
4058      [&C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt, &GetMDString](
4059          unsigned DeviceID, unsigned FileID, StringRef ParentName,
4060          unsigned Line,
4061          const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
4062        // Generate metadata for target regions. Each entry of this metadata
4063        // contains:
4064        // - Entry 0 -> Kind of this type of metadata (0).
4065        // - Entry 1 -> Device ID of the file where the entry was identified.
4066        // - Entry 2 -> File ID of the file where the entry was identified.
4067        // - Entry 3 -> Mangled name of the function where the entry was
4068        // identified.
4069        // - Entry 4 -> Line in the file where the entry was identified.
4070        // - Entry 5 -> Order the entry was created.
4071        // The first element of the metadata node is the kind.
4072        llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
4073                                 GetMDInt(FileID),      GetMDString(ParentName),
4074                                 GetMDInt(Line),        GetMDInt(E.getOrder())};
4075
4076        // Save this entry in the right position of the ordered entries array.
4077        OrderedEntries[E.getOrder()] = &E;
4078        ParentFunctions[E.getOrder()] = ParentName;
4079
4080        // Add metadata to the named metadata node.
4081        MD->addOperand(llvm::MDNode::get(C, Ops));
4082      };
4083
4084  OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
4085      TargetRegionMetadataEmitter);
4086
4087  // Create function that emits metadata for each device global variable entry;
4088  auto &&DeviceGlobalVarMetadataEmitter =
4089      [&C, &OrderedEntries, &GetMDInt, &GetMDString,
4090       MD](StringRef MangledName,
4091           const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
4092               &E) {
4093        // Generate metadata for global variables. Each entry of this metadata
4094        // contains:
4095        // - Entry 0 -> Kind of this type of metadata (1).
4096        // - Entry 1 -> Mangled name of the variable.
4097        // - Entry 2 -> Declare target kind.
4098        // - Entry 3 -> Order the entry was created.
4099        // The first element of the metadata node is the kind.
4100        llvm::Metadata *Ops[] = {
4101            GetMDInt(E.getKind()), GetMDString(MangledName),
4102            GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
4103
4104        // Save this entry in the right position of the ordered entries array.
4105        OrderedEntries[E.getOrder()] = &E;
4106
4107        // Add metadata to the named metadata node.
4108        MD->addOperand(llvm::MDNode::get(C, Ops));
4109      };
4110
4111  OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
4112      DeviceGlobalVarMetadataEmitter);
4113
4114  for (const auto *E : OrderedEntries) {
4115     (0) . __assert_fail ("E && \"All ordered entries must exist!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4115, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(E && "All ordered entries must exist!");
4116    if (const auto *CE =
4117            dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
4118                E)) {
4119      if (!CE->getID() || !CE->getAddress()) {
4120        // Do not blame the entry if the parent funtion is not emitted.
4121        StringRef FnName = ParentFunctions[CE->getOrder()];
4122        if (!CGM.GetGlobalValue(FnName))
4123          continue;
4124        unsigned DiagID = CGM.getDiags().getCustomDiagID(
4125            DiagnosticsEngine::Error,
4126            "Offloading entry for target region is incorrect: either the "
4127            "address or the ID is invalid.");
4128        CGM.getDiags().Report(DiagID);
4129        continue;
4130      }
4131      createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
4132                         CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
4133    } else if (const auto *CE =
4134                   dyn_cast<OffloadEntriesInfoManagerTy::
4135                                OffloadEntryInfoDeviceGlobalVar>(E)) {
4136      OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
4137          static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4138              CE->getFlags());
4139      switch (Flags) {
4140      case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
4141        if (!CE->getAddress()) {
4142          unsigned DiagID = CGM.getDiags().getCustomDiagID(
4143              DiagnosticsEngine::Error,
4144              "Offloading entry for declare target variable is incorrect: the "
4145              "address is invalid.");
4146          CGM.getDiags().Report(DiagID);
4147          continue;
4148        }
4149        // The vaiable has no definition - no need to add the entry.
4150        if (CE->getVarSize().isZero())
4151          continue;
4152        break;
4153      }
4154      case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
4155         (0) . __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4157, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||
4156 (0) . __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4157, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&
4157 (0) . __assert_fail ("((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && \"Declaret target link address is set.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4157, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">               "Declaret target link address is set.");
4158        if (CGM.getLangOpts().OpenMPIsDevice)
4159          continue;
4160        if (!CE->getAddress()) {
4161          unsigned DiagID = CGM.getDiags().getCustomDiagID(
4162              DiagnosticsEngine::Error,
4163              "Offloading entry for declare target variable is incorrect: the "
4164              "address is invalid.");
4165          CGM.getDiags().Report(DiagID);
4166          continue;
4167        }
4168        break;
4169      }
4170      createOffloadEntry(CE->getAddress(), CE->getAddress(),
4171                         CE->getVarSize().getQuantity(), Flags,
4172                         CE->getLinkage());
4173    } else {
4174      llvm_unreachable("Unsupported entry kind.");
4175    }
4176  }
4177}
4178
4179/// Loads all the offload entries information from the host IR
4180/// metadata.
4181void CGOpenMPRuntime::loadOffloadInfoMetadata() {
4182  // If we are in target mode, load the metadata from the host IR. This code has
4183  // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
4184
4185  if (!CGM.getLangOpts().OpenMPIsDevice)
4186    return;
4187
4188  if (CGM.getLangOpts().OMPHostIRFile.empty())
4189    return;
4190
4191  auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
4192  if (auto EC = Buf.getError()) {
4193    CGM.getDiags().Report(diag::err_cannot_open_file)
4194        << CGM.getLangOpts().OMPHostIRFile << EC.message();
4195    return;
4196  }
4197
4198  llvm::LLVMContext C;
4199  auto ME = expectedToErrorOrAndEmitErrors(
4200      C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
4201
4202  if (auto EC = ME.getError()) {
4203    unsigned DiagID = CGM.getDiags().getCustomDiagID(
4204        DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
4205    CGM.getDiags().Report(DiagID)
4206        << CGM.getLangOpts().OMPHostIRFile << EC.message();
4207    return;
4208  }
4209
4210  llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
4211  if (!MD)
4212    return;
4213
4214  for (llvm::MDNode *MN : MD->operands()) {
4215    auto &&GetMDInt = [MN](unsigned Idx) {
4216      auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
4217      return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
4218    };
4219
4220    auto &&GetMDString = [MN](unsigned Idx) {
4221      auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
4222      return V->getString();
4223    };
4224
4225    switch (GetMDInt(0)) {
4226    default:
4227      llvm_unreachable("Unexpected metadata!");
4228      break;
4229    case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4230        OffloadingEntryInfoTargetRegion:
4231      OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
4232          /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
4233          /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
4234          /*Order=*/GetMDInt(5));
4235      break;
4236    case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4237        OffloadingEntryInfoDeviceGlobalVar:
4238      OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
4239          /*MangledName=*/GetMDString(1),
4240          static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4241              /*Flags=*/GetMDInt(2)),
4242          /*Order=*/GetMDInt(3));
4243      break;
4244    }
4245  }
4246}
4247
4248void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
4249  if (!KmpRoutineEntryPtrTy) {
4250    // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
4251    ASTContext &C = CGM.getContext();
4252    QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
4253    FunctionProtoType::ExtProtoInfo EPI;
4254    KmpRoutineEntryPtrQTy = C.getPointerType(
4255        C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
4256    KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
4257  }
4258}
4259
4260QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
4261  // Make sure the type of the entry is already created. This is the type we
4262  // have to create:
4263  // struct __tgt_offload_entry{
4264  //   void      *addr;       // Pointer to the offload entry info.
4265  //                          // (function or global)
4266  //   char      *name;       // Name of the function or global.
4267  //   size_t     size;       // Size of the entry info (0 if it a function).
4268  //   int32_t    flags;      // Flags associated with the entry, e.g. 'link'.
4269  //   int32_t    reserved;   // Reserved, to use by the runtime library.
4270  // };
4271  if (TgtOffloadEntryQTy.isNull()) {
4272    ASTContext &C = CGM.getContext();
4273    RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
4274    RD->startDefinition();
4275    addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4276    addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
4277    addFieldToRecordDecl(C, RD, C.getSizeType());
4278    addFieldToRecordDecl(
4279        C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4280    addFieldToRecordDecl(
4281        C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4282    RD->completeDefinition();
4283    RD->addAttr(PackedAttr::CreateImplicit(C));
4284    TgtOffloadEntryQTy = C.getRecordType(RD);
4285  }
4286  return TgtOffloadEntryQTy;
4287}
4288
4289QualType CGOpenMPRuntime::getTgtDeviceImageQTy() {
4290  // These are the types we need to build:
4291  // struct __tgt_device_image{
4292  // void   *ImageStart;       // Pointer to the target code start.
4293  // void   *ImageEnd;         // Pointer to the target code end.
4294  // // We also add the host entries to the device image, as it may be useful
4295  // // for the target runtime to have access to that information.
4296  // __tgt_offload_entry  *EntriesBegin;   // Begin of the table with all
4297  //                                       // the entries.
4298  // __tgt_offload_entry  *EntriesEnd;     // End of the table with all the
4299  //                                       // entries (non inclusive).
4300  // };
4301  if (TgtDeviceImageQTy.isNull()) {
4302    ASTContext &C = CGM.getContext();
4303    RecordDecl *RD = C.buildImplicitRecord("__tgt_device_image");
4304    RD->startDefinition();
4305    addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4306    addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4307    addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4308    addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4309    RD->completeDefinition();
4310    TgtDeviceImageQTy = C.getRecordType(RD);
4311  }
4312  return TgtDeviceImageQTy;
4313}
4314
4315QualType CGOpenMPRuntime::getTgtBinaryDescriptorQTy() {
4316  // struct __tgt_bin_desc{
4317  //   int32_t              NumDevices;      // Number of devices supported.
4318  //   __tgt_device_image   *DeviceImages;   // Arrays of device images
4319  //                                         // (one per device).
4320  //   __tgt_offload_entry  *EntriesBegin;   // Begin of the table with all the
4321  //                                         // entries.
4322  //   __tgt_offload_entry  *EntriesEnd;     // End of the table with all the
4323  //                                         // entries (non inclusive).
4324  // };
4325  if (TgtBinaryDescriptorQTy.isNull()) {
4326    ASTContext &C = CGM.getContext();
4327    RecordDecl *RD = C.buildImplicitRecord("__tgt_bin_desc");
4328    RD->startDefinition();
4329    addFieldToRecordDecl(
4330        C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4331    addFieldToRecordDecl(C, RD, C.getPointerType(getTgtDeviceImageQTy()));
4332    addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4333    addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4334    RD->completeDefinition();
4335    TgtBinaryDescriptorQTy = C.getRecordType(RD);
4336  }
4337  return TgtBinaryDescriptorQTy;
4338}
4339
4340namespace {
4341struct PrivateHelpersTy {
4342  PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
4343                   const VarDecl *PrivateElemInit)
4344      : Original(Original), PrivateCopy(PrivateCopy),
4345        PrivateElemInit(PrivateElemInit) {}
4346  const VarDecl *Original;
4347  const VarDecl *PrivateCopy;
4348  const VarDecl *PrivateElemInit;
4349};
4350typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
4351} // anonymous namespace
4352
4353static RecordDecl *
4354createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
4355  if (!Privates.empty()) {
4356    ASTContext &C = CGM.getContext();
4357    // Build struct .kmp_privates_t. {
4358    //         /*  private vars  */
4359    //       };
4360    RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
4361    RD->startDefinition();
4362    for (const auto &Pair : Privates) {
4363      const VarDecl *VD = Pair.second.Original;
4364      QualType Type = VD->getType().getNonReferenceType();
4365      FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
4366      if (VD->hasAttrs()) {
4367        for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
4368             E(VD->getAttrs().end());
4369             I != E; ++I)
4370          FD->addAttr(*I);
4371      }
4372    }
4373    RD->completeDefinition();
4374    return RD;
4375  }
4376  return nullptr;
4377}
4378
4379static RecordDecl *
4380createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
4381                         QualType KmpInt32Ty,
4382                         QualType KmpRoutineEntryPointerQTy) {
4383  ASTContext &C = CGM.getContext();
4384  // Build struct kmp_task_t {
4385  //         void *              shareds;
4386  //         kmp_routine_entry_t routine;
4387  //         kmp_int32           part_id;
4388  //         kmp_cmplrdata_t data1;
4389  //         kmp_cmplrdata_t data2;
4390  // For taskloops additional fields:
4391  //         kmp_uint64          lb;
4392  //         kmp_uint64          ub;
4393  //         kmp_int64           st;
4394  //         kmp_int32           liter;
4395  //         void *              reductions;
4396  //       };
4397  RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
4398  UD->startDefinition();
4399  addFieldToRecordDecl(C, UD, KmpInt32Ty);
4400  addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
4401  UD->completeDefinition();
4402  QualType KmpCmplrdataTy = C.getRecordType(UD);
4403  RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
4404  RD->startDefinition();
4405  addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4406  addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
4407  addFieldToRecordDecl(C, RD, KmpInt32Ty);
4408  addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4409  addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4410  if (isOpenMPTaskLoopDirective(Kind)) {
4411    QualType KmpUInt64Ty =
4412        CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
4413    QualType KmpInt64Ty =
4414        CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
4415    addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4416    addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4417    addFieldToRecordDecl(C, RD, KmpInt64Ty);
4418    addFieldToRecordDecl(C, RD, KmpInt32Ty);
4419    addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4420  }
4421  RD->completeDefinition();
4422  return RD;
4423}
4424
4425static RecordDecl *
4426createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
4427                                     ArrayRef<PrivateDataTy> Privates) {
4428  ASTContext &C = CGM.getContext();
4429  // Build struct kmp_task_t_with_privates {
4430  //         kmp_task_t task_data;
4431  //         .kmp_privates_t. privates;
4432  //       };
4433  RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
4434  RD->startDefinition();
4435  addFieldToRecordDecl(C, RD, KmpTaskTQTy);
4436  if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
4437    addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
4438  RD->completeDefinition();
4439  return RD;
4440}
4441
4442/// Emit a proxy function which accepts kmp_task_t as the second
4443/// argument.
4444/// \code
4445/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
4446///   TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
4447///   For taskloops:
4448///   tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4449///   tt->reductions, tt->shareds);
4450///   return 0;
4451/// }
4452/// \endcode
4453static llvm::Function *
4454emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
4455                      OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
4456                      QualType KmpTaskTWithPrivatesPtrQTy,
4457                      QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
4458                      QualType SharedsPtrTy, llvm::Function *TaskFunction,
4459                      llvm::Value *TaskPrivatesMap) {
4460  ASTContext &C = CGM.getContext();
4461  FunctionArgList Args;
4462  ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4463                            ImplicitParamDecl::Other);
4464  ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4465                                KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4466                                ImplicitParamDecl::Other);
4467  Args.push_back(&GtidArg);
4468  Args.push_back(&TaskTypeArg);
4469  const auto &TaskEntryFnInfo =
4470      CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4471  llvm::FunctionType *TaskEntryTy =
4472      CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
4473  std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
4474  auto *TaskEntry = llvm::Function::Create(
4475      TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4476  CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
4477  TaskEntry->setDoesNotRecurse();
4478  CodeGenFunction CGF(CGM);
4479  CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
4480                    Loc, Loc);
4481
4482  // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
4483  // tt,
4484  // For taskloops:
4485  // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4486  // tt->task_data.shareds);
4487  llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
4488      CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
4489  LValue TDBase = CGF.EmitLoadOfPointerLValue(
4490      CGF.GetAddrOfLocalVar(&TaskTypeArg),
4491      KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4492  const auto *KmpTaskTWithPrivatesQTyRD =
4493      cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4494  LValue Base =
4495      CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4496  const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4497  auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4498  LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
4499  llvm::Value *PartidParam = PartIdLVal.getPointer();
4500
4501  auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
4502  LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
4503  llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4504      CGF.EmitLoadOfScalar(SharedsLVal, Loc),
4505      CGF.ConvertTypeForMem(SharedsPtrTy));
4506
4507  auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4508  llvm::Value *PrivatesParam;
4509  if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
4510    LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
4511    PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4512        PrivatesLVal.getPointer(), CGF.VoidPtrTy);
4513  } else {
4514    PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4515  }
4516
4517  llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
4518                               TaskPrivatesMap,
4519                               CGF.Builder
4520                                   .CreatePointerBitCastOrAddrSpaceCast(
4521                                       TDBase.getAddress(), CGF.VoidPtrTy)
4522                                   .getPointer()};
4523  SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
4524                                          std::end(CommonArgs));
4525  if (isOpenMPTaskLoopDirective(Kind)) {
4526    auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
4527    LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
4528    llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
4529    auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
4530    LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
4531    llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
4532    auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
4533    LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
4534    llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
4535    auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4536    LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4537    llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
4538    auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
4539    LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
4540    llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
4541    CallArgs.push_back(LBParam);
4542    CallArgs.push_back(UBParam);
4543    CallArgs.push_back(StParam);
4544    CallArgs.push_back(LIParam);
4545    CallArgs.push_back(RParam);
4546  }
4547  CallArgs.push_back(SharedsParam);
4548
4549  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
4550                                                  CallArgs);
4551  CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
4552                             CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
4553  CGF.FinishFunction();
4554  return TaskEntry;
4555}
4556
4557static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
4558                                            SourceLocation Loc,
4559                                            QualType KmpInt32Ty,
4560                                            QualType KmpTaskTWithPrivatesPtrQTy,
4561                                            QualType KmpTaskTWithPrivatesQTy) {
4562  ASTContext &C = CGM.getContext();
4563  FunctionArgList Args;
4564  ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4565                            ImplicitParamDecl::Other);
4566  ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4567                                KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4568                                ImplicitParamDecl::Other);
4569  Args.push_back(&GtidArg);
4570  Args.push_back(&TaskTypeArg);
4571  const auto &DestructorFnInfo =
4572      CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4573  llvm::FunctionType *DestructorFnTy =
4574      CGM.getTypes().GetFunctionType(DestructorFnInfo);
4575  std::string Name =
4576      CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
4577  auto *DestructorFn =
4578      llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
4579                             Name, &CGM.getModule());
4580  CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
4581                                    DestructorFnInfo);
4582  DestructorFn->setDoesNotRecurse();
4583  CodeGenFunction CGF(CGM);
4584  CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
4585                    Args, Loc, Loc);
4586
4587  LValue Base = CGF.EmitLoadOfPointerLValue(
4588      CGF.GetAddrOfLocalVar(&TaskTypeArg),
4589      KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4590  const auto *KmpTaskTWithPrivatesQTyRD =
4591      cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4592  auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4593  Base = CGF.EmitLValueForField(Base, *FI);
4594  for (const auto *Field :
4595       cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
4596    if (QualType::DestructionKind DtorKind =
4597            Field->getType().isDestructedType()) {
4598      LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
4599      CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
4600    }
4601  }
4602  CGF.FinishFunction();
4603  return DestructorFn;
4604}
4605
4606/// Emit a privates mapping function for correct handling of private and
4607/// firstprivate variables.
4608/// \code
4609/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
4610/// **noalias priv1,...,  <tyn> **noalias privn) {
4611///   *priv1 = &.privates.priv1;
4612///   ...;
4613///   *privn = &.privates.privn;
4614/// }
4615/// \endcode
4616static llvm::Value *
4617emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
4618                               ArrayRef<const Expr *> PrivateVars,
4619                               ArrayRef<const Expr *> FirstprivateVars,
4620                               ArrayRef<const Expr *> LastprivateVars,
4621                               QualType PrivatesQTy,
4622                               ArrayRef<PrivateDataTy> Privates) {
4623  ASTContext &C = CGM.getContext();
4624  FunctionArgList Args;
4625  ImplicitParamDecl TaskPrivatesArg(
4626      C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4627      C.getPointerType(PrivatesQTy).withConst().withRestrict(),
4628      ImplicitParamDecl::Other);
4629  Args.push_back(&TaskPrivatesArg);
4630  llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
4631  unsigned Counter = 1;
4632  for (const Expr *E : PrivateVars) {
4633    Args.push_back(ImplicitParamDecl::Create(
4634        C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4635        C.getPointerType(C.getPointerType(E->getType()))
4636            .withConst()
4637            .withRestrict(),
4638        ImplicitParamDecl::Other));
4639    const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4640    PrivateVarsPos[VD] = Counter;
4641    ++Counter;
4642  }
4643  for (const Expr *E : FirstprivateVars) {
4644    Args.push_back(ImplicitParamDecl::Create(
4645        C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4646        C.getPointerType(C.getPointerType(E->getType()))
4647            .withConst()
4648            .withRestrict(),
4649        ImplicitParamDecl::Other));
4650    const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4651    PrivateVarsPos[VD] = Counter;
4652    ++Counter;
4653  }
4654  for (const Expr *E : LastprivateVars) {
4655    Args.push_back(ImplicitParamDecl::Create(
4656        C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4657        C.getPointerType(C.getPointerType(E->getType()))
4658            .withConst()
4659            .withRestrict(),
4660        ImplicitParamDecl::Other));
4661    const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4662    PrivateVarsPos[VD] = Counter;
4663    ++Counter;
4664  }
4665  const auto &TaskPrivatesMapFnInfo =
4666      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4667  llvm::FunctionType *TaskPrivatesMapTy =
4668      CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
4669  std::string Name =
4670      CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
4671  auto *TaskPrivatesMap = llvm::Function::Create(
4672      TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
4673      &CGM.getModule());
4674  CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
4675                                    TaskPrivatesMapFnInfo);
4676  TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
4677  TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
4678  TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
4679  CodeGenFunction CGF(CGM);
4680  CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
4681                    TaskPrivatesMapFnInfo, Args, Loc, Loc);
4682
4683  // *privi = &.privates.privi;
4684  LValue Base = CGF.EmitLoadOfPointerLValue(
4685      CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
4686      TaskPrivatesArg.getType()->castAs<PointerType>());
4687  const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
4688  Counter = 0;
4689  for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
4690    LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
4691    const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4692    LValue RefLVal =
4693        CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
4694    LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
4695        RefLVal.getAddress(), RefLVal.getType()->castAs<PointerType>());
4696    CGF.EmitStoreOfScalar(FieldLVal.getPointer(), RefLoadLVal);
4697    ++Counter;
4698  }
4699  CGF.FinishFunction();
4700  return TaskPrivatesMap;
4701}
4702
4703static bool stable_sort_comparator(const PrivateDataTy P1,
4704                                   const PrivateDataTy P2) {
4705  return P1.first > P2.first;
4706}
4707
4708/// Emit initialization for private variables in task-based directives.
4709static void emitPrivatesInit(CodeGenFunction &CGF,
4710                             const OMPExecutableDirective &D,
4711                             Address KmpTaskSharedsPtr, LValue TDBase,
4712                             const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4713                             QualType SharedsTy, QualType SharedsPtrTy,
4714                             const OMPTaskDataTy &Data,
4715                             ArrayRef<PrivateDataTy> Privates, bool ForDup) {
4716  ASTContext &C = CGF.getContext();
4717  auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4718  LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
4719  OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
4720                                 ? OMPD_taskloop
4721                                 : OMPD_task;
4722  const CapturedStmt &CS = *D.getCapturedStmt(Kind);
4723  CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
4724  LValue SrcBase;
4725  bool IsTargetTask =
4726      isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
4727      isOpenMPTargetExecutionDirective(D.getDirectiveKind());
4728  // For target-based directives skip 3 firstprivate arrays BasePointersArray,
4729  // PointersArray and SizesArray. The original variables for these arrays are
4730  // not captured and we get their addresses explicitly.
4731  if ((!IsTargetTask && !Data.FirstprivateVars.empty()) ||
4732      (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
4733    SrcBase = CGF.MakeAddrLValue(
4734        CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4735            KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
4736        SharedsTy);
4737  }
4738  FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4739  for (const PrivateDataTy &Pair : Privates) {
4740    const VarDecl *VD = Pair.second.PrivateCopy;
4741    const Expr *Init = VD->getAnyInitializer();
4742    if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4743                             !CGF.isTrivialInitializer(Init)))) {
4744      LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
4745      if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
4746        const VarDecl *OriginalVD = Pair.second.Original;
4747        // Check if the variable is the target-based BasePointersArray,
4748        // PointersArray or SizesArray.
4749        LValue SharedRefLValue;
4750        QualType Type = PrivateLValue.getType();
4751        const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
4752        if (IsTargetTask && !SharedField) {
4753           (0) . __assert_fail ("isa(OriginalVD) && isa(OriginalVD->getDeclContext()) && cast(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa( cast(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4760, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(isa<ImplicitParamDecl>(OriginalVD) &&
4754 (0) . __assert_fail ("isa(OriginalVD) && isa(OriginalVD->getDeclContext()) && cast(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa( cast(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4760, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&
4755 (0) . __assert_fail ("isa(OriginalVD) && isa(OriginalVD->getDeclContext()) && cast(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa( cast(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4760, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                 cast<CapturedDecl>(OriginalVD->getDeclContext())
4756 (0) . __assert_fail ("isa(OriginalVD) && isa(OriginalVD->getDeclContext()) && cast(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa( cast(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4760, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                         ->getNumParams() == 0 &&
4757 (0) . __assert_fail ("isa(OriginalVD) && isa(OriginalVD->getDeclContext()) && cast(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa( cast(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4760, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                 isa<TranslationUnitDecl>(
4758 (0) . __assert_fail ("isa(OriginalVD) && isa(OriginalVD->getDeclContext()) && cast(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa( cast(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4760, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                     cast<CapturedDecl>(OriginalVD->getDeclContext())
4759 (0) . __assert_fail ("isa(OriginalVD) && isa(OriginalVD->getDeclContext()) && cast(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa( cast(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4760, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                         ->getDeclContext()) &&
4760 (0) . __assert_fail ("isa(OriginalVD) && isa(OriginalVD->getDeclContext()) && cast(OriginalVD->getDeclContext()) ->getNumParams() == 0 && isa( cast(OriginalVD->getDeclContext()) ->getDeclContext()) && \"Expected artificial target data variable.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4760, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                 "Expected artificial target data variable.");
4761          SharedRefLValue =
4762              CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
4763        } else {
4764          SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
4765          SharedRefLValue = CGF.MakeAddrLValue(
4766              Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)),
4767              SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
4768              SharedRefLValue.getTBAAInfo());
4769        }
4770        if (Type->isArrayType()) {
4771          // Initialize firstprivate array.
4772          if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
4773            // Perform simple memcpy.
4774            CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
4775          } else {
4776            // Initialize firstprivate array using element-by-element
4777            // initialization.
4778            CGF.EmitOMPAggregateAssign(
4779                PrivateLValue.getAddress(), SharedRefLValue.getAddress(), Type,
4780                [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
4781                                                  Address SrcElement) {
4782                  // Clean up any temporaries needed by the initialization.
4783                  CodeGenFunction::OMPPrivateScope InitScope(CGF);
4784                  InitScope.addPrivate(
4785                      Elem, [SrcElement]() -> Address { return SrcElement; });
4786                  (void)InitScope.Privatize();
4787                  // Emit initialization for single element.
4788                  CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
4789                      CGF, &CapturesInfo);
4790                  CGF.EmitAnyExprToMem(Init, DestElement,
4791                                       Init->getType().getQualifiers(),
4792                                       /*IsInitializer=*/false);
4793                });
4794          }
4795        } else {
4796          CodeGenFunction::OMPPrivateScope InitScope(CGF);
4797          InitScope.addPrivate(Elem, [SharedRefLValue]() -> Address {
4798            return SharedRefLValue.getAddress();
4799          });
4800          (void)InitScope.Privatize();
4801          CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
4802          CGF.EmitExprAsInit(Init, VD, PrivateLValue,
4803                             /*capturedByInit=*/false);
4804        }
4805      } else {
4806        CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
4807      }
4808    }
4809    ++FI;
4810  }
4811}
4812
4813/// Check if duplication function is required for taskloops.
4814static bool checkInitIsRequired(CodeGenFunction &CGF,
4815                                ArrayRef<PrivateDataTy> Privates) {
4816  bool InitRequired = false;
4817  for (const PrivateDataTy &Pair : Privates) {
4818    const VarDecl *VD = Pair.second.PrivateCopy;
4819    const Expr *Init = VD->getAnyInitializer();
4820    InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
4821                                    !CGF.isTrivialInitializer(Init));
4822    if (InitRequired)
4823      break;
4824  }
4825  return InitRequired;
4826}
4827
4828
4829/// Emit task_dup function (for initialization of
4830/// private/firstprivate/lastprivate vars and last_iter flag)
4831/// \code
4832/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
4833/// lastpriv) {
4834/// // setup lastprivate flag
4835///    task_dst->last = lastpriv;
4836/// // could be constructor calls here...
4837/// }
4838/// \endcode
4839static llvm::Value *
4840emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
4841                    const OMPExecutableDirective &D,
4842                    QualType KmpTaskTWithPrivatesPtrQTy,
4843                    const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4844                    const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
4845                    QualType SharedsPtrTy, const OMPTaskDataTy &Data,
4846                    ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
4847  ASTContext &C = CGM.getContext();
4848  FunctionArgList Args;
4849  ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4850                           KmpTaskTWithPrivatesPtrQTy,
4851                           ImplicitParamDecl::Other);
4852  ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4853                           KmpTaskTWithPrivatesPtrQTy,
4854                           ImplicitParamDecl::Other);
4855  ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
4856                                ImplicitParamDecl::Other);
4857  Args.push_back(&DstArg);
4858  Args.push_back(&SrcArg);
4859  Args.push_back(&LastprivArg);
4860  const auto &TaskDupFnInfo =
4861      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4862  llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
4863  std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
4864  auto *TaskDup = llvm::Function::Create(
4865      TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4866  CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
4867  TaskDup->setDoesNotRecurse();
4868  CodeGenFunction CGF(CGM);
4869  CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
4870                    Loc);
4871
4872  LValue TDBase = CGF.EmitLoadOfPointerLValue(
4873      CGF.GetAddrOfLocalVar(&DstArg),
4874      KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4875  // task_dst->liter = lastpriv;
4876  if (WithLastIter) {
4877    auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4878    LValue Base = CGF.EmitLValueForField(
4879        TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4880    LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4881    llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
4882        CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
4883    CGF.EmitStoreOfScalar(Lastpriv, LILVal);
4884  }
4885
4886  // Emit initial values for private copies (if any).
4887  assert(!Privates.empty());
4888  Address KmpTaskSharedsPtr = Address::invalid();
4889  if (!Data.FirstprivateVars.empty()) {
4890    LValue TDBase = CGF.EmitLoadOfPointerLValue(
4891        CGF.GetAddrOfLocalVar(&SrcArg),
4892        KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4893    LValue Base = CGF.EmitLValueForField(
4894        TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4895    KmpTaskSharedsPtr = Address(
4896        CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
4897                                 Base, *std::next(KmpTaskTQTyRD->field_begin(),
4898                                                  KmpTaskTShareds)),
4899                             Loc),
4900        CGF.getNaturalTypeAlignment(SharedsTy));
4901  }
4902  emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
4903                   SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
4904  CGF.FinishFunction();
4905  return TaskDup;
4906}
4907
4908/// Checks if destructor function is required to be generated.
4909/// \return true if cleanups are required, false otherwise.
4910static bool
4911checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD) {
4912  bool NeedsCleanup = false;
4913  auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4914  const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
4915  for (const FieldDecl *FD : PrivateRD->fields()) {
4916    NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
4917    if (NeedsCleanup)
4918      break;
4919  }
4920  return NeedsCleanup;
4921}
4922
4923CGOpenMPRuntime::TaskResultTy
4924CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
4925                              const OMPExecutableDirective &D,
4926                              llvm::Function *TaskFunction, QualType SharedsTy,
4927                              Address Shareds, const OMPTaskDataTy &Data) {
4928  ASTContext &C = CGM.getContext();
4929  llvm::SmallVector<PrivateDataTy, 4> Privates;
4930  // Aggregate privates and sort them by the alignment.
4931  auto I = Data.PrivateCopies.begin();
4932  for (const Expr *E : Data.PrivateVars) {
4933    const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4934    Privates.emplace_back(
4935        C.getDeclAlign(VD),
4936        PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4937                         /*PrivateElemInit=*/nullptr));
4938    ++I;
4939  }
4940  I = Data.FirstprivateCopies.begin();
4941  auto IElemInitRef = Data.FirstprivateInits.begin();
4942  for (const Expr *E : Data.FirstprivateVars) {
4943    const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4944    Privates.emplace_back(
4945        C.getDeclAlign(VD),
4946        PrivateHelpersTy(
4947            VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4948            cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4949    ++I;
4950    ++IElemInitRef;
4951  }
4952  I = Data.LastprivateCopies.begin();
4953  for (const Expr *E : Data.LastprivateVars) {
4954    const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4955    Privates.emplace_back(
4956        C.getDeclAlign(VD),
4957        PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4958                         /*PrivateElemInit=*/nullptr));
4959    ++I;
4960  }
4961  std::stable_sort(Privates.begin(), Privates.end(), stable_sort_comparator);
4962  QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4963  // Build type kmp_routine_entry_t (if not built yet).
4964  emitKmpRoutineEntryT(KmpInt32Ty);
4965  // Build type kmp_task_t (if not built yet).
4966  if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
4967    if (SavedKmpTaskloopTQTy.isNull()) {
4968      SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4969          CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4970    }
4971    KmpTaskTQTy = SavedKmpTaskloopTQTy;
4972  } else {
4973     (0) . __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4976, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert((D.getDirectiveKind() == OMPD_task ||
4974 (0) . __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4976, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">            isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||
4975 (0) . __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4976, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">            isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&
4976 (0) . __assert_fail ("(D.getDirectiveKind() == OMPD_task || isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && \"Expected taskloop, task or target directive\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 4976, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">           "Expected taskloop, task or target directive");
4977    if (SavedKmpTaskTQTy.isNull()) {
4978      SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4979          CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4980    }
4981    KmpTaskTQTy = SavedKmpTaskTQTy;
4982  }
4983  const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4984  // Build particular struct kmp_task_t for the given task.
4985  const RecordDecl *KmpTaskTWithPrivatesQTyRD =
4986      createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
4987  QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
4988  QualType KmpTaskTWithPrivatesPtrQTy =
4989      C.getPointerType(KmpTaskTWithPrivatesQTy);
4990  llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
4991  llvm::Type *KmpTaskTWithPrivatesPtrTy =
4992      KmpTaskTWithPrivatesTy->getPointerTo();
4993  llvm::Value *KmpTaskTWithPrivatesTySize =
4994      CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
4995  QualType SharedsPtrTy = C.getPointerType(SharedsTy);
4996
4997  // Emit initial values for private copies (if any).
4998  llvm::Value *TaskPrivatesMap = nullptr;
4999  llvm::Type *TaskPrivatesMapTy =
5000      std::next(TaskFunction->arg_begin(), 3)->getType();
5001  if (!Privates.empty()) {
5002    auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
5003    TaskPrivatesMap = emitTaskPrivateMappingFunction(
5004        CGM, Loc, Data.PrivateVars, Data.FirstprivateVars, Data.LastprivateVars,
5005        FI->getType(), Privates);
5006    TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5007        TaskPrivatesMap, TaskPrivatesMapTy);
5008  } else {
5009    TaskPrivatesMap = llvm::ConstantPointerNull::get(
5010        cast<llvm::PointerType>(TaskPrivatesMapTy));
5011  }
5012  // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
5013  // kmp_task_t *tt);
5014  llvm::Function *TaskEntry = emitProxyTaskFunction(
5015      CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5016      KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
5017      TaskPrivatesMap);
5018
5019  // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
5020  // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
5021  // kmp_routine_entry_t *task_entry);
5022  // Task flags. Format is taken from
5023  // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h,
5024  // description of kmp_tasking_flags struct.
5025  enum {
5026    TiedFlag = 0x1,
5027    FinalFlag = 0x2,
5028    DestructorsFlag = 0x8,
5029    PriorityFlag = 0x20
5030  };
5031  unsigned Flags = Data.Tied ? TiedFlag : 0;
5032  bool NeedsCleanup = false;
5033  if (!Privates.empty()) {
5034    NeedsCleanup = checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD);
5035    if (NeedsCleanup)
5036      Flags = Flags | DestructorsFlag;
5037  }
5038  if (Data.Priority.getInt())
5039    Flags = Flags | PriorityFlag;
5040  llvm::Value *TaskFlags =
5041      Data.Final.getPointer()
5042          ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
5043                                     CGF.Builder.getInt32(FinalFlag),
5044                                     CGF.Builder.getInt32(/*C=*/0))
5045          : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
5046  TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
5047  llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
5048  llvm::Value *AllocArgs[] = {emitUpdateLocation(CGF, Loc),
5049                              getThreadID(CGF, Loc), TaskFlags,
5050                              KmpTaskTWithPrivatesTySize, SharedsSize,
5051                              CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5052                                  TaskEntry, KmpRoutineEntryPtrTy)};
5053  llvm::Value *NewTask = CGF.EmitRuntimeCall(
5054      createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
5055  llvm::Value *NewTaskNewTaskTTy =
5056      CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5057          NewTask, KmpTaskTWithPrivatesPtrTy);
5058  LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
5059                                               KmpTaskTWithPrivatesQTy);
5060  LValue TDBase =
5061      CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
5062  // Fill the data in the resulting kmp_task_t record.
5063  // Copy shareds if there are any.
5064  Address KmpTaskSharedsPtr = Address::invalid();
5065  if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
5066    KmpTaskSharedsPtr =
5067        Address(CGF.EmitLoadOfScalar(
5068                    CGF.EmitLValueForField(
5069                        TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
5070                                           KmpTaskTShareds)),
5071                    Loc),
5072                CGF.getNaturalTypeAlignment(SharedsTy));
5073    LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
5074    LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
5075    CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
5076  }
5077  // Emit initial values for private copies (if any).
5078  TaskResultTy Result;
5079  if (!Privates.empty()) {
5080    emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
5081                     SharedsTy, SharedsPtrTy, Data, Privates,
5082                     /*ForDup=*/false);
5083    if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
5084        (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
5085      Result.TaskDupFn = emitTaskDupFunction(
5086          CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
5087          KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
5088          /*WithLastIter=*/!Data.LastprivateVars.empty());
5089    }
5090  }
5091  // Fields of union "kmp_cmplrdata_t" for destructors and priority.
5092  enum { Priority = 0, Destructors = 1 };
5093  // Provide pointer to function with destructors for privates.
5094  auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
5095  const RecordDecl *KmpCmplrdataUD =
5096      (*FI)->getType()->getAsUnionType()->getDecl();
5097  if (NeedsCleanup) {
5098    llvm::Value *DestructorFn = emitDestructorsFunction(
5099        CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5100        KmpTaskTWithPrivatesQTy);
5101    LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
5102    LValue DestructorsLV = CGF.EmitLValueForField(
5103        Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
5104    CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5105                              DestructorFn, KmpRoutineEntryPtrTy),
5106                          DestructorsLV);
5107  }
5108  // Set priority.
5109  if (Data.Priority.getInt()) {
5110    LValue Data2LV = CGF.EmitLValueForField(
5111        TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
5112    LValue PriorityLV = CGF.EmitLValueForField(
5113        Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
5114    CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
5115  }
5116  Result.NewTask = NewTask;
5117  Result.TaskEntry = TaskEntry;
5118  Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
5119  Result.TDBase = TDBase;
5120  Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
5121  return Result;
5122}
5123
5124void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
5125                                   const OMPExecutableDirective &D,
5126                                   llvm::Function *TaskFunction,
5127                                   QualType SharedsTy, Address Shareds,
5128                                   const Expr *IfCond,
5129                                   const OMPTaskDataTy &Data) {
5130  if (!CGF.HaveInsertPoint())
5131    return;
5132
5133  TaskResultTy Result =
5134      emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5135  llvm::Value *NewTask = Result.NewTask;
5136  llvm::Function *TaskEntry = Result.TaskEntry;
5137  llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
5138  LValue TDBase = Result.TDBase;
5139  const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
5140  ASTContext &C = CGM.getContext();
5141  // Process list of dependences.
5142  Address DependenciesArray = Address::invalid();
5143  unsigned NumDependencies = Data.Dependences.size();
5144  if (NumDependencies) {
5145    // Dependence kind for RTL.
5146    enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3, DepMutexInOutSet = 0x4 };
5147    enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
5148    RecordDecl *KmpDependInfoRD;
5149    QualType FlagsTy =
5150        C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
5151    llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5152    if (KmpDependInfoTy.isNull()) {
5153      KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
5154      KmpDependInfoRD->startDefinition();
5155      addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
5156      addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
5157      addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
5158      KmpDependInfoRD->completeDefinition();
5159      KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
5160    } else {
5161      KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5162    }
5163    // Define type kmp_depend_info[<Dependences.size()>];
5164    QualType KmpDependInfoArrayTy = C.getConstantArrayType(
5165        KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies),
5166        ArrayType::Normal, /*IndexTypeQuals=*/0);
5167    // kmp_depend_info[<Dependences.size()>] deps;
5168    DependenciesArray =
5169        CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
5170    for (unsigned I = 0; I < NumDependencies; ++I) {
5171      const Expr *E = Data.Dependences[I].second;
5172      LValue Addr = CGF.EmitLValue(E);
5173      llvm::Value *Size;
5174      QualType Ty = E->getType();
5175      if (const auto *ASE =
5176              dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
5177        LValue UpAddrLVal =
5178            CGF.EmitOMPArraySectionExpr(ASE, /*LowerBound=*/false);
5179        llvm::Value *UpAddr =
5180            CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(), /*Idx0=*/1);
5181        llvm::Value *LowIntPtr =
5182            CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy);
5183        llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
5184        Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
5185      } else {
5186        Size = CGF.getTypeSize(Ty);
5187      }
5188      LValue Base = CGF.MakeAddrLValue(
5189          CGF.Builder.CreateConstArrayGEP(DependenciesArray, I),
5190          KmpDependInfoTy);
5191      // deps[i].base_addr = &<Dependences[i].second>;
5192      LValue BaseAddrLVal = CGF.EmitLValueForField(
5193          Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
5194      CGF.EmitStoreOfScalar(
5195          CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGF.IntPtrTy),
5196          BaseAddrLVal);
5197      // deps[i].len = sizeof(<Dependences[i].second>);
5198      LValue LenLVal = CGF.EmitLValueForField(
5199          Base, *std::next(KmpDependInfoRD->field_begin(), Len));
5200      CGF.EmitStoreOfScalar(Size, LenLVal);
5201      // deps[i].flags = <Dependences[i].first>;
5202      RTLDependenceKindTy DepKind;
5203      switch (Data.Dependences[I].first) {
5204      case OMPC_DEPEND_in:
5205        DepKind = DepIn;
5206        break;
5207      // Out and InOut dependencies must use the same code.
5208      case OMPC_DEPEND_out:
5209      case OMPC_DEPEND_inout:
5210        DepKind = DepInOut;
5211        break;
5212      case OMPC_DEPEND_mutexinoutset:
5213        DepKind = DepMutexInOutSet;
5214        break;
5215      case OMPC_DEPEND_source:
5216      case OMPC_DEPEND_sink:
5217      case OMPC_DEPEND_unknown:
5218        llvm_unreachable("Unknown task dependence type");
5219      }
5220      LValue FlagsLVal = CGF.EmitLValueForField(
5221          Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5222      CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5223                            FlagsLVal);
5224    }
5225    DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5226        CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0), CGF.VoidPtrTy);
5227  }
5228
5229  // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5230  // libcall.
5231  // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5232  // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5233  // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5234  // list is not empty
5235  llvm::Value *ThreadID = getThreadID(CGF, Loc);
5236  llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5237  llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5238  llvm::Value *DepTaskArgs[7];
5239  if (NumDependencies) {
5240    DepTaskArgs[0] = UpLoc;
5241    DepTaskArgs[1] = ThreadID;
5242    DepTaskArgs[2] = NewTask;
5243    DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
5244    DepTaskArgs[4] = DependenciesArray.getPointer();
5245    DepTaskArgs[5] = CGF.Builder.getInt32(0);
5246    DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5247  }
5248  auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
5249                        &TaskArgs,
5250                        &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5251    if (!Data.Tied) {
5252      auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5253      LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5254      CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5255    }
5256    if (NumDependencies) {
5257      CGF.EmitRuntimeCall(
5258          createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), DepTaskArgs);
5259    } else {
5260      CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task),
5261                          TaskArgs);
5262    }
5263    // Check if parent region is untied and build return for untied task;
5264    if (auto *Region =
5265            dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5266      Region->emitUntiedSwitch(CGF);
5267  };
5268
5269  llvm::Value *DepWaitTaskArgs[6];
5270  if (NumDependencies) {
5271    DepWaitTaskArgs[0] = UpLoc;
5272    DepWaitTaskArgs[1] = ThreadID;
5273    DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
5274    DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5275    DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5276    DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5277  }
5278  auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
5279                        NumDependencies, &DepWaitTaskArgs,
5280                        Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5281    CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5282    CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5283    // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5284    // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5285    // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5286    // is specified.
5287    if (NumDependencies)
5288      CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
5289                          DepWaitTaskArgs);
5290    // Call proxy_task_entry(gtid, new_task);
5291    auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5292                      Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5293      Action.Enter(CGF);
5294      llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5295      CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5296                                                          OutlinedFnArgs);
5297    };
5298
5299    // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5300    // kmp_task_t *new_task);
5301    // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5302    // kmp_task_t *new_task);
5303    RegionCodeGenTy RCG(CodeGen);
5304    CommonActionTy Action(
5305        RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs,
5306        RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs);
5307    RCG.setAction(Action);
5308    RCG(CGF);
5309  };
5310
5311  if (IfCond) {
5312    emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5313  } else {
5314    RegionCodeGenTy ThenRCG(ThenCodeGen);
5315    ThenRCG(CGF);
5316  }
5317}
5318
5319void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5320                                       const OMPLoopDirective &D,
5321                                       llvm::Function *TaskFunction,
5322                                       QualType SharedsTy, Address Shareds,
5323                                       const Expr *IfCond,
5324                                       const OMPTaskDataTy &Data) {
5325  if (!CGF.HaveInsertPoint())
5326    return;
5327  TaskResultTy Result =
5328      emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5329  // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5330  // libcall.
5331  // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5332  // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5333  // sched, kmp_uint64 grainsize, void *task_dup);
5334  llvm::Value *ThreadID = getThreadID(CGF, Loc);
5335  llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5336  llvm::Value *IfVal;
5337  if (IfCond) {
5338    IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5339                                      /*isSigned=*/true);
5340  } else {
5341    IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5342  }
5343
5344  LValue LBLVal = CGF.EmitLValueForField(
5345      Result.TDBase,
5346      *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5347  const auto *LBVar =
5348      cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5349  CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),
5350                       /*IsInitializer=*/true);
5351  LValue UBLVal = CGF.EmitLValueForField(
5352      Result.TDBase,
5353      *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5354  const auto *UBVar =
5355      cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5356  CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),
5357                       /*IsInitializer=*/true);
5358  LValue StLVal = CGF.EmitLValueForField(
5359      Result.TDBase,
5360      *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5361  const auto *StVar =
5362      cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5363  CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),
5364                       /*IsInitializer=*/true);
5365  // Store reductions address.
5366  LValue RedLVal = CGF.EmitLValueForField(
5367      Result.TDBase,
5368      *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5369  if (Data.Reductions) {
5370    CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5371  } else {
5372    CGF.EmitNullInitialization(RedLVal.getAddress(),
5373                               CGF.getContext().VoidPtrTy);
5374  }
5375  enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5376  llvm::Value *TaskArgs[] = {
5377      UpLoc,
5378      ThreadID,
5379      Result.NewTask,
5380      IfVal,
5381      LBLVal.getPointer(),
5382      UBLVal.getPointer(),
5383      CGF.EmitLoadOfScalar(StLVal, Loc),
5384      llvm::ConstantInt::getSigned(
5385              CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5386      llvm::ConstantInt::getSigned(
5387          CGF.IntTy, Data.Schedule.getPointer()
5388                         ? Data.Schedule.getInt() ? NumTasks : Grainsize
5389                         : NoSchedule),
5390      Data.Schedule.getPointer()
5391          ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5392                                      /*isSigned=*/false)
5393          : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5394      Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5395                             Result.TaskDupFn, CGF.VoidPtrTy)
5396                       : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5397  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs);
5398}
5399
5400/// Emit reduction operation for each element of array (required for
5401/// array sections) LHS op = RHS.
5402/// \param Type Type of array.
5403/// \param LHSVar Variable on the left side of the reduction operation
5404/// (references element of array in original variable).
5405/// \param RHSVar Variable on the right side of the reduction operation
5406/// (references element of array in original variable).
5407/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5408/// RHSVar.
5409static void EmitOMPAggregateReduction(
5410    CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5411    const VarDecl *RHSVar,
5412    const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5413                                  const Expr *, const Expr *)> &RedOpGen,
5414    const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5415    const Expr *UpExpr = nullptr) {
5416  // Perform element-by-element initialization.
5417  QualType ElementTy;
5418  Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5419  Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5420
5421  // Drill down to the base element type on both arrays.
5422  const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5423  llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5424
5425  llvm::Value *RHSBegin = RHSAddr.getPointer();
5426  llvm::Value *LHSBegin = LHSAddr.getPointer();
5427  // Cast from pointer to array type to pointer to single element.
5428  llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5429  // The basic structure here is a while-do loop.
5430  llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5431  llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5432  llvm::Value *IsEmpty =
5433      CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5434  CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5435
5436  // Enter the loop body, making that address the current address.
5437  llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5438  CGF.EmitBlock(BodyBB);
5439
5440  CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5441
5442  llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5443      RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5444  RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5445  Address RHSElementCurrent =
5446      Address(RHSElementPHI,
5447              RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5448
5449  llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5450      LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5451  LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5452  Address LHSElementCurrent =
5453      Address(LHSElementPHI,
5454              LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5455
5456  // Emit copy.
5457  CodeGenFunction::OMPPrivateScope Scope(CGF);
5458  Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5459  Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5460  Scope.Privatize();
5461  RedOpGen(CGF, XExpr, EExpr, UpExpr);
5462  Scope.ForceCleanup();
5463
5464  // Shift the address forward by one element.
5465  llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5466      LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5467  llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5468      RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5469  // Check whether we've reached the end.
5470  llvm::Value *Done =
5471      CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5472  CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5473  LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5474  RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5475
5476  // Done.
5477  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5478}
5479
5480/// Emit reduction combiner. If the combiner is a simple expression emit it as
5481/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5482/// UDR combiner function.
5483static void emitReductionCombiner(CodeGenFunction &CGF,
5484                                  const Expr *ReductionOp) {
5485  if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5486    if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5487      if (const auto *DRE =
5488              dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5489        if (const auto *DRD =
5490                dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5491          std::pair<llvm::Function *, llvm::Function *> Reduction =
5492              CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
5493          RValue Func = RValue::get(Reduction.first);
5494          CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5495          CGF.EmitIgnoredExpr(ReductionOp);
5496          return;
5497        }
5498  CGF.EmitIgnoredExpr(ReductionOp);
5499}
5500
5501llvm::Function *CGOpenMPRuntime::emitReductionFunction(
5502    SourceLocation Loc, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates,
5503    ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
5504    ArrayRef<const Expr *> ReductionOps) {
5505  ASTContext &C = CGM.getContext();
5506
5507  // void reduction_func(void *LHSArg, void *RHSArg);
5508  FunctionArgList Args;
5509  ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5510                           ImplicitParamDecl::Other);
5511  ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5512                           ImplicitParamDecl::Other);
5513  Args.push_back(&LHSArg);
5514  Args.push_back(&RHSArg);
5515  const auto &CGFI =
5516      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5517  std::string Name = getName({"omp", "reduction", "reduction_func"});
5518  auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5519                                    llvm::GlobalValue::InternalLinkage, Name,
5520                                    &CGM.getModule());
5521  CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5522  Fn->setDoesNotRecurse();
5523  CodeGenFunction CGF(CGM);
5524  CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5525
5526  // Dst = (void*[n])(LHSArg);
5527  // Src = (void*[n])(RHSArg);
5528  Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5529      CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
5530      ArgsType), CGF.getPointerAlign());
5531  Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5532      CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
5533      ArgsType), CGF.getPointerAlign());
5534
5535  //  ...
5536  //  *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5537  //  ...
5538  CodeGenFunction::OMPPrivateScope Scope(CGF);
5539  auto IPriv = Privates.begin();
5540  unsigned Idx = 0;
5541  for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5542    const auto *RHSVar =
5543        cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5544    Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5545      return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
5546    });
5547    const auto *LHSVar =
5548        cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5549    Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5550      return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
5551    });
5552    QualType PrivTy = (*IPriv)->getType();
5553    if (PrivTy->isVariablyModifiedType()) {
5554      // Get array size and emit VLA type.
5555      ++Idx;
5556      Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);
5557      llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
5558      const VariableArrayType *VLA =
5559          CGF.getContext().getAsVariableArrayType(PrivTy);
5560      const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
5561      CodeGenFunction::OpaqueValueMapping OpaqueMap(
5562          CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
5563      CGF.EmitVariablyModifiedType(PrivTy);
5564    }
5565  }
5566  Scope.Privatize();
5567  IPriv = Privates.begin();
5568  auto ILHS = LHSExprs.begin();
5569  auto IRHS = RHSExprs.begin();
5570  for (const Expr *E : ReductionOps) {
5571    if ((*IPriv)->getType()->isArrayType()) {
5572      // Emit reduction for array section.
5573      const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5574      const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5575      EmitOMPAggregateReduction(
5576          CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5577          [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5578            emitReductionCombiner(CGF, E);
5579          });
5580    } else {
5581      // Emit reduction for array subscript or single variable.
5582      emitReductionCombiner(CGF, E);
5583    }
5584    ++IPriv;
5585    ++ILHS;
5586    ++IRHS;
5587  }
5588  Scope.ForceCleanup();
5589  CGF.FinishFunction();
5590  return Fn;
5591}
5592
5593void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5594                                                  const Expr *ReductionOp,
5595                                                  const Expr *PrivateRef,
5596                                                  const DeclRefExpr *LHS,
5597                                                  const DeclRefExpr *RHS) {
5598  if (PrivateRef->getType()->isArrayType()) {
5599    // Emit reduction for array section.
5600    const auto *LHSVar = cast<VarDecl>(LHS->getDecl());
5601    const auto *RHSVar = cast<VarDecl>(RHS->getDecl());
5602    EmitOMPAggregateReduction(
5603        CGF, PrivateRef->getType(), LHSVar, RHSVar,
5604        [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5605          emitReductionCombiner(CGF, ReductionOp);
5606        });
5607  } else {
5608    // Emit reduction for array subscript or single variable.
5609    emitReductionCombiner(CGF, ReductionOp);
5610  }
5611}
5612
5613void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5614                                    ArrayRef<const Expr *> Privates,
5615                                    ArrayRef<const Expr *> LHSExprs,
5616                                    ArrayRef<const Expr *> RHSExprs,
5617                                    ArrayRef<const Expr *> ReductionOps,
5618                                    ReductionOptionsTy Options) {
5619  if (!CGF.HaveInsertPoint())
5620    return;
5621
5622  bool WithNowait = Options.WithNowait;
5623  bool SimpleReduction = Options.SimpleReduction;
5624
5625  // Next code should be emitted for reduction:
5626  //
5627  // static kmp_critical_name lock = { 0 };
5628  //
5629  // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5630  //  *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5631  //  ...
5632  //  *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5633  //  *(Type<n>-1*)rhs[<n>-1]);
5634  // }
5635  //
5636  // ...
5637  // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5638  // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5639  // RedList, reduce_func, &<lock>)) {
5640  // case 1:
5641  //  ...
5642  //  <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5643  //  ...
5644  // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5645  // break;
5646  // case 2:
5647  //  ...
5648  //  Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5649  //  ...
5650  // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5651  // break;
5652  // default:;
5653  // }
5654  //
5655  // if SimpleReduction is true, only the next code is generated:
5656  //  ...
5657  //  <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5658  //  ...
5659
5660  ASTContext &C = CGM.getContext();
5661
5662  if (SimpleReduction) {
5663    CodeGenFunction::RunCleanupsScope Scope(CGF);
5664    auto IPriv = Privates.begin();
5665    auto ILHS = LHSExprs.begin();
5666    auto IRHS = RHSExprs.begin();
5667    for (const Expr *E : ReductionOps) {
5668      emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5669                                  cast<DeclRefExpr>(*IRHS));
5670      ++IPriv;
5671      ++ILHS;
5672      ++IRHS;
5673    }
5674    return;
5675  }
5676
5677  // 1. Build a list of reduction variables.
5678  // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5679  auto Size = RHSExprs.size();
5680  for (const Expr *E : Privates) {
5681    if (E->getType()->isVariablyModifiedType())
5682      // Reserve place for array size.
5683      ++Size;
5684  }
5685  llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5686  QualType ReductionArrayTy =
5687      C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
5688                             /*IndexTypeQuals=*/0);
5689  Address ReductionList =
5690      CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
5691  auto IPriv = Privates.begin();
5692  unsigned Idx = 0;
5693  for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5694    Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5695    CGF.Builder.CreateStore(
5696        CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5697            CGF.EmitLValue(RHSExprs[I]).getPointer(), CGF.VoidPtrTy),
5698        Elem);
5699    if ((*IPriv)->getType()->isVariablyModifiedType()) {
5700      // Store array size.
5701      ++Idx;
5702      Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5703      llvm::Value *Size = CGF.Builder.CreateIntCast(
5704          CGF.getVLASize(
5705                 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
5706              .NumElts,
5707          CGF.SizeTy, /*isSigned=*/false);
5708      CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
5709                              Elem);
5710    }
5711  }
5712
5713  // 2. Emit reduce_func().
5714  llvm::Function *ReductionFn = emitReductionFunction(
5715      Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates,
5716      LHSExprs, RHSExprs, ReductionOps);
5717
5718  // 3. Create static kmp_critical_name lock = { 0 };
5719  std::string Name = getName({"reduction"});
5720  llvm::Value *Lock = getCriticalRegionLock(Name);
5721
5722  // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5723  // RedList, reduce_func, &<lock>);
5724  llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
5725  llvm::Value *ThreadId = getThreadID(CGF, Loc);
5726  llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
5727  llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5728      ReductionList.getPointer(), CGF.VoidPtrTy);
5729  llvm::Value *Args[] = {
5730      IdentTLoc,                             // ident_t *<loc>
5731      ThreadId,                              // i32 <gtid>
5732      CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
5733      ReductionArrayTySize,                  // size_type sizeof(RedList)
5734      RL,                                    // void *RedList
5735      ReductionFn, // void (*) (void *, void *) <reduce_func>
5736      Lock         // kmp_critical_name *&<lock>
5737  };
5738  llvm::Value *Res = CGF.EmitRuntimeCall(
5739      createRuntimeFunction(WithNowait ? OMPRTL__kmpc_reduce_nowait
5740                                       : OMPRTL__kmpc_reduce),
5741      Args);
5742
5743  // 5. Build switch(res)
5744  llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
5745  llvm::SwitchInst *SwInst =
5746      CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
5747
5748  // 6. Build case 1:
5749  //  ...
5750  //  <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5751  //  ...
5752  // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5753  // break;
5754  llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
5755  SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
5756  CGF.EmitBlock(Case1BB);
5757
5758  // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5759  llvm::Value *EndArgs[] = {
5760      IdentTLoc, // ident_t *<loc>
5761      ThreadId,  // i32 <gtid>
5762      Lock       // kmp_critical_name *&<lock>
5763  };
5764  auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5765                       CodeGenFunction &CGF, PrePostActionTy &Action) {
5766    CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5767    auto IPriv = Privates.begin();
5768    auto ILHS = LHSExprs.begin();
5769    auto IRHS = RHSExprs.begin();
5770    for (const Expr *E : ReductionOps) {
5771      RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5772                                     cast<DeclRefExpr>(*IRHS));
5773      ++IPriv;
5774      ++ILHS;
5775      ++IRHS;
5776    }
5777  };
5778  RegionCodeGenTy RCG(CodeGen);
5779  CommonActionTy Action(
5780      nullptr, llvm::None,
5781      createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait
5782                                       : OMPRTL__kmpc_end_reduce),
5783      EndArgs);
5784  RCG.setAction(Action);
5785  RCG(CGF);
5786
5787  CGF.EmitBranch(DefaultBB);
5788
5789  // 7. Build case 2:
5790  //  ...
5791  //  Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5792  //  ...
5793  // break;
5794  llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
5795  SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5796  CGF.EmitBlock(Case2BB);
5797
5798  auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5799                             CodeGenFunction &CGF, PrePostActionTy &Action) {
5800    auto ILHS = LHSExprs.begin();
5801    auto IRHS = RHSExprs.begin();
5802    auto IPriv = Privates.begin();
5803    for (const Expr *E : ReductionOps) {
5804      const Expr *XExpr = nullptr;
5805      const Expr *EExpr = nullptr;
5806      const Expr *UpExpr = nullptr;
5807      BinaryOperatorKind BO = BO_Comma;
5808      if (const auto *BO = dyn_cast<BinaryOperator>(E)) {
5809        if (BO->getOpcode() == BO_Assign) {
5810          XExpr = BO->getLHS();
5811          UpExpr = BO->getRHS();
5812        }
5813      }
5814      // Try to emit update expression as a simple atomic.
5815      const Expr *RHSExpr = UpExpr;
5816      if (RHSExpr) {
5817        // Analyze RHS part of the whole expression.
5818        if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5819                RHSExpr->IgnoreParenImpCasts())) {
5820          // If this is a conditional operator, analyze its condition for
5821          // min/max reduction operator.
5822          RHSExpr = ACO->getCond();
5823        }
5824        if (const auto *BORHS =
5825                dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
5826          EExpr = BORHS->getRHS();
5827          BO = BORHS->getOpcode();
5828        }
5829      }
5830      if (XExpr) {
5831        const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5832        auto &&AtomicRedGen = [BO, VD,
5833                               Loc](CodeGenFunction &CGF, const Expr *XExpr,
5834                                    const Expr *EExpr, const Expr *UpExpr) {
5835          LValue X = CGF.EmitLValue(XExpr);
5836          RValue E;
5837          if (EExpr)
5838            E = CGF.EmitAnyExpr(EExpr);
5839          CGF.EmitOMPAtomicSimpleUpdateExpr(
5840              X, E, BO, /*IsXLHSInRHSPart=*/true,
5841              llvm::AtomicOrdering::Monotonic, Loc,
5842              [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5843                CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5844                PrivateScope.addPrivate(
5845                    VD, [&CGF, VD, XRValue, Loc]() {
5846                      Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5847                      CGF.emitOMPSimpleStore(
5848                          CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
5849                          VD->getType().getNonReferenceType(), Loc);
5850                      return LHSTemp;
5851                    });
5852                (void)PrivateScope.Privatize();
5853                return CGF.EmitAnyExpr(UpExpr);
5854              });
5855        };
5856        if ((*IPriv)->getType()->isArrayType()) {
5857          // Emit atomic reduction for array section.
5858          const auto *RHSVar =
5859              cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5860          EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
5861                                    AtomicRedGen, XExpr, EExpr, UpExpr);
5862        } else {
5863          // Emit atomic reduction for array subscript or single variable.
5864          AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5865        }
5866      } else {
5867        // Emit as a critical region.
5868        auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5869                                           const Expr *, const Expr *) {
5870          CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5871          std::string Name = RT.getName({"atomic_reduction"});
5872          RT.emitCriticalRegion(
5873              CGF, Name,
5874              [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5875                Action.Enter(CGF);
5876                emitReductionCombiner(CGF, E);
5877              },
5878              Loc);
5879        };
5880        if ((*IPriv)->getType()->isArrayType()) {
5881          const auto *LHSVar =
5882              cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5883          const auto *RHSVar =
5884              cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5885          EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5886                                    CritRedGen);
5887        } else {
5888          CritRedGen(CGF, nullptr, nullptr, nullptr);
5889        }
5890      }
5891      ++ILHS;
5892      ++IRHS;
5893      ++IPriv;
5894    }
5895  };
5896  RegionCodeGenTy AtomicRCG(AtomicCodeGen);
5897  if (!WithNowait) {
5898    // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
5899    llvm::Value *EndArgs[] = {
5900        IdentTLoc, // ident_t *<loc>
5901        ThreadId,  // i32 <gtid>
5902        Lock       // kmp_critical_name *&<lock>
5903    };
5904    CommonActionTy Action(nullptr, llvm::None,
5905                          createRuntimeFunction(OMPRTL__kmpc_end_reduce),
5906                          EndArgs);
5907    AtomicRCG.setAction(Action);
5908    AtomicRCG(CGF);
5909  } else {
5910    AtomicRCG(CGF);
5911  }
5912
5913  CGF.EmitBranch(DefaultBB);
5914  CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
5915}
5916
5917/// Generates unique name for artificial threadprivate variables.
5918/// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"
5919static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,
5920                                      const Expr *Ref) {
5921  SmallString<256> Buffer;
5922  llvm::raw_svector_ostream Out(Buffer);
5923  const clang::DeclRefExpr *DE;
5924  const VarDecl *D = ::getBaseDecl(Ref, DE);
5925  if (!D)
5926    D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());
5927  D = D->getCanonicalDecl();
5928  std::string Name = CGM.getOpenMPRuntime().getName(
5929      {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(D)});
5930  Out << Prefix << Name << "_"
5931      << D->getCanonicalDecl()->getBeginLoc().getRawEncoding();
5932  return Out.str();
5933}
5934
5935/// Emits reduction initializer function:
5936/// \code
5937/// void @.red_init(void* %arg) {
5938/// %0 = bitcast void* %arg to <type>*
5939/// store <type> <init>, <type>* %0
5940/// ret void
5941/// }
5942/// \endcode
5943static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
5944                                           SourceLocation Loc,
5945                                           ReductionCodeGen &RCG, unsigned N) {
5946  ASTContext &C = CGM.getContext();
5947  FunctionArgList Args;
5948  ImplicitParamDecl Param(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5949                          ImplicitParamDecl::Other);
5950  Args.emplace_back(&Param);
5951  const auto &FnInfo =
5952      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5953  llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
5954  std::string Name = CGM.getOpenMPRuntime().getName({"red_init", ""});
5955  auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5956                                    Name, &CGM.getModule());
5957  CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
5958  Fn->setDoesNotRecurse();
5959  CodeGenFunction CGF(CGM);
5960  CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
5961  Address PrivateAddr = CGF.EmitLoadOfPointer(
5962      CGF.GetAddrOfLocalVar(&Param),
5963      C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
5964  llvm::Value *Size = nullptr;
5965  // If the size of the reduction item is non-constant, load it from global
5966  // threadprivate variable.
5967  if (RCG.getSizes(N).second) {
5968    Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5969        CGF, CGM.getContext().getSizeType(),
5970        generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
5971    Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
5972                                CGM.getContext().getSizeType(), Loc);
5973  }
5974  RCG.emitAggregateType(CGF, N, Size);
5975  LValue SharedLVal;
5976  // If initializer uses initializer from declare reduction construct, emit a
5977  // pointer to the address of the original reduction item (reuired by reduction
5978  // initializer)
5979  if (RCG.usesReductionInitializer(N)) {
5980    Address SharedAddr =
5981        CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5982            CGF, CGM.getContext().VoidPtrTy,
5983            generateUniqueName(CGM, "reduction", RCG.getRefExpr(N)));
5984    SharedAddr = CGF.EmitLoadOfPointer(
5985        SharedAddr,
5986        CGM.getContext().VoidPtrTy.castAs<PointerType>()->getTypePtr());
5987    SharedLVal = CGF.MakeAddrLValue(SharedAddr, CGM.getContext().VoidPtrTy);
5988  } else {
5989    SharedLVal = CGF.MakeNaturalAlignAddrLValue(
5990        llvm::ConstantPointerNull::get(CGM.VoidPtrTy),
5991        CGM.getContext().VoidPtrTy);
5992  }
5993  // Emit the initializer:
5994  // %0 = bitcast void* %arg to <type>*
5995  // store <type> <init>, <type>* %0
5996  RCG.emitInitialization(CGF, N, PrivateAddr, SharedLVal,
5997                         [](CodeGenFunction &) { return false; });
5998  CGF.FinishFunction();
5999  return Fn;
6000}
6001
6002/// Emits reduction combiner function:
6003/// \code
6004/// void @.red_comb(void* %arg0, void* %arg1) {
6005/// %lhs = bitcast void* %arg0 to <type>*
6006/// %rhs = bitcast void* %arg1 to <type>*
6007/// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs)
6008/// store <type> %2, <type>* %lhs
6009/// ret void
6010/// }
6011/// \endcode
6012static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM,
6013                                           SourceLocation Loc,
6014                                           ReductionCodeGen &RCG, unsigned N,
6015                                           const Expr *ReductionOp,
6016                                           const Expr *LHS, const Expr *RHS,
6017                                           const Expr *PrivateRef) {
6018  ASTContext &C = CGM.getContext();
6019  const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());
6020  const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());
6021  FunctionArgList Args;
6022  ImplicitParamDecl ParamInOut(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
6023                               C.VoidPtrTy, ImplicitParamDecl::Other);
6024  ImplicitParamDecl ParamIn(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
6025                            ImplicitParamDecl::Other);
6026  Args.emplace_back(&ParamInOut);
6027  Args.emplace_back(&ParamIn);
6028  const auto &FnInfo =
6029      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
6030  llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
6031  std::string Name = CGM.getOpenMPRuntime().getName({"red_comb", ""});
6032  auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
6033                                    Name, &CGM.getModule());
6034  CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
6035  Fn->setDoesNotRecurse();
6036  CodeGenFunction CGF(CGM);
6037  CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
6038  llvm::Value *Size = nullptr;
6039  // If the size of the reduction item is non-constant, load it from global
6040  // threadprivate variable.
6041  if (RCG.getSizes(N).second) {
6042    Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
6043        CGF, CGM.getContext().getSizeType(),
6044        generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6045    Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
6046                                CGM.getContext().getSizeType(), Loc);
6047  }
6048  RCG.emitAggregateType(CGF, N, Size);
6049  // Remap lhs and rhs variables to the addresses of the function arguments.
6050  // %lhs = bitcast void* %arg0 to <type>*
6051  // %rhs = bitcast void* %arg1 to <type>*
6052  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6053  PrivateScope.addPrivate(LHSVD, [&C, &CGF, &ParamInOut, LHSVD]() {
6054    // Pull out the pointer to the variable.
6055    Address PtrAddr = CGF.EmitLoadOfPointer(
6056        CGF.GetAddrOfLocalVar(&ParamInOut),
6057        C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
6058    return CGF.Builder.CreateElementBitCast(
6059        PtrAddr, CGF.ConvertTypeForMem(LHSVD->getType()));
6060  });
6061  PrivateScope.addPrivate(RHSVD, [&C, &CGF, &ParamIn, RHSVD]() {
6062    // Pull out the pointer to the variable.
6063    Address PtrAddr = CGF.EmitLoadOfPointer(
6064        CGF.GetAddrOfLocalVar(&ParamIn),
6065        C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
6066    return CGF.Builder.CreateElementBitCast(
6067        PtrAddr, CGF.ConvertTypeForMem(RHSVD->getType()));
6068  });
6069  PrivateScope.Privatize();
6070  // Emit the combiner body:
6071  // %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs)
6072  // store <type> %2, <type>* %lhs
6073  CGM.getOpenMPRuntime().emitSingleReductionCombiner(
6074      CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
6075      cast<DeclRefExpr>(RHS));
6076  CGF.FinishFunction();
6077  return Fn;
6078}
6079
6080/// Emits reduction finalizer function:
6081/// \code
6082/// void @.red_fini(void* %arg) {
6083/// %0 = bitcast void* %arg to <type>*
6084/// <destroy>(<type>* %0)
6085/// ret void
6086/// }
6087/// \endcode
6088static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM,
6089                                           SourceLocation Loc,
6090                                           ReductionCodeGen &RCG, unsigned N) {
6091  if (!RCG.needCleanups(N))
6092    return nullptr;
6093  ASTContext &C = CGM.getContext();
6094  FunctionArgList Args;
6095  ImplicitParamDecl Param(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
6096                          ImplicitParamDecl::Other);
6097  Args.emplace_back(&Param);
6098  const auto &FnInfo =
6099      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
6100  llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
6101  std::string Name = CGM.getOpenMPRuntime().getName({"red_fini", ""});
6102  auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
6103                                    Name, &CGM.getModule());
6104  CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
6105  Fn->setDoesNotRecurse();
6106  CodeGenFunction CGF(CGM);
6107  CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
6108  Address PrivateAddr = CGF.EmitLoadOfPointer(
6109      CGF.GetAddrOfLocalVar(&Param),
6110      C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
6111  llvm::Value *Size = nullptr;
6112  // If the size of the reduction item is non-constant, load it from global
6113  // threadprivate variable.
6114  if (RCG.getSizes(N).second) {
6115    Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
6116        CGF, CGM.getContext().getSizeType(),
6117        generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6118    Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
6119                                CGM.getContext().getSizeType(), Loc);
6120  }
6121  RCG.emitAggregateType(CGF, N, Size);
6122  // Emit the finalizer body:
6123  // <destroy>(<type>* %0)
6124  RCG.emitCleanups(CGF, N, PrivateAddr);
6125  CGF.FinishFunction();
6126  return Fn;
6127}
6128
6129llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
6130    CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
6131    ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
6132  if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty())
6133    return nullptr;
6134
6135  // Build typedef struct:
6136  // kmp_task_red_input {
6137  //   void *reduce_shar; // shared reduction item
6138  //   size_t reduce_size; // size of data item
6139  //   void *reduce_init; // data initialization routine
6140  //   void *reduce_fini; // data finalization routine
6141  //   void *reduce_comb; // data combiner routine
6142  //   kmp_task_red_flags_t flags; // flags for additional info from compiler
6143  // } kmp_task_red_input_t;
6144  ASTContext &C = CGM.getContext();
6145  RecordDecl *RD = C.buildImplicitRecord("kmp_task_red_input_t");
6146  RD->startDefinition();
6147  const FieldDecl *SharedFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6148  const FieldDecl *SizeFD = addFieldToRecordDecl(C, RD, C.getSizeType());
6149  const FieldDecl *InitFD  = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6150  const FieldDecl *FiniFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6151  const FieldDecl *CombFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6152  const FieldDecl *FlagsFD = addFieldToRecordDecl(
6153      C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false));
6154  RD->completeDefinition();
6155  QualType RDType = C.getRecordType(RD);
6156  unsigned Size = Data.ReductionVars.size();
6157  llvm::APInt ArraySize(/*numBits=*/64, Size);
6158  QualType ArrayRDType = C.getConstantArrayType(
6159      RDType, ArraySize, ArrayType::Normal, /*IndexTypeQuals=*/0);
6160  // kmp_task_red_input_t .rd_input.[Size];
6161  Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input.");
6162  ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionCopies,
6163                       Data.ReductionOps);
6164  for (unsigned Cnt = 0; Cnt < Size; ++Cnt) {
6165    // kmp_task_red_input_t &ElemLVal = .rd_input.[Cnt];
6166    llvm::Value *Idxs[] = {llvm::ConstantInt::get(CGM.SizeTy, /*V=*/0),
6167                           llvm::ConstantInt::get(CGM.SizeTy, Cnt)};
6168    llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(
6169        TaskRedInput.getPointer(), Idxs,
6170        /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,
6171        ".rd_input.gep.");
6172    LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType);
6173    // ElemLVal.reduce_shar = &Shareds[Cnt];
6174    LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD);
6175    RCG.emitSharedLValue(CGF, Cnt);
6176    llvm::Value *CastedShared =
6177        CGF.EmitCastToVoidPtr(RCG.getSharedLValue(Cnt).getPointer());
6178    CGF.EmitStoreOfScalar(CastedShared, SharedLVal);
6179    RCG.emitAggregateType(CGF, Cnt);
6180    llvm::Value *SizeValInChars;
6181    llvm::Value *SizeVal;
6182    std::tie(SizeValInChars, SizeVal) = RCG.getSizes(Cnt);
6183    // We use delayed creation/initialization for VLAs, array sections and
6184    // custom reduction initializations. It is required because runtime does not
6185    // provide the way to pass the sizes of VLAs/array sections to
6186    // initializer/combiner/finalizer functions and does not pass the pointer to
6187    // original reduction item to the initializer. Instead threadprivate global
6188    // variables are used to store these values and use them in the functions.
6189    bool DelayedCreation = !!SizeVal;
6190    SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy,
6191                                               /*isSigned=*/false);
6192    LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD);
6193    CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal);
6194    // ElemLVal.reduce_init = init;
6195    LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD);
6196    llvm::Value *InitAddr =
6197        CGF.EmitCastToVoidPtr(emitReduceInitFunction(CGM, Loc, RCG, Cnt));
6198    CGF.EmitStoreOfScalar(InitAddr, InitLVal);
6199    DelayedCreation = DelayedCreation || RCG.usesReductionInitializer(Cnt);
6200    // ElemLVal.reduce_fini = fini;
6201    LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD);
6202    llvm::Value *Fini = emitReduceFiniFunction(CGM, Loc, RCG, Cnt);
6203    llvm::Value *FiniAddr = Fini
6204                                ? CGF.EmitCastToVoidPtr(Fini)
6205                                : llvm::ConstantPointerNull::get(CGM.VoidPtrTy);
6206    CGF.EmitStoreOfScalar(FiniAddr, FiniLVal);
6207    // ElemLVal.reduce_comb = comb;
6208    LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD);
6209    llvm::Value *CombAddr = CGF.EmitCastToVoidPtr(emitReduceCombFunction(
6210        CGM, Loc, RCG, Cnt, Data.ReductionOps[Cnt], LHSExprs[Cnt],
6211        RHSExprs[Cnt], Data.ReductionCopies[Cnt]));
6212    CGF.EmitStoreOfScalar(CombAddr, CombLVal);
6213    // ElemLVal.flags = 0;
6214    LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD);
6215    if (DelayedCreation) {
6216      CGF.EmitStoreOfScalar(
6217          llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/1, /*IsSigned=*/true),
6218          FlagsLVal);
6219    } else
6220      CGF.EmitNullInitialization(FlagsLVal.getAddress(), FlagsLVal.getType());
6221  }
6222  // Build call void *__kmpc_task_reduction_init(int gtid, int num_data, void
6223  // *data);
6224  llvm::Value *Args[] = {
6225      CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy,
6226                                /*isSigned=*/true),
6227      llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true),
6228      CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(),
6229                                                      CGM.VoidPtrTy)};
6230  return CGF.EmitRuntimeCall(
6231      createRuntimeFunction(OMPRTL__kmpc_task_reduction_init), Args);
6232}
6233
6234void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
6235                                              SourceLocation Loc,
6236                                              ReductionCodeGen &RCG,
6237                                              unsigned N) {
6238  auto Sizes = RCG.getSizes(N);
6239  // Emit threadprivate global variable if the type is non-constant
6240  // (Sizes.second = nullptr).
6241  if (Sizes.second) {
6242    llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy,
6243                                                     /*isSigned=*/false);
6244    Address SizeAddr = getAddrOfArtificialThreadPrivate(
6245        CGF, CGM.getContext().getSizeType(),
6246        generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6247    CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false);
6248  }
6249  // Store address of the original reduction item if custom initializer is used.
6250  if (RCG.usesReductionInitializer(N)) {
6251    Address SharedAddr = getAddrOfArtificialThreadPrivate(
6252        CGF, CGM.getContext().VoidPtrTy,
6253        generateUniqueName(CGM, "reduction", RCG.getRefExpr(N)));
6254    CGF.Builder.CreateStore(
6255        CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6256            RCG.getSharedLValue(N).getPointer(), CGM.VoidPtrTy),
6257        SharedAddr, /*IsVolatile=*/false);
6258  }
6259}
6260
6261Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
6262                                              SourceLocation Loc,
6263                                              llvm::Value *ReductionsPtr,
6264                                              LValue SharedLVal) {
6265  // Build call void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
6266  // *d);
6267  llvm::Value *Args[] = {
6268      CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy,
6269                                /*isSigned=*/true),
6270      ReductionsPtr,
6271      CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(SharedLVal.getPointer(),
6272                                                      CGM.VoidPtrTy)};
6273  return Address(
6274      CGF.EmitRuntimeCall(
6275          createRuntimeFunction(OMPRTL__kmpc_task_reduction_get_th_data), Args),
6276      SharedLVal.getAlignment());
6277}
6278
6279void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
6280                                       SourceLocation Loc) {
6281  if (!CGF.HaveInsertPoint())
6282    return;
6283  // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
6284  // global_tid);
6285  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
6286  // Ignore return result until untied tasks are supported.
6287  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskwait), Args);
6288  if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
6289    Region->emitUntiedSwitch(CGF);
6290}
6291
6292void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
6293                                           OpenMPDirectiveKind InnerKind,
6294                                           const RegionCodeGenTy &CodeGen,
6295                                           bool HasCancel) {
6296  if (!CGF.HaveInsertPoint())
6297    return;
6298  InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel);
6299  CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
6300}
6301
6302namespace {
6303enum RTCancelKind {
6304  CancelNoreq = 0,
6305  CancelParallel = 1,
6306  CancelLoop = 2,
6307  CancelSections = 3,
6308  CancelTaskgroup = 4
6309};
6310} // anonymous namespace
6311
6312static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) {
6313  RTCancelKind CancelKind = CancelNoreq;
6314  if (CancelRegion == OMPD_parallel)
6315    CancelKind = CancelParallel;
6316  else if (CancelRegion == OMPD_for)
6317    CancelKind = CancelLoop;
6318  else if (CancelRegion == OMPD_sections)
6319    CancelKind = CancelSections;
6320  else {
6321    assert(CancelRegion == OMPD_taskgroup);
6322    CancelKind = CancelTaskgroup;
6323  }
6324  return CancelKind;
6325}
6326
6327void CGOpenMPRuntime::emitCancellationPointCall(
6328    CodeGenFunction &CGF, SourceLocation Loc,
6329    OpenMPDirectiveKind CancelRegion) {
6330  if (!CGF.HaveInsertPoint())
6331    return;
6332  // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
6333  // global_tid, kmp_int32 cncl_kind);
6334  if (auto *OMPRegionInfo =
6335          dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
6336    // For 'cancellation point taskgroup', the task region info may not have a
6337    // cancel. This may instead happen in another adjacent task.
6338    if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
6339      llvm::Value *Args[] = {
6340          emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
6341          CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
6342      // Ignore return result until untied tasks are supported.
6343      llvm::Value *Result = CGF.EmitRuntimeCall(
6344          createRuntimeFunction(OMPRTL__kmpc_cancellationpoint), Args);
6345      // if (__kmpc_cancellationpoint()) {
6346      //   exit from construct;
6347      // }
6348      llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
6349      llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
6350      llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
6351      CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6352      CGF.EmitBlock(ExitBB);
6353      // exit from construct;
6354      CodeGenFunction::JumpDest CancelDest =
6355          CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6356      CGF.EmitBranchThroughCleanup(CancelDest);
6357      CGF.EmitBlock(ContBB, /*IsFinished=*/true);
6358    }
6359  }
6360}
6361
6362void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
6363                                     const Expr *IfCond,
6364                                     OpenMPDirectiveKind CancelRegion) {
6365  if (!CGF.HaveInsertPoint())
6366    return;
6367  // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
6368  // kmp_int32 cncl_kind);
6369  if (auto *OMPRegionInfo =
6370          dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
6371    auto &&ThenGen = [Loc, CancelRegion, OMPRegionInfo](CodeGenFunction &CGF,
6372                                                        PrePostActionTy &) {
6373      CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
6374      llvm::Value *Args[] = {
6375          RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
6376          CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
6377      // Ignore return result until untied tasks are supported.
6378      llvm::Value *Result = CGF.EmitRuntimeCall(
6379          RT.createRuntimeFunction(OMPRTL__kmpc_cancel), Args);
6380      // if (__kmpc_cancel()) {
6381      //   exit from construct;
6382      // }
6383      llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
6384      llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
6385      llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
6386      CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6387      CGF.EmitBlock(ExitBB);
6388      // exit from construct;
6389      CodeGenFunction::JumpDest CancelDest =
6390          CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6391      CGF.EmitBranchThroughCleanup(CancelDest);
6392      CGF.EmitBlock(ContBB, /*IsFinished=*/true);
6393    };
6394    if (IfCond) {
6395      emitOMPIfClause(CGF, IfCond, ThenGen,
6396                      [](CodeGenFunction &, PrePostActionTy &) {});
6397    } else {
6398      RegionCodeGenTy ThenRCG(ThenGen);
6399      ThenRCG(CGF);
6400    }
6401  }
6402}
6403
6404void CGOpenMPRuntime::emitTargetOutlinedFunction(
6405    const OMPExecutableDirective &D, StringRef ParentName,
6406    llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6407    bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6408   (0) . __assert_fail ("!ParentName.empty() && \"Invalid target region parent name!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 6408, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!ParentName.empty() && "Invalid target region parent name!");
6409  emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
6410                                   IsOffloadEntry, CodeGen);
6411}
6412
6413void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
6414    const OMPExecutableDirective &D, StringRef ParentName,
6415    llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6416    bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6417  // Create a unique name for the entry function using the source location
6418  // information of the current target region. The name will be something like:
6419  //
6420  // __omp_offloading_DD_FFFF_PP_lBB
6421  //
6422  // where DD_FFFF is an ID unique to the file (device and file IDs), PP is the
6423  // mangled name of the function that encloses the target region and BB is the
6424  // line number of the target region.
6425
6426  unsigned DeviceID;
6427  unsigned FileID;
6428  unsigned Line;
6429  getTargetEntryUniqueInfo(CGM.getContext(), D.getBeginLoc(), DeviceID, FileID,
6430                           Line);
6431  SmallString<64> EntryFnName;
6432  {
6433    llvm::raw_svector_ostream OS(EntryFnName);
6434    OS << "__omp_offloading" << llvm::format("_%x", DeviceID)
6435       << llvm::format("_%x_", FileID) << ParentName << "_l" << Line;
6436  }
6437
6438  const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
6439
6440  CodeGenFunction CGF(CGM, true);
6441  CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
6442  CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6443
6444  OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS);
6445
6446  // If this target outline function is not an offload entry, we don't need to
6447  // register it.
6448  if (!IsOffloadEntry)
6449    return;
6450
6451  // The target region ID is used by the runtime library to identify the current
6452  // target region, so it only has to be unique and not necessarily point to
6453  // anything. It could be the pointer to the outlined function that implements
6454  // the target region, but we aren't using that so that the compiler doesn't
6455  // need to keep that, and could therefore inline the host function if proven
6456  // worthwhile during optimization. In the other hand, if emitting code for the
6457  // device, the ID has to be the function address so that it can retrieved from
6458  // the offloading entry and launched by the runtime library. We also mark the
6459  // outlined function to have external linkage in case we are emitting code for
6460  // the device, because these functions will be entry points to the device.
6461
6462  if (CGM.getLangOpts().OpenMPIsDevice) {
6463    OutlinedFnID = llvm::ConstantExpr::getBitCast(OutlinedFn, CGM.Int8PtrTy);
6464    OutlinedFn->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
6465    OutlinedFn->setDSOLocal(false);
6466  } else {
6467    std::string Name = getName({EntryFnName, "region_id"});
6468    OutlinedFnID = new llvm::GlobalVariable(
6469        CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
6470        llvm::GlobalValue::WeakAnyLinkage,
6471        llvm::Constant::getNullValue(CGM.Int8Ty), Name);
6472  }
6473
6474  // Register the information for the entry associated with this target region.
6475  OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
6476      DeviceID, FileID, ParentName, Line, OutlinedFn, OutlinedFnID,
6477      OffloadEntriesInfoManagerTy::OMPTargetRegionEntryTargetRegion);
6478}
6479
6480/// discard all CompoundStmts intervening between two constructs
6481static const Stmt *ignoreCompoundStmts(const Stmt *Body) {
6482  while (const auto *CS = dyn_cast_or_null<CompoundStmt>(Body))
6483    Body = CS->body_front();
6484
6485  return Body;
6486}
6487
6488/// Emit the number of teams for a target directive.  Inspect the num_teams
6489/// clause associated with a teams construct combined or closely nested
6490/// with the target directive.
6491///
6492/// Emit a team of size one for directives such as 'target parallel' that
6493/// have no associated teams construct.
6494///
6495/// Otherwise, return nullptr.
6496static llvm::Value *
6497emitNumTeamsForTargetDirective(CGOpenMPRuntime &OMPRuntime,
6498                               CodeGenFunction &CGF,
6499                               const OMPExecutableDirective &D) {
6500   (0) . __assert_fail ("!CGF.getLangOpts().OpenMPIsDevice && \"Clauses associated with the \" \"teams directive expected to be \" \"emitted only for the host!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 6502, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the "
6501 (0) . __assert_fail ("!CGF.getLangOpts().OpenMPIsDevice && \"Clauses associated with the \" \"teams directive expected to be \" \"emitted only for the host!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 6502, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                                              "teams directive expected to be "
6502 (0) . __assert_fail ("!CGF.getLangOpts().OpenMPIsDevice && \"Clauses associated with the \" \"teams directive expected to be \" \"emitted only for the host!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 6502, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                                              "emitted only for the host!");
6503
6504  CGBuilderTy &Bld = CGF.Builder;
6505
6506  // If the target directive is combined with a teams directive:
6507  //   Return the value in the num_teams clause, if any.
6508  //   Otherwise, return 0 to denote the runtime default.
6509  if (isOpenMPTeamsDirective(D.getDirectiveKind())) {
6510    if (const auto *NumTeamsClause = D.getSingleClause<OMPNumTeamsClause>()) {
6511      CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
6512      llvm::Value *NumTeams = CGF.EmitScalarExpr(NumTeamsClause->getNumTeams(),
6513                                                 /*IgnoreResultAssign*/ true);
6514      return Bld.CreateIntCast(NumTeams, CGF.Int32Ty,
6515                               /*IsSigned=*/true);
6516    }
6517
6518    // The default value is 0.
6519    return Bld.getInt32(0);
6520  }
6521
6522  // If the target directive is combined with a parallel directive but not a
6523  // teams directive, start one team.
6524  if (isOpenMPParallelDirective(D.getDirectiveKind()))
6525    return Bld.getInt32(1);
6526
6527  // If the current target region has a teams region enclosed, we need to get
6528  // the number of teams to pass to the runtime function call. This is done
6529  // by generating the expression in a inlined region. This is required because
6530  // the expression is captured in the enclosing target environment when the
6531  // teams directive is not combined with target.
6532
6533  const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
6534
6535  if (const auto *TeamsDir = dyn_cast_or_null<OMPExecutableDirective>(
6536          ignoreCompoundStmts(CS.getCapturedStmt()))) {
6537    if (isOpenMPTeamsDirective(TeamsDir->getDirectiveKind())) {
6538      if (const auto *NTE = TeamsDir->getSingleClause<OMPNumTeamsClause>()) {
6539        CGOpenMPInnerExprInfo CGInfo(CGF, CS);
6540        CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6541        llvm::Value *NumTeams = CGF.EmitScalarExpr(NTE->getNumTeams());
6542        return Bld.CreateIntCast(NumTeams, CGF.Int32Ty,
6543                                 /*IsSigned=*/true);
6544      }
6545
6546      // If we have an enclosed teams directive but no num_teams clause we use
6547      // the default value 0.
6548      return Bld.getInt32(0);
6549    }
6550  }
6551
6552  // No teams associated with the directive.
6553  return nullptr;
6554}
6555
6556/// Emit the number of threads for a target directive.  Inspect the
6557/// thread_limit clause associated with a teams construct combined or closely
6558/// nested with the target directive.
6559///
6560/// Emit the num_threads clause for directives such as 'target parallel' that
6561/// have no associated teams construct.
6562///
6563/// Otherwise, return nullptr.
6564static llvm::Value *
6565emitNumThreadsForTargetDirective(CGOpenMPRuntime &OMPRuntime,
6566                                 CodeGenFunction &CGF,
6567                                 const OMPExecutableDirective &D) {
6568   (0) . __assert_fail ("!CGF.getLangOpts().OpenMPIsDevice && \"Clauses associated with the \" \"teams directive expected to be \" \"emitted only for the host!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 6570, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the "
6569 (0) . __assert_fail ("!CGF.getLangOpts().OpenMPIsDevice && \"Clauses associated with the \" \"teams directive expected to be \" \"emitted only for the host!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 6570, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                                              "teams directive expected to be "
6570 (0) . __assert_fail ("!CGF.getLangOpts().OpenMPIsDevice && \"Clauses associated with the \" \"teams directive expected to be \" \"emitted only for the host!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 6570, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                                              "emitted only for the host!");
6571
6572  CGBuilderTy &Bld = CGF.Builder;
6573
6574  //
6575  // If the target directive is combined with a teams directive:
6576  //   Return the value in the thread_limit clause, if any.
6577  //
6578  // If the target directive is combined with a parallel directive:
6579  //   Return the value in the num_threads clause, if any.
6580  //
6581  // If both clauses are set, select the minimum of the two.
6582  //
6583  // If neither teams or parallel combined directives set the number of threads
6584  // in a team, return 0 to denote the runtime default.
6585  //
6586  // If this is not a teams directive return nullptr.
6587
6588  if (isOpenMPTeamsDirective(D.getDirectiveKind()) ||
6589      isOpenMPParallelDirective(D.getDirectiveKind())) {
6590    llvm::Value *DefaultThreadLimitVal = Bld.getInt32(0);
6591    llvm::Value *NumThreadsVal = nullptr;
6592    llvm::Value *ThreadLimitVal = nullptr;
6593
6594    if (const auto *ThreadLimitClause =
6595            D.getSingleClause<OMPThreadLimitClause>()) {
6596      CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6597      llvm::Value *ThreadLimit =
6598          CGF.EmitScalarExpr(ThreadLimitClause->getThreadLimit(),
6599                             /*IgnoreResultAssign*/ true);
6600      ThreadLimitVal = Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty,
6601                                         /*IsSigned=*/true);
6602    }
6603
6604    if (const auto *NumThreadsClause =
6605            D.getSingleClause<OMPNumThreadsClause>()) {
6606      CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
6607      llvm::Value *NumThreads =
6608          CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
6609                             /*IgnoreResultAssign*/ true);
6610      NumThreadsVal =
6611          Bld.CreateIntCast(NumThreads, CGF.Int32Ty, /*IsSigned=*/true);
6612    }
6613
6614    // Select the lesser of thread_limit and num_threads.
6615    if (NumThreadsVal)
6616      ThreadLimitVal = ThreadLimitVal
6617                           ? Bld.CreateSelect(Bld.CreateICmpSLT(NumThreadsVal,
6618                                                                ThreadLimitVal),
6619                                              NumThreadsVal, ThreadLimitVal)
6620                           : NumThreadsVal;
6621
6622    // Set default value passed to the runtime if either teams or a target
6623    // parallel type directive is found but no clause is specified.
6624    if (!ThreadLimitVal)
6625      ThreadLimitVal = DefaultThreadLimitVal;
6626
6627    return ThreadLimitVal;
6628  }
6629
6630  // If the current target region has a teams region enclosed, we need to get
6631  // the thread limit to pass to the runtime function call. This is done
6632  // by generating the expression in a inlined region. This is required because
6633  // the expression is captured in the enclosing target environment when the
6634  // teams directive is not combined with target.
6635
6636  const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
6637
6638  if (const auto *TeamsDir = dyn_cast_or_null<OMPExecutableDirective>(
6639          ignoreCompoundStmts(CS.getCapturedStmt()))) {
6640    if (isOpenMPTeamsDirective(TeamsDir->getDirectiveKind())) {
6641      if (const auto *TLE = TeamsDir->getSingleClause<OMPThreadLimitClause>()) {
6642        CGOpenMPInnerExprInfo CGInfo(CGF, CS);
6643        CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6644        llvm::Value *ThreadLimit = CGF.EmitScalarExpr(TLE->getThreadLimit());
6645        return CGF.Builder.CreateIntCast(ThreadLimit, CGF.Int32Ty,
6646                                         /*IsSigned=*/true);
6647      }
6648
6649      // If we have an enclosed teams directive but no thread_limit clause we
6650      // use the default value 0.
6651      return CGF.Builder.getInt32(0);
6652    }
6653  }
6654
6655  // No teams associated with the directive.
6656  return nullptr;
6657}
6658
6659namespace {
6660LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
6661
6662// Utility to handle information from clauses associated with a given
6663// construct that use mappable expressions (e.g. 'map' clause, 'to' clause).
6664// It provides a convenient interface to obtain the information and generate
6665// code for that information.
6666class MappableExprsHandler {
6667public:
6668  /// Values for bit flags used to specify the mapping type for
6669  /// offloading.
6670  enum OpenMPOffloadMappingFlags : uint64_t {
6671    /// No flags
6672    OMP_MAP_NONE = 0x0,
6673    /// Allocate memory on the device and move data from host to device.
6674    OMP_MAP_TO = 0x01,
6675    /// Allocate memory on the device and move data from device to host.
6676    OMP_MAP_FROM = 0x02,
6677    /// Always perform the requested mapping action on the element, even
6678    /// if it was already mapped before.
6679    OMP_MAP_ALWAYS = 0x04,
6680    /// Delete the element from the device environment, ignoring the
6681    /// current reference count associated with the element.
6682    OMP_MAP_DELETE = 0x08,
6683    /// The element being mapped is a pointer-pointee pair; both the
6684    /// pointer and the pointee should be mapped.
6685    OMP_MAP_PTR_AND_OBJ = 0x10,
6686    /// This flags signals that the base address of an entry should be
6687    /// passed to the target kernel as an argument.
6688    OMP_MAP_TARGET_PARAM = 0x20,
6689    /// Signal that the runtime library has to return the device pointer
6690    /// in the current position for the data being mapped. Used when we have the
6691    /// use_device_ptr clause.
6692    OMP_MAP_RETURN_PARAM = 0x40,
6693    /// This flag signals that the reference being passed is a pointer to
6694    /// private data.
6695    OMP_MAP_PRIVATE = 0x80,
6696    /// Pass the element to the device by value.
6697    OMP_MAP_LITERAL = 0x100,
6698    /// Implicit map
6699    OMP_MAP_IMPLICIT = 0x200,
6700    /// The 16 MSBs of the flags indicate whether the entry is member of some
6701    /// struct/class.
6702    OMP_MAP_MEMBER_OF = 0xffff000000000000,
6703    LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ OMP_MAP_MEMBER_OF),
6704  };
6705
6706  /// Class that associates information with a base pointer to be passed to the
6707  /// runtime library.
6708  class BasePointerInfo {
6709    /// The base pointer.
6710    llvm::Value *Ptr = nullptr;
6711    /// The base declaration that refers to this device pointer, or null if
6712    /// there is none.
6713    const ValueDecl *DevPtrDecl = nullptr;
6714
6715  public:
6716    BasePointerInfo(llvm::Value *Ptr, const ValueDecl *DevPtrDecl = nullptr)
6717        : Ptr(Ptr), DevPtrDecl(DevPtrDecl) {}
6718    llvm::Value *operator*() const { return Ptr; }
6719    const ValueDecl *getDevicePtrDecl() const { return DevPtrDecl; }
6720    void setDevicePtrDecl(const ValueDecl *D) { DevPtrDecl = D; }
6721  };
6722
6723  using MapBaseValuesArrayTy = SmallVector<BasePointerInfo, 4>;
6724  using MapValuesArrayTy = SmallVector<llvm::Value *, 4>;
6725  using MapFlagsArrayTy = SmallVector<OpenMPOffloadMappingFlags, 4>;
6726
6727  /// Map between a struct and the its lowest & highest elements which have been
6728  /// mapped.
6729  /// [ValueDecl *] --> {LE(FieldIndex, Pointer),
6730  ///                    HE(FieldIndex, Pointer)}
6731  struct StructRangeInfoTy {
6732    std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> LowestElem = {
6733        0, Address::invalid()};
6734    std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> HighestElem = {
6735        0, Address::invalid()};
6736    Address Base = Address::invalid();
6737  };
6738
6739private:
6740  /// Kind that defines how a device pointer has to be returned.
6741  struct MapInfo {
6742    OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
6743    OpenMPMapClauseKind MapType = OMPC_MAP_unknown;
6744    ArrayRef<OpenMPMapModifierKind> MapModifiers;
6745    bool ReturnDevicePointer = false;
6746    bool IsImplicit = false;
6747
6748    MapInfo() = default;
6749    MapInfo(
6750        OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
6751        OpenMPMapClauseKind MapType,
6752        ArrayRef<OpenMPMapModifierKind> MapModifiers,
6753        bool ReturnDevicePointer, bool IsImplicit)
6754        : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
6755          ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit) {}
6756  };
6757
6758  /// If use_device_ptr is used on a pointer which is a struct member and there
6759  /// is no map information about it, then emission of that entry is deferred
6760  /// until the whole struct has been processed.
6761  struct DeferredDevicePtrEntryTy {
6762    const Expr *IE = nullptr;
6763    const ValueDecl *VD = nullptr;
6764
6765    DeferredDevicePtrEntryTy(const Expr *IE, const ValueDecl *VD)
6766        : IE(IE), VD(VD) {}
6767  };
6768
6769  /// Directive from where the map clauses were extracted.
6770  const OMPExecutableDirective &CurDir;
6771
6772  /// Function the directive is being generated for.
6773  CodeGenFunction &CGF;
6774
6775  /// Set of all first private variables in the current directive.
6776  llvm::SmallPtrSet<const VarDecl *, 8> FirstPrivateDecls;
6777
6778  /// Map between device pointer declarations and their expression components.
6779  /// The key value for declarations in 'this' is null.
6780  llvm::DenseMap<
6781      const ValueDecl *,
6782      SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
6783      DevPointersMap;
6784
6785  llvm::Value *getExprTypeSize(const Expr *E) const {
6786    QualType ExprTy = E->getType().getCanonicalType();
6787
6788    // Reference types are ignored for mapping purposes.
6789    if (const auto *RefTy = ExprTy->getAs<ReferenceType>())
6790      ExprTy = RefTy->getPointeeType().getCanonicalType();
6791
6792    // Given that an array section is considered a built-in type, we need to
6793    // do the calculation based on the length of the section instead of relying
6794    // on CGF.getTypeSize(E->getType()).
6795    if (const auto *OAE = dyn_cast<OMPArraySectionExpr>(E)) {
6796      QualType BaseTy = OMPArraySectionExpr::getBaseOriginalType(
6797                            OAE->getBase()->IgnoreParenImpCasts())
6798                            .getCanonicalType();
6799
6800      // If there is no length associated with the expression, that means we
6801      // are using the whole length of the base.
6802      if (!OAE->getLength() && OAE->getColonLoc().isValid())
6803        return CGF.getTypeSize(BaseTy);
6804
6805      llvm::Value *ElemSize;
6806      if (const auto *PTy = BaseTy->getAs<PointerType>()) {
6807        ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType());
6808      } else {
6809        const auto *ATy = cast<ArrayType>(BaseTy.getTypePtr());
6810         (0) . __assert_fail ("ATy && \"Expecting array type if not a pointer type.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 6810, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(ATy && "Expecting array type if not a pointer type.");
6811        ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType());
6812      }
6813
6814      // If we don't have a length at this point, that is because we have an
6815      // array section with a single element.
6816      if (!OAE->getLength())
6817        return ElemSize;
6818
6819      llvm::Value *LengthVal = CGF.EmitScalarExpr(OAE->getLength());
6820      LengthVal =
6821          CGF.Builder.CreateIntCast(LengthVal, CGF.SizeTy, /*isSigned=*/false);
6822      return CGF.Builder.CreateNUWMul(LengthVal, ElemSize);
6823    }
6824    return CGF.getTypeSize(ExprTy);
6825  }
6826
6827  /// Return the corresponding bits for a given map clause modifier. Add
6828  /// a flag marking the map as a pointer if requested. Add a flag marking the
6829  /// map as the first one of a series of maps that relate to the same map
6830  /// expression.
6831  OpenMPOffloadMappingFlags getMapTypeBits(
6832      OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
6833      bool IsImplicit, bool AddPtrFlag, bool AddIsTargetParamFlag) const {
6834    OpenMPOffloadMappingFlags Bits =
6835        IsImplicit ? OMP_MAP_IMPLICIT : OMP_MAP_NONE;
6836    switch (MapType) {
6837    case OMPC_MAP_alloc:
6838    case OMPC_MAP_release:
6839      // alloc and release is the default behavior in the runtime library,  i.e.
6840      // if we don't pass any bits alloc/release that is what the runtime is
6841      // going to do. Therefore, we don't need to signal anything for these two
6842      // type modifiers.
6843      break;
6844    case OMPC_MAP_to:
6845      Bits |= OMP_MAP_TO;
6846      break;
6847    case OMPC_MAP_from:
6848      Bits |= OMP_MAP_FROM;
6849      break;
6850    case OMPC_MAP_tofrom:
6851      Bits |= OMP_MAP_TO | OMP_MAP_FROM;
6852      break;
6853    case OMPC_MAP_delete:
6854      Bits |= OMP_MAP_DELETE;
6855      break;
6856    case OMPC_MAP_unknown:
6857      llvm_unreachable("Unexpected map type!");
6858    }
6859    if (AddPtrFlag)
6860      Bits |= OMP_MAP_PTR_AND_OBJ;
6861    if (AddIsTargetParamFlag)
6862      Bits |= OMP_MAP_TARGET_PARAM;
6863    if (llvm::find(MapModifiers, OMPC_MAP_MODIFIER_always)
6864        != MapModifiers.end())
6865      Bits |= OMP_MAP_ALWAYS;
6866    return Bits;
6867  }
6868
6869  /// Return true if the provided expression is a final array section. A
6870  /// final array section, is one whose length can't be proved to be one.
6871  bool isFinalArraySectionExpression(const Expr *E) const {
6872    const auto *OASE = dyn_cast<OMPArraySectionExpr>(E);
6873
6874    // It is not an array section and therefore not a unity-size one.
6875    if (!OASE)
6876      return false;
6877
6878    // An array section with no colon always refer to a single element.
6879    if (OASE->getColonLoc().isInvalid())
6880      return false;
6881
6882    const Expr *Length = OASE->getLength();
6883
6884    // If we don't have a length we have to check if the array has size 1
6885    // for this dimension. Also, we should always expect a length if the
6886    // base type is pointer.
6887    if (!Length) {
6888      QualType BaseQTy = OMPArraySectionExpr::getBaseOriginalType(
6889                             OASE->getBase()->IgnoreParenImpCasts())
6890                             .getCanonicalType();
6891      if (const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr()))
6892        return ATy->getSize().getSExtValue() != 1;
6893      // If we don't have a constant dimension length, we have to consider
6894      // the current section as having any size, so it is not necessarily
6895      // unitary. If it happen to be unity size, that's user fault.
6896      return true;
6897    }
6898
6899    // Check if the length evaluates to 1.
6900    Expr::EvalResult Result;
6901    if (!Length->EvaluateAsInt(Result, CGF.getContext()))
6902      return true; // Can have more that size 1.
6903
6904    llvm::APSInt ConstLength = Result.Val.getInt();
6905    return ConstLength.getSExtValue() != 1;
6906  }
6907
6908  /// Generate the base pointers, section pointers, sizes and map type
6909  /// bits for the provided map type, map modifier, and expression components.
6910  /// \a IsFirstComponent should be set to true if the provided set of
6911  /// components is the first associated with a capture.
6912  void generateInfoForComponentList(
6913      OpenMPMapClauseKind MapType,
6914      ArrayRef<OpenMPMapModifierKind> MapModifiers,
6915      OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
6916      MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
6917      MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
6918      StructRangeInfoTy &PartialStruct, bool IsFirstComponentList,
6919      bool IsImplicit,
6920      ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
6921          OverlappedElements = llvm::None) const {
6922    // The following summarizes what has to be generated for each map and the
6923    // types below. The generated information is expressed in this order:
6924    // base pointer, section pointer, size, flags
6925    // (to add to the ones that come from the map type and modifier).
6926    //
6927    // double d;
6928    // int i[100];
6929    // float *p;
6930    //
6931    // struct S1 {
6932    //   int i;
6933    //   float f[50];
6934    // }
6935    // struct S2 {
6936    //   int i;
6937    //   float f[50];
6938    //   S1 s;
6939    //   double *p;
6940    //   struct S2 *ps;
6941    // }
6942    // S2 s;
6943    // S2 *ps;
6944    //
6945    // map(d)
6946    // &d, &d, sizeof(double), TARGET_PARAM | TO | FROM
6947    //
6948    // map(i)
6949    // &i, &i, 100*sizeof(int), TARGET_PARAM | TO | FROM
6950    //
6951    // map(i[1:23])
6952    // &i(=&i[0]), &i[1], 23*sizeof(int), TARGET_PARAM | TO | FROM
6953    //
6954    // map(p)
6955    // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
6956    //
6957    // map(p[1:24])
6958    // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM
6959    //
6960    // map(s)
6961    // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM
6962    //
6963    // map(s.i)
6964    // &s, &(s.i), sizeof(int), TARGET_PARAM | TO | FROM
6965    //
6966    // map(s.s.f)
6967    // &s, &(s.s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
6968    //
6969    // map(s.p)
6970    // &s, &(s.p), sizeof(double*), TARGET_PARAM | TO | FROM
6971    //
6972    // map(to: s.p[:22])
6973    // &s, &(s.p), sizeof(double*), TARGET_PARAM (*)
6974    // &s, &(s.p), sizeof(double*), MEMBER_OF(1) (**)
6975    // &(s.p), &(s.p[0]), 22*sizeof(double),
6976    //   MEMBER_OF(1) | PTR_AND_OBJ | TO (***)
6977    // (*) alloc space for struct members, only this is a target parameter
6978    // (**) map the pointer (nothing to be mapped in this example) (the compiler
6979    //      optimizes this entry out, same in the examples below)
6980    // (***) map the pointee (map: to)
6981    //
6982    // map(s.ps)
6983    // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM
6984    //
6985    // map(from: s.ps->s.i)
6986    // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
6987    // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
6988    // &(s.ps), &(s.ps->s.i), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ  | FROM
6989    //
6990    // map(to: s.ps->ps)
6991    // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
6992    // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
6993    // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ  | TO
6994    //
6995    // map(s.ps->ps->ps)
6996    // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
6997    // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
6998    // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
6999    // &(s.ps->ps), &(s.ps->ps->ps), sizeof(S2*), PTR_AND_OBJ | TO | FROM
7000    //
7001    // map(to: s.ps->ps->s.f[:22])
7002    // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
7003    // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
7004    // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
7005    // &(s.ps->ps), &(s.ps->ps->s.f[0]), 22*sizeof(float), PTR_AND_OBJ | TO
7006    //
7007    // map(ps)
7008    // &ps, &ps, sizeof(S2*), TARGET_PARAM | TO | FROM
7009    //
7010    // map(ps->i)
7011    // ps, &(ps->i), sizeof(int), TARGET_PARAM | TO | FROM
7012    //
7013    // map(ps->s.f)
7014    // ps, &(ps->s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
7015    //
7016    // map(from: ps->p)
7017    // ps, &(ps->p), sizeof(double*), TARGET_PARAM | FROM
7018    //
7019    // map(to: ps->p[:22])
7020    // ps, &(ps->p), sizeof(double*), TARGET_PARAM
7021    // ps, &(ps->p), sizeof(double*), MEMBER_OF(1)
7022    // &(ps->p), &(ps->p[0]), 22*sizeof(double), MEMBER_OF(1) | PTR_AND_OBJ | TO
7023    //
7024    // map(ps->ps)
7025    // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM | TO | FROM
7026    //
7027    // map(from: ps->ps->s.i)
7028    // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
7029    // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
7030    // &(ps->ps), &(ps->ps->s.i), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | FROM
7031    //
7032    // map(from: ps->ps->ps)
7033    // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
7034    // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
7035    // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ | FROM
7036    //
7037    // map(ps->ps->ps->ps)
7038    // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
7039    // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
7040    // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
7041    // &(ps->ps->ps), &(ps->ps->ps->ps), sizeof(S2*), PTR_AND_OBJ | TO | FROM
7042    //
7043    // map(to: ps->ps->ps->s.f[:22])
7044    // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
7045    // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
7046    // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
7047    // &(ps->ps->ps), &(ps->ps->ps->s.f[0]), 22*sizeof(float), PTR_AND_OBJ | TO
7048    //
7049    // map(to: s.f[:22]) map(from: s.p[:33])
7050    // &s, &(s.f[0]), 50*sizeof(float) + sizeof(struct S1) +
7051    //     sizeof(double*) (**), TARGET_PARAM
7052    // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | TO
7053    // &s, &(s.p), sizeof(double*), MEMBER_OF(1)
7054    // &(s.p), &(s.p[0]), 33*sizeof(double), MEMBER_OF(1) | PTR_AND_OBJ | FROM
7055    // (*) allocate contiguous space needed to fit all mapped members even if
7056    //     we allocate space for members not mapped (in this example,
7057    //     s.f[22..49] and s.s are not mapped, yet we must allocate space for
7058    //     them as well because they fall between &s.f[0] and &s.p)
7059    //
7060    // map(from: s.f[:22]) map(to: ps->p[:33])
7061    // &s, &(s.f[0]), 22*sizeof(float), TARGET_PARAM | FROM
7062    // ps, &(ps->p), sizeof(S2*), TARGET_PARAM
7063    // ps, &(ps->p), sizeof(double*), MEMBER_OF(2) (*)
7064    // &(ps->p), &(ps->p[0]), 33*sizeof(double), MEMBER_OF(2) | PTR_AND_OBJ | TO
7065    // (*) the struct this entry pertains to is the 2nd element in the list of
7066    //     arguments, hence MEMBER_OF(2)
7067    //
7068    // map(from: s.f[:22], s.s) map(to: ps->p[:33])
7069    // &s, &(s.f[0]), 50*sizeof(float) + sizeof(struct S1), TARGET_PARAM
7070    // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | FROM
7071    // &s, &(s.s), sizeof(struct S1), MEMBER_OF(1) | FROM
7072    // ps, &(ps->p), sizeof(S2*), TARGET_PARAM
7073    // ps, &(ps->p), sizeof(double*), MEMBER_OF(4) (*)
7074    // &(ps->p), &(ps->p[0]), 33*sizeof(double), MEMBER_OF(4) | PTR_AND_OBJ | TO
7075    // (*) the struct this entry pertains to is the 4th element in the list
7076    //     of arguments, hence MEMBER_OF(4)
7077
7078    // Track if the map information being generated is the first for a capture.
7079    bool IsCaptureFirstInfo = IsFirstComponentList;
7080    bool IsLink = false; // Is this variable a "declare target link"?
7081
7082    // Scan the components from the base to the complete expression.
7083    auto CI = Components.rbegin();
7084    auto CE = Components.rend();
7085    auto I = CI;
7086
7087    // Track if the map information being generated is the first for a list of
7088    // components.
7089    bool IsExpressionFirstInfo = true;
7090    Address BP = Address::invalid();
7091    const Expr *AssocExpr = I->getAssociatedExpression();
7092    const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
7093    const auto *OASE = dyn_cast<OMPArraySectionExpr>(AssocExpr);
7094
7095    if (isa<MemberExpr>(AssocExpr)) {
7096      // The base is the 'this' pointer. The content of the pointer is going
7097      // to be the base of the field being mapped.
7098      BP = CGF.LoadCXXThisAddress();
7099    } else if ((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||
7100               (OASE &&
7101                isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) {
7102      BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress();
7103    } else {
7104      // The base is the reference to the variable.
7105      // BP = &Var.
7106      BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress();
7107      if (const auto *VD =
7108              dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {
7109        if (llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
7110                OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD))
7111          if (*Res == OMPDeclareTargetDeclAttr::MT_Link) {
7112            IsLink = true;
7113            BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetLink(VD);
7114          }
7115      }
7116
7117      // If the variable is a pointer and is being dereferenced (i.e. is not
7118      // the last component), the base has to be the pointer itself, not its
7119      // reference. References are ignored for mapping purposes.
7120      QualType Ty =
7121          I->getAssociatedDeclaration()->getType().getNonReferenceType();
7122      if (Ty->isAnyPointerType() && std::next(I) != CE) {
7123        BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
7124
7125        // We do not need to generate individual map information for the
7126        // pointer, it can be associated with the combined storage.
7127        ++I;
7128      }
7129    }
7130
7131    // Track whether a component of the list should be marked as MEMBER_OF some
7132    // combined entry (for partial structs). Only the first PTR_AND_OBJ entry
7133    // in a component list should be marked as MEMBER_OF, all subsequent entries
7134    // do not belong to the base struct. E.g.
7135    // struct S2 s;
7136    // s.ps->ps->ps->f[:]
7137    //   (1) (2) (3) (4)
7138    // ps(1) is a member pointer, ps(2) is a pointee of ps(1), so it is a
7139    // PTR_AND_OBJ entry; the PTR is ps(1), so MEMBER_OF the base struct. ps(3)
7140    // is the pointee of ps(2) which is not member of struct s, so it should not
7141    // be marked as such (it is still PTR_AND_OBJ).
7142    // The variable is initialized to false so that PTR_AND_OBJ entries which
7143    // are not struct members are not considered (e.g. array of pointers to
7144    // data).
7145    bool ShouldBeMemberOf = false;
7146
7147    // Variable keeping track of whether or not we have encountered a component
7148    // in the component list which is a member expression. Useful when we have a
7149    // pointer or a final array section, in which case it is the previous
7150    // component in the list which tells us whether we have a member expression.
7151    // E.g. X.f[:]
7152    // While processing the final array section "[:]" it is "f" which tells us
7153    // whether we are dealing with a member of a declared struct.
7154    const MemberExpr *EncounteredME = nullptr;
7155
7156    for (; I != CE; ++I) {
7157      // If the current component is member of a struct (parent struct) mark it.
7158      if (!EncounteredME) {
7159        EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
7160        // If we encounter a PTR_AND_OBJ entry from now on it should be marked
7161        // as MEMBER_OF the parent struct.
7162        if (EncounteredME)
7163          ShouldBeMemberOf = true;
7164      }
7165
7166      auto Next = std::next(I);
7167
7168      // We need to generate the addresses and sizes if this is the last
7169      // component, if the component is a pointer or if it is an array section
7170      // whose length can't be proved to be one. If this is a pointer, it
7171      // becomes the base address for the following components.
7172
7173      // A final array section, is one whose length can't be proved to be one.
7174      bool IsFinalArraySection =
7175          isFinalArraySectionExpression(I->getAssociatedExpression());
7176
7177      // Get information on whether the element is a pointer. Have to do a
7178      // special treatment for array sections given that they are built-in
7179      // types.
7180      const auto *OASE =
7181          dyn_cast<OMPArraySectionExpr>(I->getAssociatedExpression());
7182      bool IsPointer =
7183          (OASE && OMPArraySectionExpr::getBaseOriginalType(OASE)
7184                       .getCanonicalType()
7185                       ->isAnyPointerType()) ||
7186          I->getAssociatedExpression()->getType()->isAnyPointerType();
7187
7188      if (Next == CE || IsPointer || IsFinalArraySection) {
7189        // If this is not the last component, we expect the pointer to be
7190        // associated with an array expression or member expression.
7191         (0) . __assert_fail ("(Next == CE || isa(Next->getAssociatedExpression()) || isa(Next->getAssociatedExpression()) || isa(Next->getAssociatedExpression())) && \"Unexpected expression\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7195, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert((Next == CE ||
7192 (0) . __assert_fail ("(Next == CE || isa(Next->getAssociatedExpression()) || isa(Next->getAssociatedExpression()) || isa(Next->getAssociatedExpression())) && \"Unexpected expression\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7195, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                isa<MemberExpr>(Next->getAssociatedExpression()) ||
7193 (0) . __assert_fail ("(Next == CE || isa(Next->getAssociatedExpression()) || isa(Next->getAssociatedExpression()) || isa(Next->getAssociatedExpression())) && \"Unexpected expression\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7195, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
7194 (0) . __assert_fail ("(Next == CE || isa(Next->getAssociatedExpression()) || isa(Next->getAssociatedExpression()) || isa(Next->getAssociatedExpression())) && \"Unexpected expression\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7195, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                isa<OMPArraySectionExpr>(Next->getAssociatedExpression())) &&
7195 (0) . __assert_fail ("(Next == CE || isa(Next->getAssociatedExpression()) || isa(Next->getAssociatedExpression()) || isa(Next->getAssociatedExpression())) && \"Unexpected expression\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7195, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">               "Unexpected expression");
7196
7197        Address LB =
7198            CGF.EmitOMPSharedLValue(I->getAssociatedExpression()).getAddress();
7199
7200        // If this component is a pointer inside the base struct then we don't
7201        // need to create any entry for it - it will be combined with the object
7202        // it is pointing to into a single PTR_AND_OBJ entry.
7203        bool IsMemberPointer =
7204            IsPointer && EncounteredME &&
7205            (dyn_cast<MemberExpr>(I->getAssociatedExpression()) ==
7206             EncounteredME);
7207        if (!OverlappedElements.empty()) {
7208          // Handle base element with the info for overlapped elements.
7209           (0) . __assert_fail ("!PartialStruct.Base.isValid() && \"The base element is set.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7209, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!PartialStruct.Base.isValid() && "The base element is set.");
7210           (0) . __assert_fail ("Next == CE && \"Expected last element for the overlapped elements.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7211, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Next == CE &&
7211 (0) . __assert_fail ("Next == CE && \"Expected last element for the overlapped elements.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7211, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                 "Expected last element for the overlapped elements.");
7212           (0) . __assert_fail ("!IsPointer && \"Unexpected base element with the pointer type.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7213, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!IsPointer &&
7213 (0) . __assert_fail ("!IsPointer && \"Unexpected base element with the pointer type.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7213, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                 "Unexpected base element with the pointer type.");
7214          // Mark the whole struct as the struct that requires allocation on the
7215          // device.
7216          PartialStruct.LowestElem = {0, LB};
7217          CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(
7218              I->getAssociatedExpression()->getType());
7219          Address HB = CGF.Builder.CreateConstGEP(
7220              CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(LB,
7221                                                              CGF.VoidPtrTy),
7222              TypeSize.getQuantity() - 1);
7223          PartialStruct.HighestElem = {
7224              std::numeric_limits<decltype(
7225                  PartialStruct.HighestElem.first)>::max(),
7226              HB};
7227          PartialStruct.Base = BP;
7228          // Emit data for non-overlapped data.
7229          OpenMPOffloadMappingFlags Flags =
7230              OMP_MAP_MEMBER_OF |
7231              getMapTypeBits(MapType, MapModifiers, IsImplicit,
7232                             /*AddPtrFlag=*/false,
7233                             /*AddIsTargetParamFlag=*/false);
7234          LB = BP;
7235          llvm::Value *Size = nullptr;
7236          // Do bitcopy of all non-overlapped structure elements.
7237          for (OMPClauseMappableExprCommon::MappableExprComponentListRef
7238                   Component : OverlappedElements) {
7239            Address ComponentLB = Address::invalid();
7240            for (const OMPClauseMappableExprCommon::MappableComponent &MC :
7241                 Component) {
7242              if (MC.getAssociatedDeclaration()) {
7243                ComponentLB =
7244                    CGF.EmitOMPSharedLValue(MC.getAssociatedExpression())
7245                        .getAddress();
7246                Size = CGF.Builder.CreatePtrDiff(
7247                    CGF.EmitCastToVoidPtr(ComponentLB.getPointer()),
7248                    CGF.EmitCastToVoidPtr(LB.getPointer()));
7249                break;
7250              }
7251            }
7252            BasePointers.push_back(BP.getPointer());
7253            Pointers.push_back(LB.getPointer());
7254            Sizes.push_back(Size);
7255            Types.push_back(Flags);
7256            LB = CGF.Builder.CreateConstGEP(ComponentLB, 1);
7257          }
7258          BasePointers.push_back(BP.getPointer());
7259          Pointers.push_back(LB.getPointer());
7260          Size = CGF.Builder.CreatePtrDiff(
7261              CGF.EmitCastToVoidPtr(
7262                  CGF.Builder.CreateConstGEP(HB, 1).getPointer()),
7263              CGF.EmitCastToVoidPtr(LB.getPointer()));
7264          Sizes.push_back(Size);
7265          Types.push_back(Flags);
7266          break;
7267        }
7268        llvm::Value *Size = getExprTypeSize(I->getAssociatedExpression());
7269        if (!IsMemberPointer) {
7270          BasePointers.push_back(BP.getPointer());
7271          Pointers.push_back(LB.getPointer());
7272          Sizes.push_back(Size);
7273
7274          // We need to add a pointer flag for each map that comes from the
7275          // same expression except for the first one. We also need to signal
7276          // this map is the first one that relates with the current capture
7277          // (there is a set of entries for each capture).
7278          OpenMPOffloadMappingFlags Flags = getMapTypeBits(
7279              MapType, MapModifiers, IsImplicit,
7280              !IsExpressionFirstInfo || IsLink, IsCaptureFirstInfo && !IsLink);
7281
7282          if (!IsExpressionFirstInfo) {
7283            // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well,
7284            // then we reset the TO/FROM/ALWAYS/DELETE flags.
7285            if (IsPointer)
7286              Flags &= ~(OMP_MAP_TO | OMP_MAP_FROM | OMP_MAP_ALWAYS |
7287                         OMP_MAP_DELETE);
7288
7289            if (ShouldBeMemberOf) {
7290              // Set placeholder value MEMBER_OF=FFFF to indicate that the flag
7291              // should be later updated with the correct value of MEMBER_OF.
7292              Flags |= OMP_MAP_MEMBER_OF;
7293              // From now on, all subsequent PTR_AND_OBJ entries should not be
7294              // marked as MEMBER_OF.
7295              ShouldBeMemberOf = false;
7296            }
7297          }
7298
7299          Types.push_back(Flags);
7300        }
7301
7302        // If we have encountered a member expression so far, keep track of the
7303        // mapped member. If the parent is "*this", then the value declaration
7304        // is nullptr.
7305        if (EncounteredME) {
7306          const auto *FD = dyn_cast<FieldDecl>(EncounteredME->getMemberDecl());
7307          unsigned FieldIndex = FD->getFieldIndex();
7308
7309          // Update info about the lowest and highest elements for this struct
7310          if (!PartialStruct.Base.isValid()) {
7311            PartialStruct.LowestElem = {FieldIndex, LB};
7312            PartialStruct.HighestElem = {FieldIndex, LB};
7313            PartialStruct.Base = BP;
7314          } else if (FieldIndex < PartialStruct.LowestElem.first) {
7315            PartialStruct.LowestElem = {FieldIndex, LB};
7316          } else if (FieldIndex > PartialStruct.HighestElem.first) {
7317            PartialStruct.HighestElem = {FieldIndex, LB};
7318          }
7319        }
7320
7321        // If we have a final array section, we are done with this expression.
7322        if (IsFinalArraySection)
7323          break;
7324
7325        // The pointer becomes the base for the next element.
7326        if (Next != CE)
7327          BP = LB;
7328
7329        IsExpressionFirstInfo = false;
7330        IsCaptureFirstInfo = false;
7331      }
7332    }
7333  }
7334
7335  /// Return the adjusted map modifiers if the declaration a capture refers to
7336  /// appears in a first-private clause. This is expected to be used only with
7337  /// directives that start with 'target'.
7338  MappableExprsHandler::OpenMPOffloadMappingFlags
7339  getMapModifiersForPrivateClauses(const CapturedStmt::Capture &Cap) const {
7340     (0) . __assert_fail ("Cap.capturesVariable() && \"Expected capture by reference only!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7340, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Cap.capturesVariable() && "Expected capture by reference only!");
7341
7342    // A first private variable captured by reference will use only the
7343    // 'private ptr' and 'map to' flag. Return the right flags if the captured
7344    // declaration is known as first-private in this handler.
7345    if (FirstPrivateDecls.count(Cap.getCapturedVar())) {
7346      if (Cap.getCapturedVar()->getType().isConstant(CGF.getContext()) &&
7347          Cap.getCaptureKind() == CapturedStmt::VCK_ByRef)
7348        return MappableExprsHandler::OMP_MAP_ALWAYS |
7349               MappableExprsHandler::OMP_MAP_TO;
7350      return MappableExprsHandler::OMP_MAP_PRIVATE |
7351             MappableExprsHandler::OMP_MAP_TO;
7352    }
7353    return MappableExprsHandler::OMP_MAP_TO |
7354           MappableExprsHandler::OMP_MAP_FROM;
7355  }
7356
7357  static OpenMPOffloadMappingFlags getMemberOfFlag(unsigned Position) {
7358    // Member of is given by the 16 MSB of the flag, so rotate by 48 bits.
7359    return static_cast<OpenMPOffloadMappingFlags>(((uint64_t)Position + 1)
7360                                                  << 48);
7361  }
7362
7363  static void setCorrectMemberOfFlag(OpenMPOffloadMappingFlags &Flags,
7364                                     OpenMPOffloadMappingFlags MemberOfFlag) {
7365    // If the entry is PTR_AND_OBJ but has not been marked with the special
7366    // placeholder value 0xFFFF in the MEMBER_OF field, then it should not be
7367    // marked as MEMBER_OF.
7368    if ((Flags & OMP_MAP_PTR_AND_OBJ) &&
7369        ((Flags & OMP_MAP_MEMBER_OF) != OMP_MAP_MEMBER_OF))
7370      return;
7371
7372    // Reset the placeholder value to prepare the flag for the assignment of the
7373    // proper MEMBER_OF value.
7374    Flags &= ~OMP_MAP_MEMBER_OF;
7375    Flags |= MemberOfFlag;
7376  }
7377
7378  void getPlainLayout(const CXXRecordDecl *RD,
7379                      llvm::SmallVectorImpl<const FieldDecl *> &Layout,
7380                      bool AsBase) const {
7381    const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD);
7382
7383    llvm::StructType *St =
7384        AsBase ? RL.getBaseSubobjectLLVMType() : RL.getLLVMType();
7385
7386    unsigned NumElements = St->getNumElements();
7387    llvm::SmallVector<
7388        llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
7389        RecordLayout(NumElements);
7390
7391    // Fill bases.
7392    for (const auto &I : RD->bases()) {
7393      if (I.isVirtual())
7394        continue;
7395      const auto *Base = I.getType()->getAsCXXRecordDecl();
7396      // Ignore empty bases.
7397      if (Base->isEmpty() || CGF.getContext()
7398                                 .getASTRecordLayout(Base)
7399                                 .getNonVirtualSize()
7400                                 .isZero())
7401        continue;
7402
7403      unsigned FieldIndex = RL.getNonVirtualBaseLLVMFieldNo(Base);
7404      RecordLayout[FieldIndex] = Base;
7405    }
7406    // Fill in virtual bases.
7407    for (const auto &I : RD->vbases()) {
7408      const auto *Base = I.getType()->getAsCXXRecordDecl();
7409      // Ignore empty bases.
7410      if (Base->isEmpty())
7411        continue;
7412      unsigned FieldIndex = RL.getVirtualBaseIndex(Base);
7413      if (RecordLayout[FieldIndex])
7414        continue;
7415      RecordLayout[FieldIndex] = Base;
7416    }
7417    // Fill in all the fields.
7418     (0) . __assert_fail ("!RD->isUnion() && \"Unexpected union.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7418, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!RD->isUnion() && "Unexpected union.");
7419    for (const auto *Field : RD->fields()) {
7420      // Fill in non-bitfields. (Bitfields always use a zero pattern, which we
7421      // will fill in later.)
7422      if (!Field->isBitField()) {
7423        unsigned FieldIndex = RL.getLLVMFieldNo(Field);
7424        RecordLayout[FieldIndex] = Field;
7425      }
7426    }
7427    for (const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
7428             &Data : RecordLayout) {
7429      if (Data.isNull())
7430        continue;
7431      if (const auto *Base = Data.dyn_cast<const CXXRecordDecl *>())
7432        getPlainLayout(Base, Layout, /*AsBase=*/true);
7433      else
7434        Layout.push_back(Data.get<const FieldDecl *>());
7435    }
7436  }
7437
7438public:
7439  MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
7440      : CurDir(Dir), CGF(CGF) {
7441    // Extract firstprivate clause information.
7442    for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>())
7443      for (const auto *D : C->varlists())
7444        FirstPrivateDecls.insert(
7445            cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
7446    // Extract device pointer clause information.
7447    for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
7448      for (auto L : C->component_lists())
7449        DevPointersMap[L.first].push_back(L.second);
7450  }
7451
7452  /// Generate code for the combined entry if we have a partially mapped struct
7453  /// and take care of the mapping flags of the arguments corresponding to
7454  /// individual struct members.
7455  void emitCombinedEntry(MapBaseValuesArrayTy &BasePointers,
7456                         MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
7457                         MapFlagsArrayTy &Types, MapFlagsArrayTy &CurTypes,
7458                         const StructRangeInfoTy &PartialStruct) const {
7459    // Base is the base of the struct
7460    BasePointers.push_back(PartialStruct.Base.getPointer());
7461    // Pointer is the address of the lowest element
7462    llvm::Value *LB = PartialStruct.LowestElem.second.getPointer();
7463    Pointers.push_back(LB);
7464    // Size is (addr of {highest+1} element) - (addr of lowest element)
7465    llvm::Value *HB = PartialStruct.HighestElem.second.getPointer();
7466    llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(HB, /*Idx0=*/1);
7467    llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy);
7468    llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy);
7469    llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CHAddr, CLAddr);
7470    llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.SizeTy,
7471                                                  /*isSinged=*/false);
7472    Sizes.push_back(Size);
7473    // Map type is always TARGET_PARAM
7474    Types.push_back(OMP_MAP_TARGET_PARAM);
7475    // Remove TARGET_PARAM flag from the first element
7476    (*CurTypes.begin()) &= ~OMP_MAP_TARGET_PARAM;
7477
7478    // All other current entries will be MEMBER_OF the combined entry
7479    // (except for PTR_AND_OBJ entries which do not have a placeholder value
7480    // 0xFFFF in the MEMBER_OF field).
7481    OpenMPOffloadMappingFlags MemberOfFlag =
7482        getMemberOfFlag(BasePointers.size() - 1);
7483    for (auto &M : CurTypes)
7484      setCorrectMemberOfFlag(M, MemberOfFlag);
7485  }
7486
7487  /// Generate all the base pointers, section pointers, sizes and map
7488  /// types for the extracted mappable expressions. Also, for each item that
7489  /// relates with a device pointer, a pair of the relevant declaration and
7490  /// index where it occurs is appended to the device pointers info array.
7491  void generateAllInfo(MapBaseValuesArrayTy &BasePointers,
7492                       MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
7493                       MapFlagsArrayTy &Types) const {
7494    // We have to process the component lists that relate with the same
7495    // declaration in a single chunk so that we can generate the map flags
7496    // correctly. Therefore, we organize all lists in a map.
7497    llvm::MapVector<const ValueDecl *, SmallVector<MapInfo, 8>> Info;
7498
7499    // Helper function to fill the information map for the different supported
7500    // clauses.
7501    auto &&InfoGen = [&Info](
7502        const ValueDecl *D,
7503        OMPClauseMappableExprCommon::MappableExprComponentListRef L,
7504        OpenMPMapClauseKind MapType,
7505        ArrayRef<OpenMPMapModifierKind> MapModifiers,
7506        bool ReturnDevicePointer, bool IsImplicit) {
7507      const ValueDecl *VD =
7508          D ? cast<ValueDecl>(D->getCanonicalDecl()) : nullptr;
7509      Info[VD].emplace_back(L, MapType, MapModifiers, ReturnDevicePointer,
7510                            IsImplicit);
7511    };
7512
7513    // FIXME: MSVC 2013 seems to require this-> to find member CurDir.
7514    for (const auto *C : this->CurDir.getClausesOfKind<OMPMapClause>())
7515      for (const auto &L : C->component_lists()) {
7516        InfoGen(L.first, L.second, C->getMapType(), C->getMapTypeModifiers(),
7517            /*ReturnDevicePointer=*/false, C->isImplicit());
7518      }
7519    for (const auto *C : this->CurDir.getClausesOfKind<OMPToClause>())
7520      for (const auto &L : C->component_lists()) {
7521        InfoGen(L.first, L.second, OMPC_MAP_to, llvm::None,
7522            /*ReturnDevicePointer=*/false, C->isImplicit());
7523      }
7524    for (const auto *C : this->CurDir.getClausesOfKind<OMPFromClause>())
7525      for (const auto &L : C->component_lists()) {
7526        InfoGen(L.first, L.second, OMPC_MAP_from, llvm::None,
7527            /*ReturnDevicePointer=*/false, C->isImplicit());
7528      }
7529
7530    // Look at the use_device_ptr clause information and mark the existing map
7531    // entries as such. If there is no map information for an entry in the
7532    // use_device_ptr list, we create one with map type 'alloc' and zero size
7533    // section. It is the user fault if that was not mapped before. If there is
7534    // no map information and the pointer is a struct member, then we defer the
7535    // emission of that entry until the whole struct has been processed.
7536    llvm::MapVector<const ValueDecl *, SmallVector<DeferredDevicePtrEntryTy, 4>>
7537        DeferredInfo;
7538
7539    // FIXME: MSVC 2013 seems to require this-> to find member CurDir.
7540    for (const auto *C :
7541        this->CurDir.getClausesOfKind<OMPUseDevicePtrClause>()) {
7542      for (const auto &L : C->component_lists()) {
7543         (0) . __assert_fail ("!L.second.empty() && \"Not expecting empty list of components!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7543, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!L.second.empty() && "Not expecting empty list of components!");
7544        const ValueDecl *VD = L.second.back().getAssociatedDeclaration();
7545        VD = cast<ValueDecl>(VD->getCanonicalDecl());
7546        const Expr *IE = L.second.back().getAssociatedExpression();
7547        // If the first component is a member expression, we have to look into
7548        // 'this', which maps to null in the map of map information. Otherwise
7549        // look directly for the information.
7550        auto It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD);
7551
7552        // We potentially have map information for this declaration already.
7553        // Look for the first set of components that refer to it.
7554        if (It != Info.end()) {
7555          auto CI = std::find_if(
7556              It->second.begin(), It->second.end(), [VD](const MapInfo &MI) {
7557                return MI.Components.back().getAssociatedDeclaration() == VD;
7558              });
7559          // If we found a map entry, signal that the pointer has to be returned
7560          // and move on to the next declaration.
7561          if (CI != It->second.end()) {
7562            CI->ReturnDevicePointer = true;
7563            continue;
7564          }
7565        }
7566
7567        // We didn't find any match in our map information - generate a zero
7568        // size array section - if the pointer is a struct member we defer this
7569        // action until the whole struct has been processed.
7570        // FIXME: MSVC 2013 seems to require this-> to find member CGF.
7571        if (isa<MemberExpr>(IE)) {
7572          // Insert the pointer into Info to be processed by
7573          // generateInfoForComponentList. Because it is a member pointer
7574          // without a pointee, no entry will be generated for it, therefore
7575          // we need to generate one after the whole struct has been processed.
7576          // Nonetheless, generateInfoForComponentList must be called to take
7577          // the pointer into account for the calculation of the range of the
7578          // partial struct.
7579          InfoGen(nullptr, L.second, OMPC_MAP_unknown, llvm::None,
7580                  /*ReturnDevicePointer=*/false, C->isImplicit());
7581          DeferredInfo[nullptr].emplace_back(IE, VD);
7582        } else {
7583          llvm::Value *Ptr = this->CGF.EmitLoadOfScalar(
7584              this->CGF.EmitLValue(IE), IE->getExprLoc());
7585          BasePointers.emplace_back(Ptr, VD);
7586          Pointers.push_back(Ptr);
7587          Sizes.push_back(llvm::Constant::getNullValue(this->CGF.SizeTy));
7588          Types.push_back(OMP_MAP_RETURN_PARAM | OMP_MAP_TARGET_PARAM);
7589        }
7590      }
7591    }
7592
7593    for (const auto &M : Info) {
7594      // We need to know when we generate information for the first component
7595      // associated with a capture, because the mapping flags depend on it.
7596      bool IsFirstComponentList = true;
7597
7598      // Temporary versions of arrays
7599      MapBaseValuesArrayTy CurBasePointers;
7600      MapValuesArrayTy CurPointers;
7601      MapValuesArrayTy CurSizes;
7602      MapFlagsArrayTy CurTypes;
7603      StructRangeInfoTy PartialStruct;
7604
7605      for (const MapInfo &L : M.second) {
7606         (0) . __assert_fail ("!L.Components.empty() && \"Not expecting declaration with no component lists.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7607, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!L.Components.empty() &&
7607 (0) . __assert_fail ("!L.Components.empty() && \"Not expecting declaration with no component lists.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7607, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">               "Not expecting declaration with no component lists.");
7608
7609        // Remember the current base pointer index.
7610        unsigned CurrentBasePointersIdx = CurBasePointers.size();
7611        // FIXME: MSVC 2013 seems to require this-> to find the member method.
7612        this->generateInfoForComponentList(
7613            L.MapType, L.MapModifiers, L.Components, CurBasePointers,
7614            CurPointers, CurSizes, CurTypes, PartialStruct,
7615            IsFirstComponentList, L.IsImplicit);
7616
7617        // If this entry relates with a device pointer, set the relevant
7618        // declaration and add the 'return pointer' flag.
7619        if (L.ReturnDevicePointer) {
7620           (0) . __assert_fail ("CurBasePointers.size() > CurrentBasePointersIdx && \"Unexpected number of mapped base pointers.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7621, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(CurBasePointers.size() > CurrentBasePointersIdx &&
7621 (0) . __assert_fail ("CurBasePointers.size() > CurrentBasePointersIdx && \"Unexpected number of mapped base pointers.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7621, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                 "Unexpected number of mapped base pointers.");
7622
7623          const ValueDecl *RelevantVD =
7624              L.Components.back().getAssociatedDeclaration();
7625           (0) . __assert_fail ("RelevantVD && \"No relevant declaration related with device pointer??\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7626, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(RelevantVD &&
7626 (0) . __assert_fail ("RelevantVD && \"No relevant declaration related with device pointer??\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7626, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                 "No relevant declaration related with device pointer??");
7627
7628          CurBasePointers[CurrentBasePointersIdx].setDevicePtrDecl(RelevantVD);
7629          CurTypes[CurrentBasePointersIdx] |= OMP_MAP_RETURN_PARAM;
7630        }
7631        IsFirstComponentList = false;
7632      }
7633
7634      // Append any pending zero-length pointers which are struct members and
7635      // used with use_device_ptr.
7636      auto CI = DeferredInfo.find(M.first);
7637      if (CI != DeferredInfo.end()) {
7638        for (const DeferredDevicePtrEntryTy &L : CI->second) {
7639          llvm::Value *BasePtr = this->CGF.EmitLValue(L.IE).getPointer();
7640          llvm::Value *Ptr = this->CGF.EmitLoadOfScalar(
7641              this->CGF.EmitLValue(L.IE), L.IE->getExprLoc());
7642          CurBasePointers.emplace_back(BasePtr, L.VD);
7643          CurPointers.push_back(Ptr);
7644          CurSizes.push_back(llvm::Constant::getNullValue(this->CGF.SizeTy));
7645          // Entry is PTR_AND_OBJ and RETURN_PARAM. Also, set the placeholder
7646          // value MEMBER_OF=FFFF so that the entry is later updated with the
7647          // correct value of MEMBER_OF.
7648          CurTypes.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_RETURN_PARAM |
7649                             OMP_MAP_MEMBER_OF);
7650        }
7651      }
7652
7653      // If there is an entry in PartialStruct it means we have a struct with
7654      // individual members mapped. Emit an extra combined entry.
7655      if (PartialStruct.Base.isValid())
7656        emitCombinedEntry(BasePointers, Pointers, Sizes, Types, CurTypes,
7657                          PartialStruct);
7658
7659      // We need to append the results of this capture to what we already have.
7660      BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
7661      Pointers.append(CurPointers.begin(), CurPointers.end());
7662      Sizes.append(CurSizes.begin(), CurSizes.end());
7663      Types.append(CurTypes.begin(), CurTypes.end());
7664    }
7665  }
7666
7667  /// Emit capture info for lambdas for variables captured by reference.
7668  void generateInfoForLambdaCaptures(
7669      const ValueDecl *VD, llvm::Value *Arg, MapBaseValuesArrayTy &BasePointers,
7670      MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
7671      MapFlagsArrayTy &Types,
7672      llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers) const {
7673    const auto *RD = VD->getType()
7674                         .getCanonicalType()
7675                         .getNonReferenceType()
7676                         ->getAsCXXRecordDecl();
7677    if (!RD || !RD->isLambda())
7678      return;
7679    Address VDAddr = Address(Arg, CGF.getContext().getDeclAlign(VD));
7680    LValue VDLVal = CGF.MakeAddrLValue(
7681        VDAddr, VD->getType().getCanonicalType().getNonReferenceType());
7682    llvm::DenseMap<const VarDecl *, FieldDecl *> Captures;
7683    FieldDecl *ThisCapture = nullptr;
7684    RD->getCaptureFields(Captures, ThisCapture);
7685    if (ThisCapture) {
7686      LValue ThisLVal =
7687          CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture);
7688      LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture);
7689      LambdaPointers.try_emplace(ThisLVal.getPointer(), VDLVal.getPointer());
7690      BasePointers.push_back(ThisLVal.getPointer());
7691      Pointers.push_back(ThisLValVal.getPointer());
7692      Sizes.push_back(CGF.getTypeSize(CGF.getContext().VoidPtrTy));
7693      Types.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
7694                      OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT);
7695    }
7696    for (const LambdaCapture &LC : RD->captures()) {
7697      if (LC.getCaptureKind() != LCK_ByRef)
7698        continue;
7699      const VarDecl *VD = LC.getCapturedVar();
7700      auto It = Captures.find(VD);
7701       (0) . __assert_fail ("It != Captures.end() && \"Found lambda capture without field.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7701, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(It != Captures.end() && "Found lambda capture without field.");
7702      LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second);
7703      LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second);
7704      LambdaPointers.try_emplace(VarLVal.getPointer(), VDLVal.getPointer());
7705      BasePointers.push_back(VarLVal.getPointer());
7706      Pointers.push_back(VarLValVal.getPointer());
7707      Sizes.push_back(CGF.getTypeSize(
7708          VD->getType().getCanonicalType().getNonReferenceType()));
7709      Types.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
7710                      OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT);
7711    }
7712  }
7713
7714  /// Set correct indices for lambdas captures.
7715  void adjustMemberOfForLambdaCaptures(
7716      const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
7717      MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
7718      MapFlagsArrayTy &Types) const {
7719    for (unsigned I = 0, E = Types.size(); I < E; ++I) {
7720      // Set correct member_of idx for all implicit lambda captures.
7721      if (Types[I] != (OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
7722                       OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT))
7723        continue;
7724      llvm::Value *BasePtr = LambdaPointers.lookup(*BasePointers[I]);
7725       (0) . __assert_fail ("BasePtr && \"Unable to find base lambda address.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7725, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(BasePtr && "Unable to find base lambda address.");
7726      int TgtIdx = -1;
7727      for (unsigned J = I; J > 0; --J) {
7728        unsigned Idx = J - 1;
7729        if (Pointers[Idx] != BasePtr)
7730          continue;
7731        TgtIdx = Idx;
7732        break;
7733      }
7734       (0) . __assert_fail ("TgtIdx != -1 && \"Unable to find parent lambda.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7734, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(TgtIdx != -1 && "Unable to find parent lambda.");
7735      // All other current entries will be MEMBER_OF the combined entry
7736      // (except for PTR_AND_OBJ entries which do not have a placeholder value
7737      // 0xFFFF in the MEMBER_OF field).
7738      OpenMPOffloadMappingFlags MemberOfFlag = getMemberOfFlag(TgtIdx);
7739      setCorrectMemberOfFlag(Types[I], MemberOfFlag);
7740    }
7741  }
7742
7743  /// Generate the base pointers, section pointers, sizes and map types
7744  /// associated to a given capture.
7745  void generateInfoForCapture(const CapturedStmt::Capture *Cap,
7746                              llvm::Value *Arg,
7747                              MapBaseValuesArrayTy &BasePointers,
7748                              MapValuesArrayTy &Pointers,
7749                              MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
7750                              StructRangeInfoTy &PartialStruct) const {
7751     (0) . __assert_fail ("!Cap->capturesVariableArrayType() && \"Not expecting to generate map info for a variable array type!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7752, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!Cap->capturesVariableArrayType() &&
7752 (0) . __assert_fail ("!Cap->capturesVariableArrayType() && \"Not expecting to generate map info for a variable array type!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7752, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">           "Not expecting to generate map info for a variable array type!");
7753
7754    // We need to know when we generating information for the first component
7755    const ValueDecl *VD = Cap->capturesThis()
7756                              ? nullptr
7757                              : Cap->getCapturedVar()->getCanonicalDecl();
7758
7759    // If this declaration appears in a is_device_ptr clause we just have to
7760    // pass the pointer by value. If it is a reference to a declaration, we just
7761    // pass its value.
7762    if (DevPointersMap.count(VD)) {
7763      BasePointers.emplace_back(Arg, VD);
7764      Pointers.push_back(Arg);
7765      Sizes.push_back(CGF.getTypeSize(CGF.getContext().VoidPtrTy));
7766      Types.push_back(OMP_MAP_LITERAL | OMP_MAP_TARGET_PARAM);
7767      return;
7768    }
7769
7770    using MapData =
7771        std::tuple<OMPClauseMappableExprCommon::MappableExprComponentListRef,
7772                   OpenMPMapClauseKind, ArrayRef<OpenMPMapModifierKind>, bool>;
7773    SmallVector<MapData, 4> DeclComponentLists;
7774    // FIXME: MSVC 2013 seems to require this-> to find member CurDir.
7775    for (const auto *C : this->CurDir.getClausesOfKind<OMPMapClause>()) {
7776      for (const auto &L : C->decl_component_lists(VD)) {
7777         (0) . __assert_fail ("L.first == VD && \"We got information for the wrong declaration??\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7778, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(L.first == VD &&
7778 (0) . __assert_fail ("L.first == VD && \"We got information for the wrong declaration??\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7778, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">               "We got information for the wrong declaration??");
7779         (0) . __assert_fail ("!L.second.empty() && \"Not expecting declaration with no component lists.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7780, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!L.second.empty() &&
7780 (0) . __assert_fail ("!L.second.empty() && \"Not expecting declaration with no component lists.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7780, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">               "Not expecting declaration with no component lists.");
7781        DeclComponentLists.emplace_back(L.second, C->getMapType(),
7782                                        C->getMapTypeModifiers(),
7783                                        C->isImplicit());
7784      }
7785    }
7786
7787    // Find overlapping elements (including the offset from the base element).
7788    llvm::SmallDenseMap<
7789        const MapData *,
7790        llvm::SmallVector<
7791            OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>,
7792        4>
7793        OverlappedData;
7794    size_t Count = 0;
7795    for (const MapData &L : DeclComponentLists) {
7796      OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
7797      OpenMPMapClauseKind MapType;
7798      ArrayRef<OpenMPMapModifierKind> MapModifiers;
7799      bool IsImplicit;
7800      std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
7801      ++Count;
7802      for (const MapData &L1 : makeArrayRef(DeclComponentLists).slice(Count)) {
7803        OMPClauseMappableExprCommon::MappableExprComponentListRef Components1;
7804        std::tie(Components1, MapType, MapModifiers, IsImplicit) = L1;
7805        auto CI = Components.rbegin();
7806        auto CE = Components.rend();
7807        auto SI = Components1.rbegin();
7808        auto SE = Components1.rend();
7809        for (; CI != CE && SI != SE; ++CI, ++SI) {
7810          if (CI->getAssociatedExpression()->getStmtClass() !=
7811              SI->getAssociatedExpression()->getStmtClass())
7812            break;
7813          // Are we dealing with different variables/fields?
7814          if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
7815            break;
7816        }
7817        // Found overlapping if, at least for one component, reached the head of
7818        // the components list.
7819        if (CI == CE || SI == SE) {
7820           (0) . __assert_fail ("(CI != CE || SI != SE) && \"Unexpected full match of the mapping components.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7821, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert((CI != CE || SI != SE) &&
7821 (0) . __assert_fail ("(CI != CE || SI != SE) && \"Unexpected full match of the mapping components.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7821, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                 "Unexpected full match of the mapping components.");
7822          const MapData &BaseData = CI == CE ? L : L1;
7823          OMPClauseMappableExprCommon::MappableExprComponentListRef SubData =
7824              SI == SE ? Components : Components1;
7825          auto &OverlappedElements = OverlappedData.FindAndConstruct(&BaseData);
7826          OverlappedElements.getSecond().push_back(SubData);
7827        }
7828      }
7829    }
7830    // Sort the overlapped elements for each item.
7831    llvm::SmallVector<const FieldDecl *, 4> Layout;
7832    if (!OverlappedData.empty()) {
7833      if (const auto *CRD =
7834              VD->getType().getCanonicalType()->getAsCXXRecordDecl())
7835        getPlainLayout(CRD, Layout, /*AsBase=*/false);
7836      else {
7837        const auto *RD = VD->getType().getCanonicalType()->getAsRecordDecl();
7838        Layout.append(RD->field_begin(), RD->field_end());
7839      }
7840    }
7841    for (auto &Pair : OverlappedData) {
7842      llvm::sort(
7843          Pair.getSecond(),
7844          [&Layout](
7845              OMPClauseMappableExprCommon::MappableExprComponentListRef First,
7846              OMPClauseMappableExprCommon::MappableExprComponentListRef
7847                  Second) {
7848            auto CI = First.rbegin();
7849            auto CE = First.rend();
7850            auto SI = Second.rbegin();
7851            auto SE = Second.rend();
7852            for (; CI != CE && SI != SE; ++CI, ++SI) {
7853              if (CI->getAssociatedExpression()->getStmtClass() !=
7854                  SI->getAssociatedExpression()->getStmtClass())
7855                break;
7856              // Are we dealing with different variables/fields?
7857              if (CI->getAssociatedDeclaration() !=
7858                  SI->getAssociatedDeclaration())
7859                break;
7860            }
7861
7862            // Lists contain the same elements.
7863            if (CI == CE && SI == SE)
7864              return false;
7865
7866            // List with less elements is less than list with more elements.
7867            if (CI == CE || SI == SE)
7868              return CI == CE;
7869
7870            const auto *FD1 = cast<FieldDecl>(CI->getAssociatedDeclaration());
7871            const auto *FD2 = cast<FieldDecl>(SI->getAssociatedDeclaration());
7872            if (FD1->getParent() == FD2->getParent())
7873              return FD1->getFieldIndex() < FD2->getFieldIndex();
7874            const auto It =
7875                llvm::find_if(Layout, [FD1, FD2](const FieldDecl *FD) {
7876                  return FD == FD1 || FD == FD2;
7877                });
7878            return *It == FD1;
7879          });
7880    }
7881
7882    // Associated with a capture, because the mapping flags depend on it.
7883    // Go through all of the elements with the overlapped elements.
7884    for (const auto &Pair : OverlappedData) {
7885      const MapData &L = *Pair.getFirst();
7886      OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
7887      OpenMPMapClauseKind MapType;
7888      ArrayRef<OpenMPMapModifierKind> MapModifiers;
7889      bool IsImplicit;
7890      std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
7891      ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
7892          OverlappedComponents = Pair.getSecond();
7893      bool IsFirstComponentList = true;
7894      generateInfoForComponentList(MapType, MapModifiers, Components,
7895                                   BasePointers, Pointers, Sizes, Types,
7896                                   PartialStruct, IsFirstComponentList,
7897                                   IsImplicit, OverlappedComponents);
7898    }
7899    // Go through other elements without overlapped elements.
7900    bool IsFirstComponentList = OverlappedData.empty();
7901    for (const MapData &L : DeclComponentLists) {
7902      OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
7903      OpenMPMapClauseKind MapType;
7904      ArrayRef<OpenMPMapModifierKind> MapModifiers;
7905      bool IsImplicit;
7906      std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
7907      auto It = OverlappedData.find(&L);
7908      if (It == OverlappedData.end())
7909        generateInfoForComponentList(MapType, MapModifiers, Components,
7910                                     BasePointers, Pointers, Sizes, Types,
7911                                     PartialStruct, IsFirstComponentList,
7912                                     IsImplicit);
7913      IsFirstComponentList = false;
7914    }
7915  }
7916
7917  /// Generate the base pointers, section pointers, sizes and map types
7918  /// associated with the declare target link variables.
7919  void generateInfoForDeclareTargetLink(MapBaseValuesArrayTy &BasePointers,
7920                                        MapValuesArrayTy &Pointers,
7921                                        MapValuesArrayTy &Sizes,
7922                                        MapFlagsArrayTy &Types) const {
7923    // Map other list items in the map clause which are not captured variables
7924    // but "declare target link" global variables.,
7925    for (const auto *C : this->CurDir.getClausesOfKind<OMPMapClause>()) {
7926      for (const auto &L : C->component_lists()) {
7927        if (!L.first)
7928          continue;
7929        const auto *VD = dyn_cast<VarDecl>(L.first);
7930        if (!VD)
7931          continue;
7932        llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
7933            OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
7934        if (!Res || *Res != OMPDeclareTargetDeclAttr::MT_Link)
7935          continue;
7936        StructRangeInfoTy PartialStruct;
7937        generateInfoForComponentList(
7938            C->getMapType(), C->getMapTypeModifiers(), L.second, BasePointers,
7939            Pointers, Sizes, Types, PartialStruct,
7940            /*IsFirstComponentList=*/true, C->isImplicit());
7941         (0) . __assert_fail ("!PartialStruct.Base.isValid() && \"No partial structs for declare target link expected.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7942, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!PartialStruct.Base.isValid() &&
7942 (0) . __assert_fail ("!PartialStruct.Base.isValid() && \"No partial structs for declare target link expected.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7942, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">               "No partial structs for declare target link expected.");
7943      }
7944    }
7945  }
7946
7947  /// Generate the default map information for a given capture \a CI,
7948  /// record field declaration \a RI and captured value \a CV.
7949  void generateDefaultMapInfo(const CapturedStmt::Capture &CI,
7950                              const FieldDecl &RI, llvm::Value *CV,
7951                              MapBaseValuesArrayTy &CurBasePointers,
7952                              MapValuesArrayTy &CurPointers,
7953                              MapValuesArrayTy &CurSizes,
7954                              MapFlagsArrayTy &CurMapTypes) const {
7955    // Do the default mapping.
7956    if (CI.capturesThis()) {
7957      CurBasePointers.push_back(CV);
7958      CurPointers.push_back(CV);
7959      const auto *PtrTy = cast<PointerType>(RI.getType().getTypePtr());
7960      CurSizes.push_back(CGF.getTypeSize(PtrTy->getPointeeType()));
7961      // Default map type.
7962      CurMapTypes.push_back(OMP_MAP_TO | OMP_MAP_FROM);
7963    } else if (CI.capturesVariableByCopy()) {
7964      CurBasePointers.push_back(CV);
7965      CurPointers.push_back(CV);
7966      if (!RI.getType()->isAnyPointerType()) {
7967        // We have to signal to the runtime captures passed by value that are
7968        // not pointers.
7969        CurMapTypes.push_back(OMP_MAP_LITERAL);
7970        CurSizes.push_back(CGF.getTypeSize(RI.getType()));
7971      } else {
7972        // Pointers are implicitly mapped with a zero size and no flags
7973        // (other than first map that is added for all implicit maps).
7974        CurMapTypes.push_back(OMP_MAP_NONE);
7975        CurSizes.push_back(llvm::Constant::getNullValue(CGF.SizeTy));
7976      }
7977    } else {
7978       (0) . __assert_fail ("CI.capturesVariable() && \"Expected captured reference.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 7978, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(CI.capturesVariable() && "Expected captured reference.");
7979      const auto *PtrTy = cast<ReferenceType>(RI.getType().getTypePtr());
7980      QualType ElementType = PtrTy->getPointeeType();
7981      CurSizes.push_back(CGF.getTypeSize(ElementType));
7982      // The default map type for a scalar/complex type is 'to' because by
7983      // default the value doesn't have to be retrieved. For an aggregate
7984      // type, the default is 'tofrom'.
7985      CurMapTypes.push_back(getMapModifiersForPrivateClauses(CI));
7986      const VarDecl *VD = CI.getCapturedVar();
7987      if (FirstPrivateDecls.count(VD) &&
7988          VD->getType().isConstant(CGF.getContext())) {
7989        llvm::Constant *Addr =
7990            CGF.CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(CGF, VD);
7991        // Copy the value of the original variable to the new global copy.
7992        CGF.Builder.CreateMemCpy(
7993            CGF.MakeNaturalAlignAddrLValue(Addr, ElementType).getAddress(),
7994            Address(CV, CGF.getContext().getTypeAlignInChars(ElementType)),
7995            CurSizes.back(),
7996            /*isVolatile=*/false);
7997        // Use new global variable as the base pointers.
7998        CurBasePointers.push_back(Addr);
7999        CurPointers.push_back(Addr);
8000      } else {
8001        CurBasePointers.push_back(CV);
8002        CurPointers.push_back(CV);
8003      }
8004    }
8005    // Every default map produces a single argument which is a target parameter.
8006    CurMapTypes.back() |= OMP_MAP_TARGET_PARAM;
8007
8008    // Add flag stating this is an implicit map.
8009    CurMapTypes.back() |= OMP_MAP_IMPLICIT;
8010  }
8011};
8012
8013enum OpenMPOffloadingReservedDeviceIDs {
8014  /// Device ID if the device was not defined, runtime should get it
8015  /// from environment variables in the spec.
8016  OMP_DEVICEID_UNDEF = -1,
8017};
8018} // anonymous namespace
8019
8020/// Emit the arrays used to pass the captures and map information to the
8021/// offloading runtime library. If there is no map or capture information,
8022/// return nullptr by reference.
8023static void
8024emitOffloadingArrays(CodeGenFunction &CGF,
8025                     MappableExprsHandler::MapBaseValuesArrayTy &BasePointers,
8026                     MappableExprsHandler::MapValuesArrayTy &Pointers,
8027                     MappableExprsHandler::MapValuesArrayTy &Sizes,
8028                     MappableExprsHandler::MapFlagsArrayTy &MapTypes,
8029                     CGOpenMPRuntime::TargetDataInfo &Info) {
8030  CodeGenModule &CGM = CGF.CGM;
8031  ASTContext &Ctx = CGF.getContext();
8032
8033  // Reset the array information.
8034  Info.clearArrayInfo();
8035  Info.NumberOfPtrs = BasePointers.size();
8036
8037  if (Info.NumberOfPtrs) {
8038    // Detect if we have any capture size requiring runtime evaluation of the
8039    // size so that a constant array could be eventually used.
8040    bool hasRuntimeEvaluationCaptureSize = false;
8041    for (llvm::Value *S : Sizes)
8042      if (!isa<llvm::Constant>(S)) {
8043        hasRuntimeEvaluationCaptureSize = true;
8044        break;
8045      }
8046
8047    llvm::APInt PointerNumAP(32, Info.NumberOfPtrs, /*isSigned=*/true);
8048    QualType PointerArrayType =
8049        Ctx.getConstantArrayType(Ctx.VoidPtrTy, PointerNumAP, ArrayType::Normal,
8050                                 /*IndexTypeQuals=*/0);
8051
8052    Info.BasePointersArray =
8053        CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer();
8054    Info.PointersArray =
8055        CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer();
8056
8057    // If we don't have any VLA types or other types that require runtime
8058    // evaluation, we can use a constant array for the map sizes, otherwise we
8059    // need to fill up the arrays as we do for the pointers.
8060    if (hasRuntimeEvaluationCaptureSize) {
8061      QualType SizeArrayType = Ctx.getConstantArrayType(
8062          Ctx.getSizeType(), PointerNumAP, ArrayType::Normal,
8063          /*IndexTypeQuals=*/0);
8064      Info.SizesArray =
8065          CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer();
8066    } else {
8067      // We expect all the sizes to be constant, so we collect them to create
8068      // a constant array.
8069      SmallVector<llvm::Constant *, 16> ConstSizes;
8070      for (llvm::Value *S : Sizes)
8071        ConstSizes.push_back(cast<llvm::Constant>(S));
8072
8073      auto *SizesArrayInit = llvm::ConstantArray::get(
8074          llvm::ArrayType::get(CGM.SizeTy, ConstSizes.size()), ConstSizes);
8075      std::string Name = CGM.getOpenMPRuntime().getName({"offload_sizes"});
8076      auto *SizesArrayGbl = new llvm::GlobalVariable(
8077          CGM.getModule(), SizesArrayInit->getType(),
8078          /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage,
8079          SizesArrayInit, Name);
8080      SizesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
8081      Info.SizesArray = SizesArrayGbl;
8082    }
8083
8084    // The map types are always constant so we don't need to generate code to
8085    // fill arrays. Instead, we create an array constant.
8086    SmallVector<uint64_t, 4> Mapping(MapTypes.size(), 0);
8087    llvm::copy(MapTypes, Mapping.begin());
8088    llvm::Constant *MapTypesArrayInit =
8089        llvm::ConstantDataArray::get(CGF.Builder.getContext(), Mapping);
8090    std::string MaptypesName =
8091        CGM.getOpenMPRuntime().getName({"offload_maptypes"});
8092    auto *MapTypesArrayGbl = new llvm::GlobalVariable(
8093        CGM.getModule(), MapTypesArrayInit->getType(),
8094        /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage,
8095        MapTypesArrayInit, MaptypesName);
8096    MapTypesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
8097    Info.MapTypesArray = MapTypesArrayGbl;
8098
8099    for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
8100      llvm::Value *BPVal = *BasePointers[I];
8101      llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32(
8102          llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
8103          Info.BasePointersArray, 0, I);
8104      BP = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
8105          BP, BPVal->getType()->getPointerTo(/*AddrSpace=*/0));
8106      Address BPAddr(BP, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
8107      CGF.Builder.CreateStore(BPVal, BPAddr);
8108
8109      if (Info.requiresDevicePointerInfo())
8110        if (const ValueDecl *DevVD = BasePointers[I].getDevicePtrDecl())
8111          Info.CaptureDeviceAddrMap.try_emplace(DevVD, BPAddr);
8112
8113      llvm::Value *PVal = Pointers[I];
8114      llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32(
8115          llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
8116          Info.PointersArray, 0, I);
8117      P = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
8118          P, PVal->getType()->getPointerTo(/*AddrSpace=*/0));
8119      Address PAddr(P, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
8120      CGF.Builder.CreateStore(PVal, PAddr);
8121
8122      if (hasRuntimeEvaluationCaptureSize) {
8123        llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32(
8124            llvm::ArrayType::get(CGM.SizeTy, Info.NumberOfPtrs),
8125            Info.SizesArray,
8126            /*Idx0=*/0,
8127            /*Idx1=*/I);
8128        Address SAddr(S, Ctx.getTypeAlignInChars(Ctx.getSizeType()));
8129        CGF.Builder.CreateStore(
8130            CGF.Builder.CreateIntCast(Sizes[I], CGM.SizeTy, /*isSigned=*/true),
8131            SAddr);
8132      }
8133    }
8134  }
8135}
8136/// Emit the arguments to be passed to the runtime library based on the
8137/// arrays of pointers, sizes and map types.
8138static void emitOffloadingArraysArgument(
8139    CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg,
8140    llvm::Value *&PointersArrayArg, llvm::Value *&SizesArrayArg,
8141    llvm::Value *&MapTypesArrayArg, CGOpenMPRuntime::TargetDataInfo &Info) {
8142  CodeGenModule &CGM = CGF.CGM;
8143  if (Info.NumberOfPtrs) {
8144    BasePointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
8145        llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
8146        Info.BasePointersArray,
8147        /*Idx0=*/0, /*Idx1=*/0);
8148    PointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
8149        llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
8150        Info.PointersArray,
8151        /*Idx0=*/0,
8152        /*Idx1=*/0);
8153    SizesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
8154        llvm::ArrayType::get(CGM.SizeTy, Info.NumberOfPtrs), Info.SizesArray,
8155        /*Idx0=*/0, /*Idx1=*/0);
8156    MapTypesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
8157        llvm::ArrayType::get(CGM.Int64Ty, Info.NumberOfPtrs),
8158        Info.MapTypesArray,
8159        /*Idx0=*/0,
8160        /*Idx1=*/0);
8161  } else {
8162    BasePointersArrayArg = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy);
8163    PointersArrayArg = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy);
8164    SizesArrayArg = llvm::ConstantPointerNull::get(CGM.SizeTy->getPointerTo());
8165    MapTypesArrayArg =
8166        llvm::ConstantPointerNull::get(CGM.Int64Ty->getPointerTo());
8167  }
8168}
8169
8170/// Checks if the expression is constant or does not have non-trivial function
8171/// calls.
8172static bool isTrivial(ASTContext &Ctx, const Expr * E) {
8173  // We can skip constant expressions.
8174  // We can skip expressions with trivial calls or simple expressions.
8175  return (E->isEvaluatable(Ctx, Expr::SE_AllowUndefinedBehavior) ||
8176          !E->hasNonTrivialCall(Ctx)) &&
8177         !E->HasSideEffects(Ctx, /*IncludePossibleEffects=*/true);
8178}
8179
8180/// Checks if the \p Body is the \a CompoundStmt and returns its child statement
8181/// iff there is only one that is not evaluatable at the compile time.
8182static const Stmt *getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body) {
8183  if (const auto *C = dyn_cast<CompoundStmt>(Body)) {
8184    const Stmt *Child = nullptr;
8185    for (const Stmt *S : C->body()) {
8186      if (const auto *E = dyn_cast<Expr>(S)) {
8187        if (isTrivial(Ctx, E))
8188          continue;
8189      }
8190      // Some of the statements can be ignored.
8191      if (isa<AsmStmt>(S) || isa<NullStmt>(S) || isa<OMPFlushDirective>(S) ||
8192          isa<OMPBarrierDirective>(S) || isa<OMPTaskyieldDirective>(S))
8193        continue;
8194      // Analyze declarations.
8195      if (const auto *DS = dyn_cast<DeclStmt>(S)) {
8196        if (llvm::all_of(DS->decls(), [&Ctx](const Decl *D) {
8197              if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||
8198                  isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||
8199                  isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||
8200                  isa<UsingDirectiveDecl>(D) ||
8201                  isa<OMPDeclareReductionDecl>(D) ||
8202                  isa<OMPThreadPrivateDecl>(D))
8203                return true;
8204              const auto *VD = dyn_cast<VarDecl>(D);
8205              if (!VD)
8206                return false;
8207              return VD->isConstexpr() ||
8208                     ((VD->getType().isTrivialType(Ctx) ||
8209                       VD->getType()->isReferenceType()) &&
8210                      (!VD->hasInit() || isTrivial(Ctx, VD->getInit())));
8211            }))
8212          continue;
8213      }
8214      // Found multiple children - cannot get the one child only.
8215      if (Child)
8216        return Body;
8217      Child = S;
8218    }
8219    if (Child)
8220      return Child;
8221  }
8222  return Body;
8223}
8224
8225/// Check for inner distribute directive.
8226static const OMPExecutableDirective *
8227getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) {
8228  const auto *CS = D.getInnermostCapturedStmt();
8229  const auto *Body =
8230      CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
8231  const Stmt *ChildStmt = getSingleCompoundChild(Ctx, Body);
8232
8233  if (const auto *NestedDir = dyn_cast<OMPExecutableDirective>(ChildStmt)) {
8234    OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind();
8235    switch (D.getDirectiveKind()) {
8236    case OMPD_target:
8237      if (isOpenMPDistributeDirective(DKind))
8238        return NestedDir;
8239      if (DKind == OMPD_teams) {
8240        Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
8241            /*IgnoreCaptured=*/true);
8242        if (!Body)
8243          return nullptr;
8244        ChildStmt = getSingleCompoundChild(Ctx, Body);
8245        if (const auto *NND = dyn_cast<OMPExecutableDirective>(ChildStmt)) {
8246          DKind = NND->getDirectiveKind();
8247          if (isOpenMPDistributeDirective(DKind))
8248            return NND;
8249        }
8250      }
8251      return nullptr;
8252    case OMPD_target_teams:
8253      if (isOpenMPDistributeDirective(DKind))
8254        return NestedDir;
8255      return nullptr;
8256    case OMPD_target_parallel:
8257    case OMPD_target_simd:
8258    case OMPD_target_parallel_for:
8259    case OMPD_target_parallel_for_simd:
8260      return nullptr;
8261    case OMPD_target_teams_distribute:
8262    case OMPD_target_teams_distribute_simd:
8263    case OMPD_target_teams_distribute_parallel_for:
8264    case OMPD_target_teams_distribute_parallel_for_simd:
8265    case OMPD_parallel:
8266    case OMPD_for:
8267    case OMPD_parallel_for:
8268    case OMPD_parallel_sections:
8269    case OMPD_for_simd:
8270    case OMPD_parallel_for_simd:
8271    case OMPD_cancel:
8272    case OMPD_cancellation_point:
8273    case OMPD_ordered:
8274    case OMPD_threadprivate:
8275    case OMPD_allocate:
8276    case OMPD_task:
8277    case OMPD_simd:
8278    case OMPD_sections:
8279    case OMPD_section:
8280    case OMPD_single:
8281    case OMPD_master:
8282    case OMPD_critical:
8283    case OMPD_taskyield:
8284    case OMPD_barrier:
8285    case OMPD_taskwait:
8286    case OMPD_taskgroup:
8287    case OMPD_atomic:
8288    case OMPD_flush:
8289    case OMPD_teams:
8290    case OMPD_target_data:
8291    case OMPD_target_exit_data:
8292    case OMPD_target_enter_data:
8293    case OMPD_distribute:
8294    case OMPD_distribute_simd:
8295    case OMPD_distribute_parallel_for:
8296    case OMPD_distribute_parallel_for_simd:
8297    case OMPD_teams_distribute:
8298    case OMPD_teams_distribute_simd:
8299    case OMPD_teams_distribute_parallel_for:
8300    case OMPD_teams_distribute_parallel_for_simd:
8301    case OMPD_target_update:
8302    case OMPD_declare_simd:
8303    case OMPD_declare_target:
8304    case OMPD_end_declare_target:
8305    case OMPD_declare_reduction:
8306    case OMPD_declare_mapper:
8307    case OMPD_taskloop:
8308    case OMPD_taskloop_simd:
8309    case OMPD_requires:
8310    case OMPD_unknown:
8311      llvm_unreachable("Unexpected directive.");
8312    }
8313  }
8314
8315  return nullptr;
8316}
8317
8318void CGOpenMPRuntime::emitTargetNumIterationsCall(
8319    CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *Device,
8320    const llvm::function_ref<llvm::Value *(
8321        CodeGenFunction &CGF, const OMPLoopDirective &D)> &SizeEmitter) {
8322  OpenMPDirectiveKind Kind = D.getDirectiveKind();
8323  const OMPExecutableDirective *TD = &D;
8324  // Get nested teams distribute kind directive, if any.
8325  if (!isOpenMPDistributeDirective(Kind) || !isOpenMPTeamsDirective(Kind))
8326    TD = getNestedDistributeDirective(CGM.getContext(), D);
8327  if (!TD)
8328    return;
8329  const auto *LD = cast<OMPLoopDirective>(TD);
8330  auto &&CodeGen = [LD, &Device, &SizeEmitter, this](CodeGenFunction &CGF,
8331                                                     PrePostActionTy &) {
8332    llvm::Value *NumIterations = SizeEmitter(CGF, *LD);
8333
8334    // Emit device ID if any.
8335    llvm::Value *DeviceID;
8336    if (Device)
8337      DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
8338                                           CGF.Int64Ty, /*isSigned=*/true);
8339    else
8340      DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
8341
8342    llvm::Value *Args[] = {DeviceID, NumIterations};
8343    CGF.EmitRuntimeCall(
8344        createRuntimeFunction(OMPRTL__kmpc_push_target_tripcount), Args);
8345  };
8346  emitInlinedDirective(CGF, OMPD_unknown, CodeGen);
8347}
8348
8349void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF,
8350                                     const OMPExecutableDirective &D,
8351                                     llvm::Function *OutlinedFn,
8352                                     llvm::Value *OutlinedFnID,
8353                                     const Expr *IfCond, const Expr *Device) {
8354  if (!CGF.HaveInsertPoint())
8355    return;
8356
8357   (0) . __assert_fail ("OutlinedFn && \"Invalid outlined function!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8357, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(OutlinedFn && "Invalid outlined function!");
8358
8359  const bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>();
8360  llvm::SmallVector<llvm::Value *, 16> CapturedVars;
8361  const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
8362  auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF,
8363                                            PrePostActionTy &) {
8364    CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
8365  };
8366  emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen);
8367
8368  CodeGenFunction::OMPTargetDataInfo InputInfo;
8369  llvm::Value *MapTypesArray = nullptr;
8370  // Fill up the pointer arrays and transfer execution to the device.
8371  auto &&ThenGen = [this, Device, OutlinedFn, OutlinedFnID, &D, &InputInfo,
8372                    &MapTypesArray, &CS, RequiresOuterTask,
8373                    &CapturedVars](CodeGenFunction &CGF, PrePostActionTy &) {
8374    // On top of the arrays that were filled up, the target offloading call
8375    // takes as arguments the device id as well as the host pointer. The host
8376    // pointer is used by the runtime library to identify the current target
8377    // region, so it only has to be unique and not necessarily point to
8378    // anything. It could be the pointer to the outlined function that
8379    // implements the target region, but we aren't using that so that the
8380    // compiler doesn't need to keep that, and could therefore inline the host
8381    // function if proven worthwhile during optimization.
8382
8383    // From this point on, we need to have an ID of the target region defined.
8384     (0) . __assert_fail ("OutlinedFnID && \"Invalid outlined function ID!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8384, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(OutlinedFnID && "Invalid outlined function ID!");
8385
8386    // Emit device ID if any.
8387    llvm::Value *DeviceID;
8388    if (Device) {
8389      DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
8390                                           CGF.Int64Ty, /*isSigned=*/true);
8391    } else {
8392      DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
8393    }
8394
8395    // Emit the number of elements in the offloading arrays.
8396    llvm::Value *PointerNum =
8397        CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
8398
8399    // Return value of the runtime offloading call.
8400    llvm::Value *Return;
8401
8402    llvm::Value *NumTeams = emitNumTeamsForTargetDirective(*this, CGF, D);
8403    llvm::Value *NumThreads = emitNumThreadsForTargetDirective(*this, CGF, D);
8404
8405    bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
8406    // The target region is an outlined function launched by the runtime
8407    // via calls __tgt_target() or __tgt_target_teams().
8408    //
8409    // __tgt_target() launches a target region with one team and one thread,
8410    // executing a serial region.  This master thread may in turn launch
8411    // more threads within its team upon encountering a parallel region,
8412    // however, no additional teams can be launched on the device.
8413    //
8414    // __tgt_target_teams() launches a target region with one or more teams,
8415    // each with one or more threads.  This call is required for target
8416    // constructs such as:
8417    //  'target teams'
8418    //  'target' / 'teams'
8419    //  'target teams distribute parallel for'
8420    //  'target parallel'
8421    // and so on.
8422    //
8423    // Note that on the host and CPU targets, the runtime implementation of
8424    // these calls simply call the outlined function without forking threads.
8425    // The outlined functions themselves have runtime calls to
8426    // __kmpc_fork_teams() and __kmpc_fork() for this purpose, codegen'd by
8427    // the compiler in emitTeamsCall() and emitParallelCall().
8428    //
8429    // In contrast, on the NVPTX target, the implementation of
8430    // __tgt_target_teams() launches a GPU kernel with the requested number
8431    // of teams and threads so no additional calls to the runtime are required.
8432    if (NumTeams) {
8433      // If we have NumTeams defined this means that we have an enclosed teams
8434      // region. Therefore we also expect to have NumThreads defined. These two
8435      // values should be defined in the presence of a teams directive,
8436      // regardless of having any clauses associated. If the user is using teams
8437      // but no clauses, these two values will be the default that should be
8438      // passed to the runtime library - a 32-bit integer with the value zero.
8439       (0) . __assert_fail ("NumThreads && \"Thread limit expression should be available along \" \"with number of teams.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8440, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(NumThreads && "Thread limit expression should be available along "
8440 (0) . __assert_fail ("NumThreads && \"Thread limit expression should be available along \" \"with number of teams.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8440, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">                           "with number of teams.");
8441      llvm::Value *OffloadingArgs[] = {DeviceID,
8442                                       OutlinedFnID,
8443                                       PointerNum,
8444                                       InputInfo.BasePointersArray.getPointer(),
8445                                       InputInfo.PointersArray.getPointer(),
8446                                       InputInfo.SizesArray.getPointer(),
8447                                       MapTypesArray,
8448                                       NumTeams,
8449                                       NumThreads};
8450      Return = CGF.EmitRuntimeCall(
8451          createRuntimeFunction(HasNowait ? OMPRTL__tgt_target_teams_nowait
8452                                          : OMPRTL__tgt_target_teams),
8453          OffloadingArgs);
8454    } else {
8455      llvm::Value *OffloadingArgs[] = {DeviceID,
8456                                       OutlinedFnID,
8457                                       PointerNum,
8458                                       InputInfo.BasePointersArray.getPointer(),
8459                                       InputInfo.PointersArray.getPointer(),
8460                                       InputInfo.SizesArray.getPointer(),
8461                                       MapTypesArray};
8462      Return = CGF.EmitRuntimeCall(
8463          createRuntimeFunction(HasNowait ? OMPRTL__tgt_target_nowait
8464                                          : OMPRTL__tgt_target),
8465          OffloadingArgs);
8466    }
8467
8468    // Check the error code and execute the host version if required.
8469    llvm::BasicBlock *OffloadFailedBlock =
8470        CGF.createBasicBlock("omp_offload.failed");
8471    llvm::BasicBlock *OffloadContBlock =
8472        CGF.createBasicBlock("omp_offload.cont");
8473    llvm::Value *Failed = CGF.Builder.CreateIsNotNull(Return);
8474    CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);
8475
8476    CGF.EmitBlock(OffloadFailedBlock);
8477    if (RequiresOuterTask) {
8478      CapturedVars.clear();
8479      CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
8480    }
8481    emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars);
8482    CGF.EmitBranch(OffloadContBlock);
8483
8484    CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true);
8485  };
8486
8487  // Notify that the host version must be executed.
8488  auto &&ElseGen = [this, &D, OutlinedFn, &CS, &CapturedVars,
8489                    RequiresOuterTask](CodeGenFunction &CGF,
8490                                       PrePostActionTy &) {
8491    if (RequiresOuterTask) {
8492      CapturedVars.clear();
8493      CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
8494    }
8495    emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars);
8496  };
8497
8498  auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray,
8499                          &CapturedVars, RequiresOuterTask,
8500                          &CS](CodeGenFunction &CGF, PrePostActionTy &) {
8501    // Fill up the arrays with all the captured variables.
8502    MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
8503    MappableExprsHandler::MapValuesArrayTy Pointers;
8504    MappableExprsHandler::MapValuesArrayTy Sizes;
8505    MappableExprsHandler::MapFlagsArrayTy MapTypes;
8506
8507    // Get mappable expression information.
8508    MappableExprsHandler MEHandler(D, CGF);
8509    llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
8510
8511    auto RI = CS.getCapturedRecordDecl()->field_begin();
8512    auto CV = CapturedVars.begin();
8513    for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(),
8514                                              CE = CS.capture_end();
8515         CI != CE; ++CI, ++RI, ++CV) {
8516      MappableExprsHandler::MapBaseValuesArrayTy CurBasePointers;
8517      MappableExprsHandler::MapValuesArrayTy CurPointers;
8518      MappableExprsHandler::MapValuesArrayTy CurSizes;
8519      MappableExprsHandler::MapFlagsArrayTy CurMapTypes;
8520      MappableExprsHandler::StructRangeInfoTy PartialStruct;
8521
8522      // VLA sizes are passed to the outlined region by copy and do not have map
8523      // information associated.
8524      if (CI->capturesVariableArrayType()) {
8525        CurBasePointers.push_back(*CV);
8526        CurPointers.push_back(*CV);
8527        CurSizes.push_back(CGF.getTypeSize(RI->getType()));
8528        // Copy to the device as an argument. No need to retrieve it.
8529        CurMapTypes.push_back(MappableExprsHandler::OMP_MAP_LITERAL |
8530                              MappableExprsHandler::OMP_MAP_TARGET_PARAM);
8531      } else {
8532        // If we have any information in the map clause, we use it, otherwise we
8533        // just do a default mapping.
8534        MEHandler.generateInfoForCapture(CI, *CV, CurBasePointers, CurPointers,
8535                                         CurSizes, CurMapTypes, PartialStruct);
8536        if (CurBasePointers.empty())
8537          MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurBasePointers,
8538                                           CurPointers, CurSizes, CurMapTypes);
8539        // Generate correct mapping for variables captured by reference in
8540        // lambdas.
8541        if (CI->capturesVariable())
8542          MEHandler.generateInfoForLambdaCaptures(
8543              CI->getCapturedVar(), *CV, CurBasePointers, CurPointers, CurSizes,
8544              CurMapTypes, LambdaPointers);
8545      }
8546      // We expect to have at least an element of information for this capture.
8547       (0) . __assert_fail ("!CurBasePointers.empty() && \"Non-existing map pointer for capture!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8548, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!CurBasePointers.empty() &&
8548 (0) . __assert_fail ("!CurBasePointers.empty() && \"Non-existing map pointer for capture!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8548, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">             "Non-existing map pointer for capture!");
8549       (0) . __assert_fail ("CurBasePointers.size() == CurPointers.size() && CurBasePointers.size() == CurSizes.size() && CurBasePointers.size() == CurMapTypes.size() && \"Inconsistent map information sizes!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8552, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(CurBasePointers.size() == CurPointers.size() &&
8550 (0) . __assert_fail ("CurBasePointers.size() == CurPointers.size() && CurBasePointers.size() == CurSizes.size() && CurBasePointers.size() == CurMapTypes.size() && \"Inconsistent map information sizes!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8552, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">             CurBasePointers.size() == CurSizes.size() &&
8551 (0) . __assert_fail ("CurBasePointers.size() == CurPointers.size() && CurBasePointers.size() == CurSizes.size() && CurBasePointers.size() == CurMapTypes.size() && \"Inconsistent map information sizes!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8552, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">             CurBasePointers.size() == CurMapTypes.size() &&
8552 (0) . __assert_fail ("CurBasePointers.size() == CurPointers.size() && CurBasePointers.size() == CurSizes.size() && CurBasePointers.size() == CurMapTypes.size() && \"Inconsistent map information sizes!\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8552, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">             "Inconsistent map information sizes!");
8553
8554      // If there is an entry in PartialStruct it means we have a struct with
8555      // individual members mapped. Emit an extra combined entry.
8556      if (PartialStruct.Base.isValid())
8557        MEHandler.emitCombinedEntry(BasePointers, Pointers, Sizes, MapTypes,
8558                                    CurMapTypes, PartialStruct);
8559
8560      // We need to append the results of this capture to what we already have.
8561      BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
8562      Pointers.append(CurPointers.begin(), CurPointers.end());
8563      Sizes.append(CurSizes.begin(), CurSizes.end());
8564      MapTypes.append(CurMapTypes.begin(), CurMapTypes.end());
8565    }
8566    // Adjust MEMBER_OF flags for the lambdas captures.
8567    MEHandler.adjustMemberOfForLambdaCaptures(LambdaPointers, BasePointers,
8568                                              Pointers, MapTypes);
8569    // Map other list items in the map clause which are not captured variables
8570    // but "declare target link" global variables.
8571    MEHandler.generateInfoForDeclareTargetLink(BasePointers, Pointers, Sizes,
8572                                               MapTypes);
8573
8574    TargetDataInfo Info;
8575    // Fill up the arrays and create the arguments.
8576    emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
8577    emitOffloadingArraysArgument(CGF, Info.BasePointersArray,
8578                                 Info.PointersArray, Info.SizesArray,
8579                                 Info.MapTypesArray, Info);
8580    InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
8581    InputInfo.BasePointersArray =
8582        Address(Info.BasePointersArray, CGM.getPointerAlign());
8583    InputInfo.PointersArray =
8584        Address(Info.PointersArray, CGM.getPointerAlign());
8585    InputInfo.SizesArray = Address(Info.SizesArray, CGM.getPointerAlign());
8586    MapTypesArray = Info.MapTypesArray;
8587    if (RequiresOuterTask)
8588      CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
8589    else
8590      emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
8591  };
8592
8593  auto &&TargetElseGen = [this, &ElseGen, &D, RequiresOuterTask](
8594                             CodeGenFunction &CGF, PrePostActionTy &) {
8595    if (RequiresOuterTask) {
8596      CodeGenFunction::OMPTargetDataInfo InputInfo;
8597      CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo);
8598    } else {
8599      emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen);
8600    }
8601  };
8602
8603  // If we have a target function ID it means that we need to support
8604  // offloading, otherwise, just execute on the host. We need to execute on host
8605  // regardless of the conditional in the if clause if, e.g., the user do not
8606  // specify target triples.
8607  if (OutlinedFnID) {
8608    if (IfCond) {
8609      emitOMPIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
8610    } else {
8611      RegionCodeGenTy ThenRCG(TargetThenGen);
8612      ThenRCG(CGF);
8613    }
8614  } else {
8615    RegionCodeGenTy ElseRCG(TargetElseGen);
8616    ElseRCG(CGF);
8617  }
8618}
8619
8620void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
8621                                                    StringRef ParentName) {
8622  if (!S)
8623    return;
8624
8625  // Codegen OMP target directives that offload compute to the device.
8626  bool RequiresDeviceCodegen =
8627      isa<OMPExecutableDirective>(S) &&
8628      isOpenMPTargetExecutionDirective(
8629          cast<OMPExecutableDirective>(S)->getDirectiveKind());
8630
8631  if (RequiresDeviceCodegen) {
8632    const auto &E = *cast<OMPExecutableDirective>(S);
8633    unsigned DeviceID;
8634    unsigned FileID;
8635    unsigned Line;
8636    getTargetEntryUniqueInfo(CGM.getContext(), E.getBeginLoc(), DeviceID,
8637                             FileID, Line);
8638
8639    // Is this a target region that should not be emitted as an entry point? If
8640    // so just signal we are done with this target region.
8641    if (!OffloadEntriesInfoManager.hasTargetRegionEntryInfo(DeviceID, FileID,
8642                                                            ParentName, Line))
8643      return;
8644
8645    switch (E.getDirectiveKind()) {
8646    case OMPD_target:
8647      CodeGenFunction::EmitOMPTargetDeviceFunction(CGM, ParentName,
8648                                                   cast<OMPTargetDirective>(E));
8649      break;
8650    case OMPD_target_parallel:
8651      CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
8652          CGM, ParentName, cast<OMPTargetParallelDirective>(E));
8653      break;
8654    case OMPD_target_teams:
8655      CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
8656          CGM, ParentName, cast<OMPTargetTeamsDirective>(E));
8657      break;
8658    case OMPD_target_teams_distribute:
8659      CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
8660          CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(E));
8661      break;
8662    case OMPD_target_teams_distribute_simd:
8663      CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
8664          CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(E));
8665      break;
8666    case OMPD_target_parallel_for:
8667      CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
8668          CGM, ParentName, cast<OMPTargetParallelForDirective>(E));
8669      break;
8670    case OMPD_target_parallel_for_simd:
8671      CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
8672          CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(E));
8673      break;
8674    case OMPD_target_simd:
8675      CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
8676          CGM, ParentName, cast<OMPTargetSimdDirective>(E));
8677      break;
8678    case OMPD_target_teams_distribute_parallel_for:
8679      CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
8680          CGM, ParentName,
8681          cast<OMPTargetTeamsDistributeParallelForDirective>(E));
8682      break;
8683    case OMPD_target_teams_distribute_parallel_for_simd:
8684      CodeGenFunction::
8685          EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
8686              CGM, ParentName,
8687              cast<OMPTargetTeamsDistributeParallelForSimdDirective>(E));
8688      break;
8689    case OMPD_parallel:
8690    case OMPD_for:
8691    case OMPD_parallel_for:
8692    case OMPD_parallel_sections:
8693    case OMPD_for_simd:
8694    case OMPD_parallel_for_simd:
8695    case OMPD_cancel:
8696    case OMPD_cancellation_point:
8697    case OMPD_ordered:
8698    case OMPD_threadprivate:
8699    case OMPD_allocate:
8700    case OMPD_task:
8701    case OMPD_simd:
8702    case OMPD_sections:
8703    case OMPD_section:
8704    case OMPD_single:
8705    case OMPD_master:
8706    case OMPD_critical:
8707    case OMPD_taskyield:
8708    case OMPD_barrier:
8709    case OMPD_taskwait:
8710    case OMPD_taskgroup:
8711    case OMPD_atomic:
8712    case OMPD_flush:
8713    case OMPD_teams:
8714    case OMPD_target_data:
8715    case OMPD_target_exit_data:
8716    case OMPD_target_enter_data:
8717    case OMPD_distribute:
8718    case OMPD_distribute_simd:
8719    case OMPD_distribute_parallel_for:
8720    case OMPD_distribute_parallel_for_simd:
8721    case OMPD_teams_distribute:
8722    case OMPD_teams_distribute_simd:
8723    case OMPD_teams_distribute_parallel_for:
8724    case OMPD_teams_distribute_parallel_for_simd:
8725    case OMPD_target_update:
8726    case OMPD_declare_simd:
8727    case OMPD_declare_target:
8728    case OMPD_end_declare_target:
8729    case OMPD_declare_reduction:
8730    case OMPD_declare_mapper:
8731    case OMPD_taskloop:
8732    case OMPD_taskloop_simd:
8733    case OMPD_requires:
8734    case OMPD_unknown:
8735      llvm_unreachable("Unknown target directive for OpenMP device codegen.");
8736    }
8737    return;
8738  }
8739
8740  if (const auto *E = dyn_cast<OMPExecutableDirective>(S)) {
8741    if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
8742      return;
8743
8744    scanForTargetRegionsFunctions(
8745        E->getInnermostCapturedStmt()->getCapturedStmt(), ParentName);
8746    return;
8747  }
8748
8749  // If this is a lambda function, look into its body.
8750  if (const auto *L = dyn_cast<LambdaExpr>(S))
8751    S = L->getBody();
8752
8753  // Keep looking for target regions recursively.
8754  for (const Stmt *II : S->children())
8755    scanForTargetRegionsFunctions(II, ParentName);
8756}
8757
8758bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) {
8759  // If emitting code for the host, we do not process FD here. Instead we do
8760  // the normal code generation.
8761  if (!CGM.getLangOpts().OpenMPIsDevice)
8762    return false;
8763
8764  const ValueDecl *VD = cast<ValueDecl>(GD.getDecl());
8765  StringRef Name = CGM.getMangledName(GD);
8766  // Try to detect target regions in the function.
8767  if (const auto *FD = dyn_cast<FunctionDecl>(VD))
8768    scanForTargetRegionsFunctions(FD->getBody(), Name);
8769
8770  // Do not to emit function if it is not marked as declare target.
8771  return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
8772         AlreadyEmittedTargetFunctions.count(Name) == 0;
8773}
8774
8775bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
8776  if (!CGM.getLangOpts().OpenMPIsDevice)
8777    return false;
8778
8779  // Check if there are Ctors/Dtors in this declaration and look for target
8780  // regions in it. We use the complete variant to produce the kernel name
8781  // mangling.
8782  QualType RDTy = cast<VarDecl>(GD.getDecl())->getType();
8783  if (const auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) {
8784    for (const CXXConstructorDecl *Ctor : RD->ctors()) {
8785      StringRef ParentName =
8786          CGM.getMangledName(GlobalDecl(Ctor, Ctor_Complete));
8787      scanForTargetRegionsFunctions(Ctor->getBody(), ParentName);
8788    }
8789    if (const CXXDestructorDecl *Dtor = RD->getDestructor()) {
8790      StringRef ParentName =
8791          CGM.getMangledName(GlobalDecl(Dtor, Dtor_Complete));
8792      scanForTargetRegionsFunctions(Dtor->getBody(), ParentName);
8793    }
8794  }
8795
8796  // Do not to emit variable if it is not marked as declare target.
8797  llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
8798      OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
8799          cast<VarDecl>(GD.getDecl()));
8800  if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link) {
8801    DeferredGlobalVariables.insert(cast<VarDecl>(GD.getDecl()));
8802    return true;
8803  }
8804  return false;
8805}
8806
8807llvm::Constant *
8808CGOpenMPRuntime::registerTargetFirstprivateCopy(CodeGenFunction &CGF,
8809                                                const VarDecl *VD) {
8810   (0) . __assert_fail ("VD->getType().isConstant(CGM.getContext()) && \"Expected constant variable.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8811, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(VD->getType().isConstant(CGM.getContext()) &&
8811 (0) . __assert_fail ("VD->getType().isConstant(CGM.getContext()) && \"Expected constant variable.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8811, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">         "Expected constant variable.");
8812  StringRef VarName;
8813  llvm::Constant *Addr;
8814  llvm::GlobalValue::LinkageTypes Linkage;
8815  QualType Ty = VD->getType();
8816  SmallString<128> Buffer;
8817  {
8818    unsigned DeviceID;
8819    unsigned FileID;
8820    unsigned Line;
8821    getTargetEntryUniqueInfo(CGM.getContext(), VD->getLocation(), DeviceID,
8822                             FileID, Line);
8823    llvm::raw_svector_ostream OS(Buffer);
8824    OS << "__omp_offloading_firstprivate_" << llvm::format("_%x", DeviceID)
8825       << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
8826    VarName = OS.str();
8827  }
8828  Linkage = llvm::GlobalValue::InternalLinkage;
8829  Addr =
8830      getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(Ty), VarName,
8831                                  getDefaultFirstprivateAddressSpace());
8832  cast<llvm::GlobalValue>(Addr)->setLinkage(Linkage);
8833  CharUnits VarSize = CGM.getContext().getTypeSizeInChars(Ty);
8834  CGM.addCompilerUsedGlobal(cast<llvm::GlobalValue>(Addr));
8835  OffloadEntriesInfoManager.registerDeviceGlobalVarEntryInfo(
8836      VarName, Addr, VarSize,
8837      OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo, Linkage);
8838  return Addr;
8839}
8840
8841void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD,
8842                                                   llvm::Constant *Addr) {
8843  llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
8844      OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
8845  if (!Res) {
8846    if (CGM.getLangOpts().OpenMPIsDevice) {
8847      // Register non-target variables being emitted in device code (debug info
8848      // may cause this).
8849      StringRef VarName = CGM.getMangledName(VD);
8850      EmittedNonTargetVariables.try_emplace(VarName, Addr);
8851    }
8852    return;
8853  }
8854  // Register declare target variables.
8855  OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags;
8856  StringRef VarName;
8857  CharUnits VarSize;
8858  llvm::GlobalValue::LinkageTypes Linkage;
8859  switch (*Res) {
8860  case OMPDeclareTargetDeclAttr::MT_To:
8861    Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo;
8862    VarName = CGM.getMangledName(VD);
8863    if (VD->hasDefinition(CGM.getContext()) != VarDecl::DeclarationOnly) {
8864      VarSize = CGM.getContext().getTypeSizeInChars(VD->getType());
8865       (0) . __assert_fail ("!VarSize.isZero() && \"Expected non-zero size of the variable\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8865, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!VarSize.isZero() && "Expected non-zero size of the variable");
8866    } else {
8867      VarSize = CharUnits::Zero();
8868    }
8869    Linkage = CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false);
8870    // Temp solution to prevent optimizations of the internal variables.
8871    if (CGM.getLangOpts().OpenMPIsDevice && !VD->isExternallyVisible()) {
8872      std::string RefName = getName({VarName, "ref"});
8873      if (!CGM.GetGlobalValue(RefName)) {
8874        llvm::Constant *AddrRef =
8875            getOrCreateInternalVariable(Addr->getType(), RefName);
8876        auto *GVAddrRef = cast<llvm::GlobalVariable>(AddrRef);
8877        GVAddrRef->setConstant(/*Val=*/true);
8878        GVAddrRef->setLinkage(llvm::GlobalValue::InternalLinkage);
8879        GVAddrRef->setInitializer(Addr);
8880        CGM.addCompilerUsedGlobal(GVAddrRef);
8881      }
8882    }
8883    break;
8884  case OMPDeclareTargetDeclAttr::MT_Link:
8885    Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink;
8886    if (CGM.getLangOpts().OpenMPIsDevice) {
8887      VarName = Addr->getName();
8888      Addr = nullptr;
8889    } else {
8890      VarName = getAddrOfDeclareTargetLink(VD).getName();
8891      Addr = cast<llvm::Constant>(getAddrOfDeclareTargetLink(VD).getPointer());
8892    }
8893    VarSize = CGM.getPointerSize();
8894    Linkage = llvm::GlobalValue::WeakAnyLinkage;
8895    break;
8896  }
8897  OffloadEntriesInfoManager.registerDeviceGlobalVarEntryInfo(
8898      VarName, Addr, VarSize, Flags, Linkage);
8899}
8900
8901bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) {
8902  if (isa<FunctionDecl>(GD.getDecl()) ||
8903      isa<OMPDeclareReductionDecl>(GD.getDecl()))
8904    return emitTargetFunctions(GD);
8905
8906  return emitTargetGlobalVariable(GD);
8907}
8908
8909void CGOpenMPRuntime::emitDeferredTargetDecls() const {
8910  for (const VarDecl *VD : DeferredGlobalVariables) {
8911    llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
8912        OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
8913    if (!Res)
8914      continue;
8915    if (*Res == OMPDeclareTargetDeclAttr::MT_To) {
8916      CGM.EmitGlobal(VD);
8917    } else {
8918       (0) . __assert_fail ("*Res == OMPDeclareTargetDeclAttr..MT_Link && \"Expected to or link clauses.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8919, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(*Res == OMPDeclareTargetDeclAttr::MT_Link &&
8919 (0) . __assert_fail ("*Res == OMPDeclareTargetDeclAttr..MT_Link && \"Expected to or link clauses.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8919, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">             "Expected to or link clauses.");
8920      (void)CGM.getOpenMPRuntime().getAddrOfDeclareTargetLink(VD);
8921    }
8922  }
8923}
8924
8925void CGOpenMPRuntime::adjustTargetSpecificDataForLambdas(
8926    CodeGenFunction &CGF, const OMPExecutableDirective &D) const {
8927   (0) . __assert_fail ("isOpenMPTargetExecutionDirective(D.getDirectiveKind()) && \" Expected target-based directive.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8928, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(isOpenMPTargetExecutionDirective(D.getDirectiveKind()) &&
8928 (0) . __assert_fail ("isOpenMPTargetExecutionDirective(D.getDirectiveKind()) && \" Expected target-based directive.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 8928, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">         " Expected target-based directive.");
8929}
8930
8931bool CGOpenMPRuntime::hasAllocateAttributeForGlobalVar(const VarDecl *VD,
8932                                                       LangAS &AS) {
8933  if (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())
8934    return false;
8935  const auto *A = VD->getAttr<OMPAllocateDeclAttr>();
8936  switch(A->getAllocatorType()) {
8937  case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
8938  // Not supported, fallback to the default mem space.
8939  case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
8940  case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
8941  case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
8942  case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
8943  case OMPAllocateDeclAttr::OMPThreadMemAlloc:
8944  case OMPAllocateDeclAttr::OMPConstMemAlloc:
8945  case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
8946    AS = LangAS::Default;
8947    return true;
8948  case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
8949    llvm_unreachable("Expected predefined allocator for the variables with the "
8950                     "static storage.");
8951  }
8952  return false;
8953}
8954
8955CGOpenMPRuntime::DisableAutoDeclareTargetRAII::DisableAutoDeclareTargetRAII(
8956    CodeGenModule &CGM)
8957    : CGM(CGM) {
8958  if (CGM.getLangOpts().OpenMPIsDevice) {
8959    SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal;
8960    CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false;
8961  }
8962}
8963
8964CGOpenMPRuntime::DisableAutoDeclareTargetRAII::~DisableAutoDeclareTargetRAII() {
8965  if (CGM.getLangOpts().OpenMPIsDevice)
8966    CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal;
8967}
8968
8969bool CGOpenMPRuntime::markAsGlobalTarget(GlobalDecl GD) {
8970  if (!CGM.getLangOpts().OpenMPIsDevice || !ShouldMarkAsGlobal)
8971    return true;
8972
8973  StringRef Name = CGM.getMangledName(GD);
8974  const auto *D = cast<FunctionDecl>(GD.getDecl());
8975  // Do not to emit function if it is marked as declare target as it was already
8976  // emitted.
8977  if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) {
8978    if (D->hasBody() && AlreadyEmittedTargetFunctions.count(Name) == 0) {
8979      if (auto *F = dyn_cast_or_null<llvm::Function>(CGM.GetGlobalValue(Name)))
8980        return !F->isDeclaration();
8981      return false;
8982    }
8983    return true;
8984  }
8985
8986  return !AlreadyEmittedTargetFunctions.insert(Name).second;
8987}
8988
8989llvm::Function *CGOpenMPRuntime::emitRegistrationFunction() {
8990  // If we have offloading in the current module, we need to emit the entries
8991  // now and register the offloading descriptor.
8992  createOffloadEntriesAndInfoMetadata();
8993
8994  // Create and register the offloading binary descriptors. This is the main
8995  // entity that captures all the information about offloading in the current
8996  // compilation unit.
8997  return createOffloadingBinaryDescriptorRegistration();
8998}
8999
9000void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,
9001                                    const OMPExecutableDirective &D,
9002                                    SourceLocation Loc,
9003                                    llvm::Function *OutlinedFn,
9004                                    ArrayRef<llvm::Value *> CapturedVars) {
9005  if (!CGF.HaveInsertPoint())
9006    return;
9007
9008  llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
9009  CodeGenFunction::RunCleanupsScope Scope(CGF);
9010
9011  // Build call __kmpc_fork_teams(loc, n, microtask, var1, .., varn);
9012  llvm::Value *Args[] = {
9013      RTLoc,
9014      CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
9015      CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())};
9016  llvm::SmallVector<llvm::Value *, 16> RealArgs;
9017  RealArgs.append(std::begin(Args), std::end(Args));
9018  RealArgs.append(CapturedVars.begin(), CapturedVars.end());
9019
9020  llvm::FunctionCallee RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_teams);
9021  CGF.EmitRuntimeCall(RTLFn, RealArgs);
9022}
9023
9024void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
9025                                         const Expr *NumTeams,
9026                                         const Expr *ThreadLimit,
9027                                         SourceLocation Loc) {
9028  if (!CGF.HaveInsertPoint())
9029    return;
9030
9031  llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
9032
9033  llvm::Value *NumTeamsVal =
9034      NumTeams
9035          ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams),
9036                                      CGF.CGM.Int32Ty, /* isSigned = */ true)
9037          : CGF.Builder.getInt32(0);
9038
9039  llvm::Value *ThreadLimitVal =
9040      ThreadLimit
9041          ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit),
9042                                      CGF.CGM.Int32Ty, /* isSigned = */ true)
9043          : CGF.Builder.getInt32(0);
9044
9045  // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit)
9046  llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
9047                                     ThreadLimitVal};
9048  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_teams),
9049                      PushNumTeamsArgs);
9050}
9051
9052void CGOpenMPRuntime::emitTargetDataCalls(
9053    CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
9054    const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info) {
9055  if (!CGF.HaveInsertPoint())
9056    return;
9057
9058  // Action used to replace the default codegen action and turn privatization
9059  // off.
9060  PrePostActionTy NoPrivAction;
9061
9062  // Generate the code for the opening of the data environment. Capture all the
9063  // arguments of the runtime call by reference because they are used in the
9064  // closing of the region.
9065  auto &&BeginThenGen = [this, &D, Device, &Info,
9066                         &CodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
9067    // Fill up the arrays with all the mapped variables.
9068    MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
9069    MappableExprsHandler::MapValuesArrayTy Pointers;
9070    MappableExprsHandler::MapValuesArrayTy Sizes;
9071    MappableExprsHandler::MapFlagsArrayTy MapTypes;
9072
9073    // Get map clause information.
9074    MappableExprsHandler MCHandler(D, CGF);
9075    MCHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
9076
9077    // Fill up the arrays and create the arguments.
9078    emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
9079
9080    llvm::Value *BasePointersArrayArg = nullptr;
9081    llvm::Value *PointersArrayArg = nullptr;
9082    llvm::Value *SizesArrayArg = nullptr;
9083    llvm::Value *MapTypesArrayArg = nullptr;
9084    emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg,
9085                                 SizesArrayArg, MapTypesArrayArg, Info);
9086
9087    // Emit device ID if any.
9088    llvm::Value *DeviceID = nullptr;
9089    if (Device) {
9090      DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
9091                                           CGF.Int64Ty, /*isSigned=*/true);
9092    } else {
9093      DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
9094    }
9095
9096    // Emit the number of elements in the offloading arrays.
9097    llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
9098
9099    llvm::Value *OffloadingArgs[] = {
9100        DeviceID,         PointerNum,    BasePointersArrayArg,
9101        PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
9102    CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target_data_begin),
9103                        OffloadingArgs);
9104
9105    // If device pointer privatization is required, emit the body of the region
9106    // here. It will have to be duplicated: with and without privatization.
9107    if (!Info.CaptureDeviceAddrMap.empty())
9108      CodeGen(CGF);
9109  };
9110
9111  // Generate code for the closing of the data region.
9112  auto &&EndThenGen = [this, Device, &Info](CodeGenFunction &CGF,
9113                                            PrePostActionTy &) {
9114     (0) . __assert_fail ("Info.isValid() && \"Invalid data environment closing arguments.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 9114, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Info.isValid() && "Invalid data environment closing arguments.");
9115
9116    llvm::Value *BasePointersArrayArg = nullptr;
9117    llvm::Value *PointersArrayArg = nullptr;
9118    llvm::Value *SizesArrayArg = nullptr;
9119    llvm::Value *MapTypesArrayArg = nullptr;
9120    emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg,
9121                                 SizesArrayArg, MapTypesArrayArg, Info);
9122
9123    // Emit device ID if any.
9124    llvm::Value *DeviceID = nullptr;
9125    if (Device) {
9126      DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
9127                                           CGF.Int64Ty, /*isSigned=*/true);
9128    } else {
9129      DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
9130    }
9131
9132    // Emit the number of elements in the offloading arrays.
9133    llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
9134
9135    llvm::Value *OffloadingArgs[] = {
9136        DeviceID,         PointerNum,    BasePointersArrayArg,
9137        PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
9138    CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target_data_end),
9139                        OffloadingArgs);
9140  };
9141
9142  // If we need device pointer privatization, we need to emit the body of the
9143  // region with no privatization in the 'else' branch of the conditional.
9144  // Otherwise, we don't have to do anything.
9145  auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](CodeGenFunction &CGF,
9146                                                         PrePostActionTy &) {
9147    if (!Info.CaptureDeviceAddrMap.empty()) {
9148      CodeGen.setAction(NoPrivAction);
9149      CodeGen(CGF);
9150    }
9151  };
9152
9153  // We don't have to do anything to close the region if the if clause evaluates
9154  // to false.
9155  auto &&EndElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {};
9156
9157  if (IfCond) {
9158    emitOMPIfClause(CGF, IfCond, BeginThenGen, BeginElseGen);
9159  } else {
9160    RegionCodeGenTy RCG(BeginThenGen);
9161    RCG(CGF);
9162  }
9163
9164  // If we don't require privatization of device pointers, we emit the body in
9165  // between the runtime calls. This avoids duplicating the body code.
9166  if (Info.CaptureDeviceAddrMap.empty()) {
9167    CodeGen.setAction(NoPrivAction);
9168    CodeGen(CGF);
9169  }
9170
9171  if (IfCond) {
9172    emitOMPIfClause(CGF, IfCond, EndThenGen, EndElseGen);
9173  } else {
9174    RegionCodeGenTy RCG(EndThenGen);
9175    RCG(CGF);
9176  }
9177}
9178
9179void CGOpenMPRuntime::emitTargetDataStandAloneCall(
9180    CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
9181    const Expr *Device) {
9182  if (!CGF.HaveInsertPoint())
9183    return;
9184
9185   (0) . __assert_fail ("(isa(D) || isa(D) || isa(D)) && \"Expecting either target enter, exit data, or update directives.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 9188, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert((isa<OMPTargetEnterDataDirective>(D) ||
9186 (0) . __assert_fail ("(isa(D) || isa(D) || isa(D)) && \"Expecting either target enter, exit data, or update directives.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 9188, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">          isa<OMPTargetExitDataDirective>(D) ||
9187 (0) . __assert_fail ("(isa(D) || isa(D) || isa(D)) && \"Expecting either target enter, exit data, or update directives.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 9188, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">          isa<OMPTargetUpdateDirective>(D)) &&
9188 (0) . __assert_fail ("(isa(D) || isa(D) || isa(D)) && \"Expecting either target enter, exit data, or update directives.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 9188, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">         "Expecting either target enter, exit data, or update directives.");
9189
9190  CodeGenFunction::OMPTargetDataInfo InputInfo;
9191  llvm::Value *MapTypesArray = nullptr;
9192  // Generate the code for the opening of the data environment.
9193  auto &&ThenGen = [this, &D, Device, &InputInfo,
9194                    &MapTypesArray](CodeGenFunction &CGF, PrePostActionTy &) {
9195    // Emit device ID if any.
9196    llvm::Value *DeviceID = nullptr;
9197    if (Device) {
9198      DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
9199                                           CGF.Int64Ty, /*isSigned=*/true);
9200    } else {
9201      DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
9202    }
9203
9204    // Emit the number of elements in the offloading arrays.
9205    llvm::Constant *PointerNum =
9206        CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
9207
9208    llvm::Value *OffloadingArgs[] = {DeviceID,
9209                                     PointerNum,
9210                                     InputInfo.BasePointersArray.getPointer(),
9211                                     InputInfo.PointersArray.getPointer(),
9212                                     InputInfo.SizesArray.getPointer(),
9213                                     MapTypesArray};
9214
9215    // Select the right runtime function call for each expected standalone
9216    // directive.
9217    const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
9218    OpenMPRTLFunction RTLFn;
9219    switch (D.getDirectiveKind()) {
9220    case OMPD_target_enter_data:
9221      RTLFn = HasNowait ? OMPRTL__tgt_target_data_begin_nowait
9222                        : OMPRTL__tgt_target_data_begin;
9223      break;
9224    case OMPD_target_exit_data:
9225      RTLFn = HasNowait ? OMPRTL__tgt_target_data_end_nowait
9226                        : OMPRTL__tgt_target_data_end;
9227      break;
9228    case OMPD_target_update:
9229      RTLFn = HasNowait ? OMPRTL__tgt_target_data_update_nowait
9230                        : OMPRTL__tgt_target_data_update;
9231      break;
9232    case OMPD_parallel:
9233    case OMPD_for:
9234    case OMPD_parallel_for:
9235    case OMPD_parallel_sections:
9236    case OMPD_for_simd:
9237    case OMPD_parallel_for_simd:
9238    case OMPD_cancel:
9239    case OMPD_cancellation_point:
9240    case OMPD_ordered:
9241    case OMPD_threadprivate:
9242    case OMPD_allocate:
9243    case OMPD_task:
9244    case OMPD_simd:
9245    case OMPD_sections:
9246    case OMPD_section:
9247    case OMPD_single:
9248    case OMPD_master:
9249    case OMPD_critical:
9250    case OMPD_taskyield:
9251    case OMPD_barrier:
9252    case OMPD_taskwait:
9253    case OMPD_taskgroup:
9254    case OMPD_atomic:
9255    case OMPD_flush:
9256    case OMPD_teams:
9257    case OMPD_target_data:
9258    case OMPD_distribute:
9259    case OMPD_distribute_simd:
9260    case OMPD_distribute_parallel_for:
9261    case OMPD_distribute_parallel_for_simd:
9262    case OMPD_teams_distribute:
9263    case OMPD_teams_distribute_simd:
9264    case OMPD_teams_distribute_parallel_for:
9265    case OMPD_teams_distribute_parallel_for_simd:
9266    case OMPD_declare_simd:
9267    case OMPD_declare_target:
9268    case OMPD_end_declare_target:
9269    case OMPD_declare_reduction:
9270    case OMPD_declare_mapper:
9271    case OMPD_taskloop:
9272    case OMPD_taskloop_simd:
9273    case OMPD_target:
9274    case OMPD_target_simd:
9275    case OMPD_target_teams_distribute:
9276    case OMPD_target_teams_distribute_simd:
9277    case OMPD_target_teams_distribute_parallel_for:
9278    case OMPD_target_teams_distribute_parallel_for_simd:
9279    case OMPD_target_teams:
9280    case OMPD_target_parallel:
9281    case OMPD_target_parallel_for:
9282    case OMPD_target_parallel_for_simd:
9283    case OMPD_requires:
9284    case OMPD_unknown:
9285      llvm_unreachable("Unexpected standalone target data directive.");
9286      break;
9287    }
9288    CGF.EmitRuntimeCall(createRuntimeFunction(RTLFn), OffloadingArgs);
9289  };
9290
9291  auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray](
9292                             CodeGenFunction &CGF, PrePostActionTy &) {
9293    // Fill up the arrays with all the mapped variables.
9294    MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
9295    MappableExprsHandler::MapValuesArrayTy Pointers;
9296    MappableExprsHandler::MapValuesArrayTy Sizes;
9297    MappableExprsHandler::MapFlagsArrayTy MapTypes;
9298
9299    // Get map clause information.
9300    MappableExprsHandler MEHandler(D, CGF);
9301    MEHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
9302
9303    TargetDataInfo Info;
9304    // Fill up the arrays and create the arguments.
9305    emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
9306    emitOffloadingArraysArgument(CGF, Info.BasePointersArray,
9307                                 Info.PointersArray, Info.SizesArray,
9308                                 Info.MapTypesArray, Info);
9309    InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
9310    InputInfo.BasePointersArray =
9311        Address(Info.BasePointersArray, CGM.getPointerAlign());
9312    InputInfo.PointersArray =
9313        Address(Info.PointersArray, CGM.getPointerAlign());
9314    InputInfo.SizesArray =
9315        Address(Info.SizesArray, CGM.getPointerAlign());
9316    MapTypesArray = Info.MapTypesArray;
9317    if (D.hasClausesOfKind<OMPDependClause>())
9318      CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
9319    else
9320      emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
9321  };
9322
9323  if (IfCond) {
9324    emitOMPIfClause(CGF, IfCond, TargetThenGen,
9325                    [](CodeGenFunction &CGF, PrePostActionTy &) {});
9326  } else {
9327    RegionCodeGenTy ThenRCG(TargetThenGen);
9328    ThenRCG(CGF);
9329  }
9330}
9331
9332namespace {
9333  /// Kind of parameter in a function with 'declare simd' directive.
9334  enum ParamKindTy { LinearWithVarStride, Linear, Uniform, Vector };
9335  /// Attribute set of the parameter.
9336  struct ParamAttrTy {
9337    ParamKindTy Kind = Vector;
9338    llvm::APSInt StrideOrArg;
9339    llvm::APSInt Alignment;
9340  };
9341} // namespace
9342
9343static unsigned evaluateCDTSize(const FunctionDecl *FD,
9344                                ArrayRef<ParamAttrTy> ParamAttrs) {
9345  // Every vector variant of a SIMD-enabled function has a vector length (VLEN).
9346  // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument
9347  // of that clause. The VLEN value must be power of 2.
9348  // In other case the notion of the function`s "characteristic data type" (CDT)
9349  // is used to compute the vector length.
9350  // CDT is defined in the following order:
9351  //   a) For non-void function, the CDT is the return type.
9352  //   b) If the function has any non-uniform, non-linear parameters, then the
9353  //   CDT is the type of the first such parameter.
9354  //   c) If the CDT determined by a) or b) above is struct, union, or class
9355  //   type which is pass-by-value (except for the type that maps to the
9356  //   built-in complex data type), the characteristic data type is int.
9357  //   d) If none of the above three cases is applicable, the CDT is int.
9358  // The VLEN is then determined based on the CDT and the size of vector
9359  // register of that ISA for which current vector version is generated. The
9360  // VLEN is computed using the formula below:
9361  //   VLEN  = sizeof(vector_register) / sizeof(CDT),
9362  // where vector register size specified in section 3.2.1 Registers and the
9363  // Stack Frame of original AMD64 ABI document.
9364  QualType RetType = FD->getReturnType();
9365  if (RetType.isNull())
9366    return 0;
9367  ASTContext &C = FD->getASTContext();
9368  QualType CDT;
9369  if (!RetType.isNull() && !RetType->isVoidType()) {
9370    CDT = RetType;
9371  } else {
9372    unsigned Offset = 0;
9373    if (const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
9374      if (ParamAttrs[Offset].Kind == Vector)
9375        CDT = C.getPointerType(C.getRecordType(MD->getParent()));
9376      ++Offset;
9377    }
9378    if (CDT.isNull()) {
9379      for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
9380        if (ParamAttrs[I + Offset].Kind == Vector) {
9381          CDT = FD->getParamDecl(I)->getType();
9382          break;
9383        }
9384      }
9385    }
9386  }
9387  if (CDT.isNull())
9388    CDT = C.IntTy;
9389  CDT = CDT->getCanonicalTypeUnqualified();
9390  if (CDT->isRecordType() || CDT->isUnionType())
9391    CDT = C.IntTy;
9392  return C.getTypeSize(CDT);
9393}
9394
9395static void
9396emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn,
9397                           const llvm::APSInt &VLENVal,
9398                           ArrayRef<ParamAttrTy> ParamAttrs,
9399                           OMPDeclareSimdDeclAttr::BranchStateTy State) {
9400  struct ISADataTy {
9401    char ISA;
9402    unsigned VecRegSize;
9403  };
9404  ISADataTy ISAData[] = {
9405      {
9406          'b', 128
9407      }, // SSE
9408      {
9409          'c', 256
9410      }, // AVX
9411      {
9412          'd', 256
9413      }, // AVX2
9414      {
9415          'e', 512
9416      }, // AVX512
9417  };
9418  llvm::SmallVector<char, 2> Masked;
9419  switch (State) {
9420  case OMPDeclareSimdDeclAttr::BS_Undefined:
9421    Masked.push_back('N');
9422    Masked.push_back('M');
9423    break;
9424  case OMPDeclareSimdDeclAttr::BS_Notinbranch:
9425    Masked.push_back('N');
9426    break;
9427  case OMPDeclareSimdDeclAttr::BS_Inbranch:
9428    Masked.push_back('M');
9429    break;
9430  }
9431  for (char Mask : Masked) {
9432    for (const ISADataTy &Data : ISAData) {
9433      SmallString<256> Buffer;
9434      llvm::raw_svector_ostream Out(Buffer);
9435      Out << "_ZGV" << Data.ISA << Mask;
9436      if (!VLENVal) {
9437        Out << llvm::APSInt::getUnsigned(Data.VecRegSize /
9438                                         evaluateCDTSize(FD, ParamAttrs));
9439      } else {
9440        Out << VLENVal;
9441      }
9442      for (const ParamAttrTy &ParamAttr : ParamAttrs) {
9443        switch (ParamAttr.Kind){
9444        case LinearWithVarStride:
9445          Out << 's' << ParamAttr.StrideOrArg;
9446          break;
9447        case Linear:
9448          Out << 'l';
9449          if (!!ParamAttr.StrideOrArg)
9450            Out << ParamAttr.StrideOrArg;
9451          break;
9452        case Uniform:
9453          Out << 'u';
9454          break;
9455        case Vector:
9456          Out << 'v';
9457          break;
9458        }
9459        if (!!ParamAttr.Alignment)
9460          Out << 'a' << ParamAttr.Alignment;
9461      }
9462      Out << '_' << Fn->getName();
9463      Fn->addFnAttr(Out.str());
9464    }
9465  }
9466}
9467
9468void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
9469                                              llvm::Function *Fn) {
9470  ASTContext &C = CGM.getContext();
9471  FD = FD->getMostRecentDecl();
9472  // Map params to their positions in function decl.
9473  llvm::DenseMap<const Decl *, unsigned> ParamPositions;
9474  if (isa<CXXMethodDecl>(FD))
9475    ParamPositions.try_emplace(FD, 0);
9476  unsigned ParamPos = ParamPositions.size();
9477  for (const ParmVarDecl *P : FD->parameters()) {
9478    ParamPositions.try_emplace(P->getCanonicalDecl(), ParamPos);
9479    ++ParamPos;
9480  }
9481  while (FD) {
9482    for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) {
9483      llvm::SmallVector<ParamAttrTy, 8> ParamAttrs(ParamPositions.size());
9484      // Mark uniform parameters.
9485      for (const Expr *E : Attr->uniforms()) {
9486        E = E->IgnoreParenImpCasts();
9487        unsigned Pos;
9488        if (isa<CXXThisExpr>(E)) {
9489          Pos = ParamPositions[FD];
9490        } else {
9491          const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
9492                                ->getCanonicalDecl();
9493          Pos = ParamPositions[PVD];
9494        }
9495        ParamAttrs[Pos].Kind = Uniform;
9496      }
9497      // Get alignment info.
9498      auto NI = Attr->alignments_begin();
9499      for (const Expr *E : Attr->aligneds()) {
9500        E = E->IgnoreParenImpCasts();
9501        unsigned Pos;
9502        QualType ParmTy;
9503        if (isa<CXXThisExpr>(E)) {
9504          Pos = ParamPositions[FD];
9505          ParmTy = E->getType();
9506        } else {
9507          const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
9508                                ->getCanonicalDecl();
9509          Pos = ParamPositions[PVD];
9510          ParmTy = PVD->getType();
9511        }
9512        ParamAttrs[Pos].Alignment =
9513            (*NI)
9514                ? (*NI)->EvaluateKnownConstInt(C)
9515                : llvm::APSInt::getUnsigned(
9516                      C.toCharUnitsFromBits(C.getOpenMPDefaultSimdAlign(ParmTy))
9517                          .getQuantity());
9518        ++NI;
9519      }
9520      // Mark linear parameters.
9521      auto SI = Attr->steps_begin();
9522      auto MI = Attr->modifiers_begin();
9523      for (const Expr *E : Attr->linears()) {
9524        E = E->IgnoreParenImpCasts();
9525        unsigned Pos;
9526        if (isa<CXXThisExpr>(E)) {
9527          Pos = ParamPositions[FD];
9528        } else {
9529          const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
9530                                ->getCanonicalDecl();
9531          Pos = ParamPositions[PVD];
9532        }
9533        ParamAttrTy &ParamAttr = ParamAttrs[Pos];
9534        ParamAttr.Kind = Linear;
9535        if (*SI) {
9536          Expr::EvalResult Result;
9537          if (!(*SI)->EvaluateAsInt(Result, C, Expr::SE_AllowSideEffects)) {
9538            if (const auto *DRE =
9539                    cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
9540              if (const auto *StridePVD = cast<ParmVarDecl>(DRE->getDecl())) {
9541                ParamAttr.Kind = LinearWithVarStride;
9542                ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(
9543                    ParamPositions[StridePVD->getCanonicalDecl()]);
9544              }
9545            }
9546          } else {
9547            ParamAttr.StrideOrArg = Result.Val.getInt();
9548          }
9549        }
9550        ++SI;
9551        ++MI;
9552      }
9553      llvm::APSInt VLENVal;
9554      if (const Expr *VLEN = Attr->getSimdlen())
9555        VLENVal = VLEN->EvaluateKnownConstInt(C);
9556      OMPDeclareSimdDeclAttr::BranchStateTy State = Attr->getBranchState();
9557      if (CGM.getTriple().getArch() == llvm::Triple::x86 ||
9558          CGM.getTriple().getArch() == llvm::Triple::x86_64)
9559        emitX86DeclareSimdFunction(FD, Fn, VLENVal, ParamAttrs, State);
9560    }
9561    FD = FD->getPreviousDecl();
9562  }
9563}
9564
9565namespace {
9566/// Cleanup action for doacross support.
9567class DoacrossCleanupTy final : public EHScopeStack::Cleanup {
9568public:
9569  static const int DoacrossFinArgs = 2;
9570
9571private:
9572  llvm::FunctionCallee RTLFn;
9573  llvm::Value *Args[DoacrossFinArgs];
9574
9575public:
9576  DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
9577                    ArrayRef<llvm::Value *> CallArgs)
9578      : RTLFn(RTLFn) {
9579    assert(CallArgs.size() == DoacrossFinArgs);
9580    std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
9581  }
9582  void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
9583    if (!CGF.HaveInsertPoint())
9584      return;
9585    CGF.EmitRuntimeCall(RTLFn, Args);
9586  }
9587};
9588} // namespace
9589
9590void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
9591                                       const OMPLoopDirective &D,
9592                                       ArrayRef<Expr *> NumIterations) {
9593  if (!CGF.HaveInsertPoint())
9594    return;
9595
9596  ASTContext &C = CGM.getContext();
9597  QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true);
9598  RecordDecl *RD;
9599  if (KmpDimTy.isNull()) {
9600    // Build struct kmp_dim {  // loop bounds info casted to kmp_int64
9601    //  kmp_int64 lo; // lower
9602    //  kmp_int64 up; // upper
9603    //  kmp_int64 st; // stride
9604    // };
9605    RD = C.buildImplicitRecord("kmp_dim");
9606    RD->startDefinition();
9607    addFieldToRecordDecl(C, RD, Int64Ty);
9608    addFieldToRecordDecl(C, RD, Int64Ty);
9609    addFieldToRecordDecl(C, RD, Int64Ty);
9610    RD->completeDefinition();
9611    KmpDimTy = C.getRecordType(RD);
9612  } else {
9613    RD = cast<RecordDecl>(KmpDimTy->getAsTagDecl());
9614  }
9615  llvm::APInt Size(/*numBits=*/32, NumIterations.size());
9616  QualType ArrayTy =
9617      C.getConstantArrayType(KmpDimTy, Size, ArrayType::Normal, 0);
9618
9619  Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims");
9620  CGF.EmitNullInitialization(DimsAddr, ArrayTy);
9621  enum { LowerFD = 0, UpperFD, StrideFD };
9622  // Fill dims with data.
9623  for (unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
9624    LValue DimsLVal = CGF.MakeAddrLValue(
9625        CGF.Builder.CreateConstArrayGEP(DimsAddr, I), KmpDimTy);
9626    // dims.upper = num_iterations;
9627    LValue UpperLVal = CGF.EmitLValueForField(
9628        DimsLVal, *std::next(RD->field_begin(), UpperFD));
9629    llvm::Value *NumIterVal =
9630        CGF.EmitScalarConversion(CGF.EmitScalarExpr(NumIterations[I]),
9631                                 D.getNumIterations()->getType(), Int64Ty,
9632                                 D.getNumIterations()->getExprLoc());
9633    CGF.EmitStoreOfScalar(NumIterVal, UpperLVal);
9634    // dims.stride = 1;
9635    LValue StrideLVal = CGF.EmitLValueForField(
9636        DimsLVal, *std::next(RD->field_begin(), StrideFD));
9637    CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1),
9638                          StrideLVal);
9639  }
9640
9641  // Build call void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid,
9642  // kmp_int32 num_dims, struct kmp_dim * dims);
9643  llvm::Value *Args[] = {
9644      emitUpdateLocation(CGF, D.getBeginLoc()),
9645      getThreadID(CGF, D.getBeginLoc()),
9646      llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()),
9647      CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
9648          CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(),
9649          CGM.VoidPtrTy)};
9650
9651  llvm::FunctionCallee RTLFn =
9652      createRuntimeFunction(OMPRTL__kmpc_doacross_init);
9653  CGF.EmitRuntimeCall(RTLFn, Args);
9654  llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
9655      emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())};
9656  llvm::FunctionCallee FiniRTLFn =
9657      createRuntimeFunction(OMPRTL__kmpc_doacross_fini);
9658  CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn,
9659                                             llvm::makeArrayRef(FiniArgs));
9660}
9661
9662void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
9663                                          const OMPDependClause *C) {
9664  QualType Int64Ty =
9665      CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
9666  llvm::APInt Size(/*numBits=*/32, C->getNumLoops());
9667  QualType ArrayTy = CGM.getContext().getConstantArrayType(
9668      Int64Ty, Size, ArrayType::Normal, 0);
9669  Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr");
9670  for (unsigned I = 0, E = C->getNumLoops(); I < E; ++I) {
9671    const Expr *CounterVal = C->getLoopData(I);
9672    assert(CounterVal);
9673    llvm::Value *CntVal = CGF.EmitScalarConversion(
9674        CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,
9675        CounterVal->getExprLoc());
9676    CGF.EmitStoreOfScalar(CntVal, CGF.Builder.CreateConstArrayGEP(CntAddr, I),
9677                          /*Volatile=*/false, Int64Ty);
9678  }
9679  llvm::Value *Args[] = {
9680      emitUpdateLocation(CGF, C->getBeginLoc()),
9681      getThreadID(CGF, C->getBeginLoc()),
9682      CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()};
9683  llvm::FunctionCallee RTLFn;
9684  if (C->getDependencyKind() == OMPC_DEPEND_source) {
9685    RTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_post);
9686  } else {
9687    getDependencyKind() == OMPC_DEPEND_sink", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 9687, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(C->getDependencyKind() == OMPC_DEPEND_sink);
9688    RTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_wait);
9689  }
9690  CGF.EmitRuntimeCall(RTLFn, Args);
9691}
9692
9693void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc,
9694                               llvm::FunctionCallee Callee,
9695                               ArrayRef<llvm::Value *> Args) const {
9696   (0) . __assert_fail ("Loc.isValid() && \"Outlined function call location must be valid.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 9696, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Loc.isValid() && "Outlined function call location must be valid.");
9697  auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
9698
9699  if (auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {
9700    if (Fn->doesNotThrow()) {
9701      CGF.EmitNounwindRuntimeCall(Fn, Args);
9702      return;
9703    }
9704  }
9705  CGF.EmitRuntimeCall(Callee, Args);
9706}
9707
9708void CGOpenMPRuntime::emitOutlinedFunctionCall(
9709    CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
9710    ArrayRef<llvm::Value *> Args) const {
9711  emitCall(CGF, Loc, OutlinedFn, Args);
9712}
9713
9714Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF,
9715                                             const VarDecl *NativeParam,
9716                                             const VarDecl *TargetParam) const {
9717  return CGF.GetAddrOfLocalVar(NativeParam);
9718}
9719
9720namespace {
9721/// Cleanup action for allocate support.
9722class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
9723public:
9724  static const int CleanupArgs = 3;
9725
9726private:
9727  llvm::FunctionCallee RTLFn;
9728  llvm::Value *Args[CleanupArgs];
9729
9730public:
9731  OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
9732                       ArrayRef<llvm::Value *> CallArgs)
9733      : RTLFn(RTLFn) {
9734     (0) . __assert_fail ("CallArgs.size() == CleanupArgs && \"Size of arguments does not match.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 9735, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(CallArgs.size() == CleanupArgs &&
9735 (0) . __assert_fail ("CallArgs.size() == CleanupArgs && \"Size of arguments does not match.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 9735, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">           "Size of arguments does not match.");
9736    std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
9737  }
9738  void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
9739    if (!CGF.HaveInsertPoint())
9740      return;
9741    CGF.EmitRuntimeCall(RTLFn, Args);
9742  }
9743};
9744} // namespace
9745
9746Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
9747                                                   const VarDecl *VD) {
9748  if (!VD)
9749    return Address::invalid();
9750  const VarDecl *CVD = VD->getCanonicalDecl();
9751  if (!CVD->hasAttr<OMPAllocateDeclAttr>())
9752    return Address::invalid();
9753  const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
9754  // Use the default allocation.
9755  if (AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc)
9756    return Address::invalid();
9757  auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
9758  if (!Elem.second.ServiceInsertPt)
9759    setLocThreadIdInsertPt(CGF);
9760  CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
9761  CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
9762  llvm::Value *Size;
9763  CharUnits Align = CGM.getContext().getDeclAlign(CVD);
9764  if (CVD->getType()->isVariablyModifiedType()) {
9765    Size = CGF.getTypeSize(CVD->getType());
9766    Align = CGM.getContext().getTypeAlignInChars(CVD->getType());
9767  } else {
9768    CharUnits Sz = CGM.getContext().getTypeSizeInChars(CVD->getType());
9769    Align = CGM.getContext().getDeclAlign(CVD);
9770    Size = CGM.getSize(Sz.alignTo(Align));
9771  }
9772  llvm::Value *ThreadID = getThreadID(CGF, CVD->getBeginLoc());
9773   (0) . __assert_fail ("AA->getAllocator() && \"Expected allocator expression for non-default allocator.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 9774, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(AA->getAllocator() &&
9774 (0) . __assert_fail ("AA->getAllocator() && \"Expected allocator expression for non-default allocator.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 9774, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">         "Expected allocator expression for non-default allocator.");
9775  llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
9776  llvm::Value *Args[] = {ThreadID, Size, Allocator};
9777
9778  llvm::Value *Addr =
9779      CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_alloc), Args,
9780                          CVD->getName() + ".void.addr");
9781  llvm::Value *FiniArgs[OMPAllocateCleanupTy::CleanupArgs] = {ThreadID, Addr,
9782                                                              Allocator};
9783  llvm::FunctionCallee FiniRTLFn = createRuntimeFunction(OMPRTL__kmpc_free);
9784
9785  CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FiniRTLFn,
9786                                                llvm::makeArrayRef(FiniArgs));
9787  Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
9788      Addr,
9789      CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())),
9790      CVD->getName() + ".addr");
9791  return Address(Addr, Align);
9792}
9793
9794llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction(
9795    const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
9796    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
9797  llvm_unreachable("Not supported in SIMD-only mode");
9798}
9799
9800llvm::Function *CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction(
9801    const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
9802    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
9803  llvm_unreachable("Not supported in SIMD-only mode");
9804}
9805
9806llvm::Function *CGOpenMPSIMDRuntime::emitTaskOutlinedFunction(
9807    const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
9808    const VarDecl *PartIDVar, const VarDecl *TaskTVar,
9809    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
9810    bool Tied, unsigned &NumberOfParts) {
9811  llvm_unreachable("Not supported in SIMD-only mode");
9812}
9813
9814void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF,
9815                                           SourceLocation Loc,
9816                                           llvm::Function *OutlinedFn,
9817                                           ArrayRef<llvm::Value *> CapturedVars,
9818                                           const Expr *IfCond) {
9819  llvm_unreachable("Not supported in SIMD-only mode");
9820}
9821
9822void CGOpenMPSIMDRuntime::emitCriticalRegion(
9823    CodeGenFunction &CGF, StringRef CriticalName,
9824    const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc,
9825    const Expr *Hint) {
9826  llvm_unreachable("Not supported in SIMD-only mode");
9827}
9828
9829void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF,
9830                                           const RegionCodeGenTy &MasterOpGen,
9831                                           SourceLocation Loc) {
9832  llvm_unreachable("Not supported in SIMD-only mode");
9833}
9834
9835void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
9836                                            SourceLocation Loc) {
9837  llvm_unreachable("Not supported in SIMD-only mode");
9838}
9839
9840void CGOpenMPSIMDRuntime::emitTaskgroupRegion(
9841    CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,
9842    SourceLocation Loc) {
9843  llvm_unreachable("Not supported in SIMD-only mode");
9844}
9845
9846void CGOpenMPSIMDRuntime::emitSingleRegion(
9847    CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen,
9848    SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars,
9849    ArrayRef<const Expr *> DestExprs, ArrayRef<const Expr *> SrcExprs,
9850    ArrayRef<const Expr *> AssignmentOps) {
9851  llvm_unreachable("Not supported in SIMD-only mode");
9852}
9853
9854void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF,
9855                                            const RegionCodeGenTy &OrderedOpGen,
9856                                            SourceLocation Loc,
9857                                            bool IsThreads) {
9858  llvm_unreachable("Not supported in SIMD-only mode");
9859}
9860
9861void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF,
9862                                          SourceLocation Loc,
9863                                          OpenMPDirectiveKind Kind,
9864                                          bool EmitChecks,
9865                                          bool ForceSimpleCall) {
9866  llvm_unreachable("Not supported in SIMD-only mode");
9867}
9868
9869void CGOpenMPSIMDRuntime::emitForDispatchInit(
9870    CodeGenFunction &CGF, SourceLocation Loc,
9871    const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
9872    bool Ordered, const DispatchRTInput &DispatchValues) {
9873  llvm_unreachable("Not supported in SIMD-only mode");
9874}
9875
9876void CGOpenMPSIMDRuntime::emitForStaticInit(
9877    CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind,
9878    const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) {
9879  llvm_unreachable("Not supported in SIMD-only mode");
9880}
9881
9882void CGOpenMPSIMDRuntime::emitDistributeStaticInit(
9883    CodeGenFunction &CGF, SourceLocation Loc,
9884    OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) {
9885  llvm_unreachable("Not supported in SIMD-only mode");
9886}
9887
9888void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
9889                                                     SourceLocation Loc,
9890                                                     unsigned IVSize,
9891                                                     bool IVSigned) {
9892  llvm_unreachable("Not supported in SIMD-only mode");
9893}
9894
9895void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF,
9896                                              SourceLocation Loc,
9897                                              OpenMPDirectiveKind DKind) {
9898  llvm_unreachable("Not supported in SIMD-only mode");
9899}
9900
9901llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF,
9902                                              SourceLocation Loc,
9903                                              unsigned IVSize, bool IVSigned,
9904                                              Address IL, Address LB,
9905                                              Address UB, Address ST) {
9906  llvm_unreachable("Not supported in SIMD-only mode");
9907}
9908
9909void CGOpenMPSIMDRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
9910                                               llvm::Value *NumThreads,
9911                                               SourceLocation Loc) {
9912  llvm_unreachable("Not supported in SIMD-only mode");
9913}
9914
9915void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF,
9916                                             OpenMPProcBindClauseKind ProcBind,
9917                                             SourceLocation Loc) {
9918  llvm_unreachable("Not supported in SIMD-only mode");
9919}
9920
9921Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
9922                                                    const VarDecl *VD,
9923                                                    Address VDAddr,
9924                                                    SourceLocation Loc) {
9925  llvm_unreachable("Not supported in SIMD-only mode");
9926}
9927
9928llvm::Function *CGOpenMPSIMDRuntime::emitThreadPrivateVarDefinition(
9929    const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit,
9930    CodeGenFunction *CGF) {
9931  llvm_unreachable("Not supported in SIMD-only mode");
9932}
9933
9934Address CGOpenMPSIMDRuntime::getAddrOfArtificialThreadPrivate(
9935    CodeGenFunction &CGF, QualType VarType, StringRef Name) {
9936  llvm_unreachable("Not supported in SIMD-only mode");
9937}
9938
9939void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF,
9940                                    ArrayRef<const Expr *> Vars,
9941                                    SourceLocation Loc) {
9942  llvm_unreachable("Not supported in SIMD-only mode");
9943}
9944
9945void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
9946                                       const OMPExecutableDirective &D,
9947                                       llvm::Function *TaskFunction,
9948                                       QualType SharedsTy, Address Shareds,
9949                                       const Expr *IfCond,
9950                                       const OMPTaskDataTy &Data) {
9951  llvm_unreachable("Not supported in SIMD-only mode");
9952}
9953
9954void CGOpenMPSIMDRuntime::emitTaskLoopCall(
9955    CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
9956    llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
9957    const Expr *IfCond, const OMPTaskDataTy &Data) {
9958  llvm_unreachable("Not supported in SIMD-only mode");
9959}
9960
9961void CGOpenMPSIMDRuntime::emitReduction(
9962    CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates,
9963    ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
9964    ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) {
9965   (0) . __assert_fail ("Options.SimpleReduction && \"Only simple reduction is expected.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGOpenMPRuntime.cpp", 9965, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Options.SimpleReduction && "Only simple reduction is expected.");
9966  CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs,
9967                                 ReductionOps, Options);
9968}
9969
9970llvm::Value *CGOpenMPSIMDRuntime::emitTaskReductionInit(
9971    CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
9972    ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
9973  llvm_unreachable("Not supported in SIMD-only mode");
9974}
9975
9976void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
9977                                                  SourceLocation Loc,
9978                                                  ReductionCodeGen &RCG,
9979                                                  unsigned N) {
9980  llvm_unreachable("Not supported in SIMD-only mode");
9981}
9982
9983Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF,
9984                                                  SourceLocation Loc,
9985                                                  llvm::Value *ReductionsPtr,
9986                                                  LValue SharedLVal) {
9987  llvm_unreachable("Not supported in SIMD-only mode");
9988}
9989
9990void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
9991                                           SourceLocation Loc) {
9992  llvm_unreachable("Not supported in SIMD-only mode");
9993}
9994
9995void CGOpenMPSIMDRuntime::emitCancellationPointCall(
9996    CodeGenFunction &CGF, SourceLocation Loc,
9997    OpenMPDirectiveKind CancelRegion) {
9998  llvm_unreachable("Not supported in SIMD-only mode");
9999}
10000
10001void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF,
10002                                         SourceLocation Loc, const Expr *IfCond,
10003                                         OpenMPDirectiveKind CancelRegion) {
10004  llvm_unreachable("Not supported in SIMD-only mode");
10005}
10006
10007void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction(
10008    const OMPExecutableDirective &D, StringRef ParentName,
10009    llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
10010    bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
10011  llvm_unreachable("Not supported in SIMD-only mode");
10012}
10013
10014void CGOpenMPSIMDRuntime::emitTargetCall(CodeGenFunction &CGF,
10015                                         const OMPExecutableDirective &D,
10016                                         llvm::Function *OutlinedFn,
10017                                         llvm::Value *OutlinedFnID,
10018                                         const Expr *IfCond,
10019                                         const Expr *Device) {
10020  llvm_unreachable("Not supported in SIMD-only mode");
10021}
10022
10023bool CGOpenMPSIMDRuntime::emitTargetFunctions(GlobalDecl GD) {
10024  llvm_unreachable("Not supported in SIMD-only mode");
10025}
10026
10027bool CGOpenMPSIMDRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
10028  llvm_unreachable("Not supported in SIMD-only mode");
10029}
10030
10031bool CGOpenMPSIMDRuntime::emitTargetGlobal(GlobalDecl GD) {
10032  return false;
10033}
10034
10035llvm::Function *CGOpenMPSIMDRuntime::emitRegistrationFunction() {
10036  return nullptr;
10037}
10038
10039void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF,
10040                                        const OMPExecutableDirective &D,
10041                                        SourceLocation Loc,
10042                                        llvm::Function *OutlinedFn,
10043                                        ArrayRef<llvm::Value *> CapturedVars) {
10044  llvm_unreachable("Not supported in SIMD-only mode");
10045}
10046
10047void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
10048                                             const Expr *NumTeams,
10049                                             const Expr *ThreadLimit,
10050                                             SourceLocation Loc) {
10051  llvm_unreachable("Not supported in SIMD-only mode");
10052}
10053
10054void CGOpenMPSIMDRuntime::emitTargetDataCalls(
10055    CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
10056    const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info) {
10057  llvm_unreachable("Not supported in SIMD-only mode");
10058}
10059
10060void CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall(
10061    CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
10062    const Expr *Device) {
10063  llvm_unreachable("Not supported in SIMD-only mode");
10064}
10065
10066void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF,
10067                                           const OMPLoopDirective &D,
10068                                           ArrayRef<Expr *> NumIterations) {
10069  llvm_unreachable("Not supported in SIMD-only mode");
10070}
10071
10072void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
10073                                              const OMPDependClause *C) {
10074  llvm_unreachable("Not supported in SIMD-only mode");
10075}
10076
10077const VarDecl *
10078CGOpenMPSIMDRuntime::translateParameter(const FieldDecl *FD,
10079                                        const VarDecl *NativeParam) const {
10080  llvm_unreachable("Not supported in SIMD-only mode");
10081}
10082
10083Address
10084CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF,
10085                                         const VarDecl *NativeParam,
10086                                         const VarDecl *TargetParam) const {
10087  llvm_unreachable("Not supported in SIMD-only mode");
10088}
10089
clang::CodeGen::ReductionCodeGen::emitSharedLValue
clang::CodeGen::ReductionCodeGen::emitSharedLValueUB
clang::CodeGen::ReductionCodeGen::emitAggregateInitialization
clang::CodeGen::ReductionCodeGen::emitSharedLValue
clang::CodeGen::ReductionCodeGen::emitAggregateType
clang::CodeGen::ReductionCodeGen::emitAggregateType
clang::CodeGen::ReductionCodeGen::emitInitialization
clang::CodeGen::ReductionCodeGen::needCleanups
clang::CodeGen::ReductionCodeGen::emitCleanups
clang::CodeGen::ReductionCodeGen::adjustPrivateAddress
clang::CodeGen::ReductionCodeGen::usesReductionInitializer
clang::CodeGen::CGOpenMPRuntime::clear
clang::CodeGen::CGOpenMPRuntime::getName
clang::CodeGen::CGOpenMPRuntime::emitUserDefinedReduction
clang::CodeGen::CGOpenMPRuntime::getUserDefinedReduction
clang::CodeGen::CGOpenMPRuntime::emitParallelOutlinedFunction
clang::CodeGen::CGOpenMPRuntime::emitTeamsOutlinedFunction
clang::CodeGen::CGOpenMPRuntime::emitTaskOutlinedFunction
clang::CodeGen::CGOpenMPRuntime::getOrCreateDefaultLocation
clang::CodeGen::CGOpenMPRuntime::setLocThreadIdInsertPt
clang::CodeGen::CGOpenMPRuntime::clearLocThreadIdInsertPt
clang::CodeGen::CGOpenMPRuntime::emitUpdateLocation
clang::CodeGen::CGOpenMPRuntime::getThreadID
clang::CodeGen::CGOpenMPRuntime::functionFinished
clang::CodeGen::CGOpenMPRuntime::getIdentTyPointerTy
clang::CodeGen::CGOpenMPRuntime::getKmpc_MicroPointerTy
clang::CodeGen::CGOpenMPRuntime::createRuntimeFunction
clang::CodeGen::CGOpenMPRuntime::createForStaticInitFunction
clang::CodeGen::CGOpenMPRuntime::createDispatchInitFunction
clang::CodeGen::CGOpenMPRuntime::createDispatchFiniFunction
clang::CodeGen::CGOpenMPRuntime::createDispatchNextFunction
clang::CodeGen::CGOpenMPRuntime::getAddrOfDeclareTargetLink
clang::CodeGen::CGOpenMPRuntime::getOrCreateThreadPrivateCache
clang::CodeGen::CGOpenMPRuntime::getAddrOfThreadPrivate
clang::CodeGen::CGOpenMPRuntime::emitThreadPrivateVarInit
clang::CodeGen::CGOpenMPRuntime::emitThreadPrivateVarDefinition
clang::CodeGen::CGOpenMPRuntime::emitDeclareTargetVarDefinition
clang::CodeGen::CGOpenMPRuntime::getAddrOfArtificialThreadPrivate
clang::CodeGen::CGOpenMPRuntime::emitOMPIfClause
clang::CodeGen::CGOpenMPRuntime::emitParallelCall
clang::CodeGen::CGOpenMPRuntime::emitThreadIDAddress
clang::CodeGen::CGOpenMPRuntime::getOrCreateInternalVariable
clang::CodeGen::CGOpenMPRuntime::getCriticalRegionLock
clang::CodeGen::CGOpenMPRuntime::emitCriticalRegion
clang::CodeGen::CGOpenMPRuntime::emitMasterRegion
clang::CodeGen::CGOpenMPRuntime::emitTaskyieldCall
clang::CodeGen::CGOpenMPRuntime::emitTaskgroupRegion
clang::CodeGen::CGOpenMPRuntime::emitSingleRegion
clang::CodeGen::CGOpenMPRuntime::emitOrderedRegion
clang::CodeGen::CGOpenMPRuntime::getDefaultFlagsForBarriers
clang::CodeGen::CGOpenMPRuntime::getDefaultScheduleAndChunk
clang::CodeGen::CGOpenMPRuntime::emitBarrierCall
clang::CodeGen::CGOpenMPRuntime::isStaticNonchunked
clang::CodeGen::CGOpenMPRuntime::isStaticNonchunked
clang::CodeGen::CGOpenMPRuntime::isStaticChunked
clang::CodeGen::CGOpenMPRuntime::isStaticChunked
clang::CodeGen::CGOpenMPRuntime::isDynamic
clang::CodeGen::CGOpenMPRuntime::emitForDispatchInit
clang::CodeGen::CGOpenMPRuntime::emitForStaticInit
clang::CodeGen::CGOpenMPRuntime::emitDistributeStaticInit
clang::CodeGen::CGOpenMPRuntime::emitForStaticFinish
clang::CodeGen::CGOpenMPRuntime::emitForOrderedIterationEnd
clang::CodeGen::CGOpenMPRuntime::emitForNext
clang::CodeGen::CGOpenMPRuntime::emitNumThreadsClause
clang::CodeGen::CGOpenMPRuntime::emitProcBindClause
clang::CodeGen::CGOpenMPRuntime::emitFlush
clang::CodeGen::CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty
clang::CodeGen::CGOpenMPRuntime::OffloadEntriesInfoManagerTy::initializeTargetRegionEntryInfo
clang::CodeGen::CGOpenMPRuntime::OffloadEntriesInfoManagerTy::registerTargetRegionEntryInfo
clang::CodeGen::CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo
clang::CodeGen::CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo
clang::CodeGen::CGOpenMPRuntime::OffloadEntriesInfoManagerTy::initializeDeviceGlobalVarEntryInfo
clang::CodeGen::CGOpenMPRuntime::OffloadEntriesInfoManagerTy::registerDeviceGlobalVarEntryInfo
clang::CodeGen::CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnDeviceGlobalVarEntriesInfo
clang::CodeGen::CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration
clang::CodeGen::CGOpenMPRuntime::createOffloadEntry
clang::CodeGen::CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata
clang::CodeGen::CGOpenMPRuntime::loadOffloadInfoMetadata
clang::CodeGen::CGOpenMPRuntime::emitKmpRoutineEntryT
clang::CodeGen::CGOpenMPRuntime::getTgtOffloadEntryQTy
clang::CodeGen::CGOpenMPRuntime::getTgtDeviceImageQTy
clang::CodeGen::CGOpenMPRuntime::getTgtBinaryDescriptorQTy
clang::CodeGen::CGOpenMPRuntime::emitTaskInit
clang::CodeGen::CGOpenMPRuntime::emitTaskCall
clang::CodeGen::CGOpenMPRuntime::emitTaskLoopCall
clang::CodeGen::CGOpenMPRuntime::emitReductionFunction
clang::CodeGen::CGOpenMPRuntime::emitSingleReductionCombiner
clang::CodeGen::CGOpenMPRuntime::emitReduction
clang::CodeGen::CGOpenMPRuntime::emitTaskReductionInit
clang::CodeGen::CGOpenMPRuntime::emitTaskReductionFixups
clang::CodeGen::CGOpenMPRuntime::getTaskReductionItem
clang::CodeGen::CGOpenMPRuntime::emitTaskwaitCall
clang::CodeGen::CGOpenMPRuntime::emitInlinedDirective
clang::CodeGen::CGOpenMPRuntime::emitCancellationPointCall
clang::CodeGen::CGOpenMPRuntime::emitCancelCall
clang::CodeGen::CGOpenMPRuntime::emitTargetOutlinedFunction
clang::CodeGen::CGOpenMPRuntime::emitTargetOutlinedFunctionHelper
clang::CodeGen::CGOpenMPRuntime::emitTargetNumIterationsCall
clang::CodeGen::CGOpenMPRuntime::emitTargetCall
clang::CodeGen::CGOpenMPRuntime::scanForTargetRegionsFunctions
clang::CodeGen::CGOpenMPRuntime::emitTargetFunctions
clang::CodeGen::CGOpenMPRuntime::emitTargetGlobalVariable
clang::CodeGen::CGOpenMPRuntime::registerTargetFirstprivateCopy
clang::CodeGen::CGOpenMPRuntime::registerTargetGlobalVariable
clang::CodeGen::CGOpenMPRuntime::emitTargetGlobal
clang::CodeGen::CGOpenMPRuntime::emitDeferredTargetDecls
clang::CodeGen::CGOpenMPRuntime::adjustTargetSpecificDataForLambdas
clang::CodeGen::CGOpenMPRuntime::hasAllocateAttributeForGlobalVar
clang::CodeGen::CGOpenMPRuntime::markAsGlobalTarget
clang::CodeGen::CGOpenMPRuntime::emitRegistrationFunction
clang::CodeGen::CGOpenMPRuntime::emitTeamsCall
clang::CodeGen::CGOpenMPRuntime::emitNumTeamsClause
clang::CodeGen::CGOpenMPRuntime::emitTargetDataCalls
clang::CodeGen::CGOpenMPRuntime::emitTargetDataStandAloneCall
clang::CodeGen::CGOpenMPRuntime::emitDeclareSimdFunction
clang::CodeGen::CGOpenMPRuntime::emitDoacrossInit
clang::CodeGen::CGOpenMPRuntime::emitDoacrossOrdered
clang::CodeGen::CGOpenMPRuntime::emitCall
clang::CodeGen::CGOpenMPRuntime::emitOutlinedFunctionCall
clang::CodeGen::CGOpenMPRuntime::getParameterAddress
clang::CodeGen::CGOpenMPRuntime::getAddressOfLocalVariable
clang::CodeGen::CGOpenMPSIMDRuntime::emitParallelOutlinedFunction
clang::CodeGen::CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction
clang::CodeGen::CGOpenMPSIMDRuntime::emitTaskOutlinedFunction
clang::CodeGen::CGOpenMPSIMDRuntime::emitParallelCall
clang::CodeGen::CGOpenMPSIMDRuntime::emitCriticalRegion
clang::CodeGen::CGOpenMPSIMDRuntime::emitMasterRegion
clang::CodeGen::CGOpenMPSIMDRuntime::emitTaskyieldCall
clang::CodeGen::CGOpenMPSIMDRuntime::emitTaskgroupRegion
clang::CodeGen::CGOpenMPSIMDRuntime::emitSingleRegion
clang::CodeGen::CGOpenMPSIMDRuntime::emitOrderedRegion
clang::CodeGen::CGOpenMPSIMDRuntime::emitBarrierCall
clang::CodeGen::CGOpenMPSIMDRuntime::emitForDispatchInit
clang::CodeGen::CGOpenMPSIMDRuntime::emitForStaticInit
clang::CodeGen::CGOpenMPSIMDRuntime::emitDistributeStaticInit
clang::CodeGen::CGOpenMPSIMDRuntime::emitForOrderedIterationEnd
clang::CodeGen::CGOpenMPSIMDRuntime::emitForStaticFinish
clang::CodeGen::CGOpenMPSIMDRuntime::emitForNext
clang::CodeGen::CGOpenMPSIMDRuntime::emitNumThreadsClause
clang::CodeGen::CGOpenMPSIMDRuntime::emitProcBindClause
clang::CodeGen::CGOpenMPSIMDRuntime::getAddrOfThreadPrivate
clang::CodeGen::CGOpenMPSIMDRuntime::emitThreadPrivateVarDefinition
clang::CodeGen::CGOpenMPSIMDRuntime::getAddrOfArtificialThreadPrivate
clang::CodeGen::CGOpenMPSIMDRuntime::emitFlush
clang::CodeGen::CGOpenMPSIMDRuntime::emitTaskCall
clang::CodeGen::CGOpenMPSIMDRuntime::emitTaskLoopCall
clang::CodeGen::CGOpenMPSIMDRuntime::emitReduction
clang::CodeGen::CGOpenMPSIMDRuntime::emitTaskReductionInit
clang::CodeGen::CGOpenMPSIMDRuntime::emitTaskReductionFixups
clang::CodeGen::CGOpenMPSIMDRuntime::getTaskReductionItem
clang::CodeGen::CGOpenMPSIMDRuntime::emitTaskwaitCall
clang::CodeGen::CGOpenMPSIMDRuntime::emitCancellationPointCall
clang::CodeGen::CGOpenMPSIMDRuntime::emitCancelCall
clang::CodeGen::CGOpenMPSIMDRuntime::emitTargetOutlinedFunction
clang::CodeGen::CGOpenMPSIMDRuntime::emitTargetCall
clang::CodeGen::CGOpenMPSIMDRuntime::emitTargetFunctions
clang::CodeGen::CGOpenMPSIMDRuntime::emitTargetGlobalVariable
clang::CodeGen::CGOpenMPSIMDRuntime::emitTargetGlobal
clang::CodeGen::CGOpenMPSIMDRuntime::emitRegistrationFunction
clang::CodeGen::CGOpenMPSIMDRuntime::emitTeamsCall
clang::CodeGen::CGOpenMPSIMDRuntime::emitNumTeamsClause
clang::CodeGen::CGOpenMPSIMDRuntime::emitTargetDataCalls
clang::CodeGen::CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall
clang::CodeGen::CGOpenMPSIMDRuntime::emitDoacrossInit
clang::CodeGen::CGOpenMPSIMDRuntime::emitDoacrossOrdered
clang::CodeGen::CGOpenMPSIMDRuntime::translateParameter
clang::CodeGen::CGOpenMPSIMDRuntime::getParameterAddress
CodeThinker, Enjoy your reading code