Clang Project

clang_source_code/lib/CodeGen/CGLoopInfo.cpp
1//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "CGLoopInfo.h"
10#include "clang/AST/ASTContext.h"
11#include "clang/AST/Attr.h"
12#include "llvm/IR/BasicBlock.h"
13#include "llvm/IR/CFG.h"
14#include "llvm/IR/Constants.h"
15#include "llvm/IR/InstrTypes.h"
16#include "llvm/IR/Instructions.h"
17#include "llvm/IR/Metadata.h"
18using namespace clang::CodeGen;
19using namespace llvm;
20
21static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
22                              const llvm::DebugLoc &StartLoc,
23                              const llvm::DebugLoc &EndLoc, MDNode *&AccGroup) {
24
25  if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
26      Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
27      Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
28      Attrs.PipelineInitiationInterval == 0 &&
29      Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
30      Attrs.UnrollEnable == LoopAttributes::Unspecified &&
31      Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
32      Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
33      !EndLoc)
34    return nullptr;
35
36  SmallVector<Metadata *, 4> Args;
37  // Reserve operand 0 for loop id self reference.
38  auto TempNode = MDNode::getTemporary(Ctx, None);
39  Args.push_back(TempNode.get());
40
41  // If we have a valid start debug location for the loop, add it.
42  if (StartLoc) {
43    Args.push_back(StartLoc.getAsMDNode());
44
45    // If we also have a valid end debug location for the loop, add it.
46    if (EndLoc)
47      Args.push_back(EndLoc.getAsMDNode());
48  }
49
50  // Setting vectorize.width
51  if (Attrs.VectorizeWidth > 0) {
52    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"),
53                        ConstantAsMetadata::get(ConstantInt::get(
54                            Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))};
55    Args.push_back(MDNode::get(Ctx, Vals));
56  }
57
58  // Setting interleave.count
59  if (Attrs.InterleaveCount > 0) {
60    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"),
61                        ConstantAsMetadata::get(ConstantInt::get(
62                            Type::getInt32Ty(Ctx), Attrs.InterleaveCount))};
63    Args.push_back(MDNode::get(Ctx, Vals));
64  }
65
66  // Setting unroll.count
67  if (Attrs.UnrollCount > 0) {
68    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
69                        ConstantAsMetadata::get(ConstantInt::get(
70                            Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
71    Args.push_back(MDNode::get(Ctx, Vals));
72  }
73
74  // Setting unroll_and_jam.count
75  if (Attrs.UnrollAndJamCount > 0) {
76    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
77                        ConstantAsMetadata::get(ConstantInt::get(
78                            Type::getInt32Ty(Ctx), Attrs.UnrollAndJamCount))};
79    Args.push_back(MDNode::get(Ctx, Vals));
80  }
81
82  // Setting vectorize.enable
83  if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
84    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
85                        ConstantAsMetadata::get(ConstantInt::get(
86                            Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable ==
87                                                   LoopAttributes::Enable)))};
88    Args.push_back(MDNode::get(Ctx, Vals));
89  }
90
91  // Setting unroll.full or unroll.disable
92  if (Attrs.UnrollEnable != LoopAttributes::Unspecified) {
93    std::string Name;
94    if (Attrs.UnrollEnable == LoopAttributes::Enable)
95      Name = "llvm.loop.unroll.enable";
96    else if (Attrs.UnrollEnable == LoopAttributes::Full)
97      Name = "llvm.loop.unroll.full";
98    else
99      Name = "llvm.loop.unroll.disable";
100    Metadata *Vals[] = {MDString::get(Ctx, Name)};
101    Args.push_back(MDNode::get(Ctx, Vals));
102  }
103
104  // Setting unroll_and_jam.full or unroll_and_jam.disable
105  if (Attrs.UnrollAndJamEnable != LoopAttributes::Unspecified) {
106    std::string Name;
107    if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable)
108      Name = "llvm.loop.unroll_and_jam.enable";
109    else if (Attrs.UnrollAndJamEnable == LoopAttributes::Full)
110      Name = "llvm.loop.unroll_and_jam.full";
111    else
112      Name = "llvm.loop.unroll_and_jam.disable";
113    Metadata *Vals[] = {MDString::get(Ctx, Name)};
114    Args.push_back(MDNode::get(Ctx, Vals));
115  }
116
117  if (Attrs.DistributeEnable != LoopAttributes::Unspecified) {
118    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
119                        ConstantAsMetadata::get(ConstantInt::get(
120                            Type::getInt1Ty(Ctx), (Attrs.DistributeEnable ==
121                                                   LoopAttributes::Enable)))};
122    Args.push_back(MDNode::get(Ctx, Vals));
123  }
124
125  if (Attrs.IsParallel) {
126    AccGroup = MDNode::getDistinct(Ctx, {});
127    Args.push_back(MDNode::get(
128        Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
129  }
130
131  if (Attrs.PipelineDisabled) {
132    Metadata *Vals[] = {
133        MDString::get(Ctx, "llvm.loop.pipeline.disable"),
134        ConstantAsMetadata::get(ConstantInt::get(
135            Type::getInt1Ty(Ctx), (Attrs.PipelineDisabled == true)))};
136    Args.push_back(MDNode::get(Ctx, Vals));
137  }
138
139  if (Attrs.PipelineInitiationInterval > 0) {
140    Metadata *Vals[] = {
141        MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
142        ConstantAsMetadata::get(ConstantInt::get(
143            Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
144    Args.push_back(MDNode::get(Ctx, Vals));
145  }
146
147  // Set the first operand to itself.
148  MDNode *LoopID = MDNode::get(Ctx, Args);
149  LoopID->replaceOperandWith(0, LoopID);
150  return LoopID;
151}
152
153LoopAttributes::LoopAttributes(bool IsParallel)
154    : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
155      UnrollEnable(LoopAttributes::Unspecified),
156      UnrollAndJamEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
157      InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
158      DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
159      PipelineInitiationInterval(0) {}
160
161void LoopAttributes::clear() {
162  IsParallel = false;
163  VectorizeWidth = 0;
164  InterleaveCount = 0;
165  UnrollCount = 0;
166  UnrollAndJamCount = 0;
167  VectorizeEnable = LoopAttributes::Unspecified;
168  UnrollEnable = LoopAttributes::Unspecified;
169  UnrollAndJamEnable = LoopAttributes::Unspecified;
170  DistributeEnable = LoopAttributes::Unspecified;
171  PipelineDisabled = false;
172  PipelineInitiationInterval = 0;
173}
174
175LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
176                   const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
177    : LoopID(nullptr), Header(Header), Attrs(Attrs) {
178  LoopID =
179      createMetadata(Header->getContext(), Attrs, StartLoc, EndLoc, AccGroup);
180}
181
182void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
183                         const llvm::DebugLoc &EndLoc) {
184  Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc));
185  // Clear the attributes so nested loops do not inherit them.
186  StagedAttrs.clear();
187}
188
189void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
190                         ArrayRef<const clang::Attr *> Attrs,
191                         const llvm::DebugLoc &StartLoc,
192                         const llvm::DebugLoc &EndLoc) {
193
194  // Identify loop hint attributes from Attrs.
195  for (const auto *Attr : Attrs) {
196    const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
197    const OpenCLUnrollHintAttr *OpenCLHint =
198        dyn_cast<OpenCLUnrollHintAttr>(Attr);
199
200    // Skip non loop hint attributes
201    if (!LH && !OpenCLHint) {
202      continue;
203    }
204
205    LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
206    LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
207    unsigned ValueInt = 1;
208    // Translate opencl_unroll_hint attribute argument to
209    // equivalent LoopHintAttr enums.
210    // OpenCL v2.0 s6.11.5:
211    // 0 - enable unroll (no argument).
212    // 1 - disable unroll.
213    // other positive integer n - unroll by n.
214    if (OpenCLHint) {
215      ValueInt = OpenCLHint->getUnrollHint();
216      if (ValueInt == 0) {
217        State = LoopHintAttr::Enable;
218      } else if (ValueInt != 1) {
219        Option = LoopHintAttr::UnrollCount;
220        State = LoopHintAttr::Numeric;
221      }
222    } else if (LH) {
223      auto *ValueExpr = LH->getValue();
224      if (ValueExpr) {
225        llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
226        ValueInt = ValueAPS.getSExtValue();
227      }
228
229      Option = LH->getOption();
230      State = LH->getState();
231    }
232    switch (State) {
233    case LoopHintAttr::Disable:
234      switch (Option) {
235      case LoopHintAttr::Vectorize:
236        // Disable vectorization by specifying a width of 1.
237        setVectorizeWidth(1);
238        break;
239      case LoopHintAttr::Interleave:
240        // Disable interleaving by speciyfing a count of 1.
241        setInterleaveCount(1);
242        break;
243      case LoopHintAttr::Unroll:
244        setUnrollState(LoopAttributes::Disable);
245        break;
246      case LoopHintAttr::UnrollAndJam:
247        setUnrollAndJamState(LoopAttributes::Disable);
248        break;
249      case LoopHintAttr::Distribute:
250        setDistributeState(false);
251        break;
252      case LoopHintAttr::PipelineDisabled:
253        setPipelineDisabled(true);
254        break;
255      case LoopHintAttr::UnrollCount:
256      case LoopHintAttr::UnrollAndJamCount:
257      case LoopHintAttr::VectorizeWidth:
258      case LoopHintAttr::InterleaveCount:
259      case LoopHintAttr::PipelineInitiationInterval:
260        llvm_unreachable("Options cannot be disabled.");
261        break;
262      }
263      break;
264    case LoopHintAttr::Enable:
265      switch (Option) {
266      case LoopHintAttr::Vectorize:
267      case LoopHintAttr::Interleave:
268        setVectorizeEnable(true);
269        break;
270      case LoopHintAttr::Unroll:
271        setUnrollState(LoopAttributes::Enable);
272        break;
273      case LoopHintAttr::UnrollAndJam:
274        setUnrollAndJamState(LoopAttributes::Enable);
275        break;
276      case LoopHintAttr::Distribute:
277        setDistributeState(true);
278        break;
279      case LoopHintAttr::UnrollCount:
280      case LoopHintAttr::UnrollAndJamCount:
281      case LoopHintAttr::VectorizeWidth:
282      case LoopHintAttr::InterleaveCount:
283      case LoopHintAttr::PipelineDisabled:
284      case LoopHintAttr::PipelineInitiationInterval:
285        llvm_unreachable("Options cannot enabled.");
286        break;
287      }
288      break;
289    case LoopHintAttr::AssumeSafety:
290      switch (Option) {
291      case LoopHintAttr::Vectorize:
292      case LoopHintAttr::Interleave:
293        // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
294        setParallel(true);
295        setVectorizeEnable(true);
296        break;
297      case LoopHintAttr::Unroll:
298      case LoopHintAttr::UnrollAndJam:
299      case LoopHintAttr::UnrollCount:
300      case LoopHintAttr::UnrollAndJamCount:
301      case LoopHintAttr::VectorizeWidth:
302      case LoopHintAttr::InterleaveCount:
303      case LoopHintAttr::Distribute:
304      case LoopHintAttr::PipelineDisabled:
305      case LoopHintAttr::PipelineInitiationInterval:
306        llvm_unreachable("Options cannot be used to assume mem safety.");
307        break;
308      }
309      break;
310    case LoopHintAttr::Full:
311      switch (Option) {
312      case LoopHintAttr::Unroll:
313        setUnrollState(LoopAttributes::Full);
314        break;
315      case LoopHintAttr::UnrollAndJam:
316        setUnrollAndJamState(LoopAttributes::Full);
317        break;
318      case LoopHintAttr::Vectorize:
319      case LoopHintAttr::Interleave:
320      case LoopHintAttr::UnrollCount:
321      case LoopHintAttr::UnrollAndJamCount:
322      case LoopHintAttr::VectorizeWidth:
323      case LoopHintAttr::InterleaveCount:
324      case LoopHintAttr::Distribute:
325      case LoopHintAttr::PipelineDisabled:
326      case LoopHintAttr::PipelineInitiationInterval:
327        llvm_unreachable("Options cannot be used with 'full' hint.");
328        break;
329      }
330      break;
331    case LoopHintAttr::Numeric:
332      switch (Option) {
333      case LoopHintAttr::VectorizeWidth:
334        setVectorizeWidth(ValueInt);
335        break;
336      case LoopHintAttr::InterleaveCount:
337        setInterleaveCount(ValueInt);
338        break;
339      case LoopHintAttr::UnrollCount:
340        setUnrollCount(ValueInt);
341        break;
342      case LoopHintAttr::UnrollAndJamCount:
343        setUnrollAndJamCount(ValueInt);
344        break;
345      case LoopHintAttr::PipelineInitiationInterval:
346        setPipelineInitiationInterval(ValueInt);
347        break;
348      case LoopHintAttr::Unroll:
349      case LoopHintAttr::UnrollAndJam:
350      case LoopHintAttr::Vectorize:
351      case LoopHintAttr::Interleave:
352      case LoopHintAttr::Distribute:
353      case LoopHintAttr::PipelineDisabled:
354        llvm_unreachable("Options cannot be assigned a value.");
355        break;
356      }
357      break;
358    }
359  }
360
361  /// Stage the attributes.
362  push(Header, StartLoc, EndLoc);
363}
364
365void LoopInfoStack::pop() {
366   (0) . __assert_fail ("!Active.empty() && \"No active loops to pop\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGLoopInfo.cpp", 366, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!Active.empty() && "No active loops to pop");
367  Active.pop_back();
368}
369
370void LoopInfoStack::InsertHelper(Instruction *I) const {
371  if (I->mayReadOrWriteMemory()) {
372    SmallVector<Metadata *, 4> AccessGroups;
373    for (const LoopInfo &AL : Active) {
374      // Here we assume that every loop that has an access group is parallel.
375      if (MDNode *Group = AL.getAccessGroup())
376        AccessGroups.push_back(Group);
377    }
378    MDNode *UnionMD = nullptr;
379    if (AccessGroups.size() == 1)
380      UnionMD = cast<MDNode>(AccessGroups[0]);
381    else if (AccessGroups.size() >= 2)
382      UnionMD = MDNode::get(I->getContext(), AccessGroups);
383    I->setMetadata("llvm.access.group", UnionMD);
384  }
385
386  if (!hasInfo())
387    return;
388
389  const LoopInfo &L = getInfo();
390  if (!L.getLoopID())
391    return;
392
393  if (I->isTerminator()) {
394    for (BasicBlock *Succ : successors(I))
395      if (Succ == L.getHeader()) {
396        I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
397        break;
398      }
399    return;
400  }
401}
402
clang::CodeGen::LoopAttributes::clear
clang::CodeGen::LoopInfoStack::push
clang::CodeGen::LoopInfoStack::push
clang::CodeGen::LoopInfoStack::pop
clang::CodeGen::LoopInfoStack::InsertHelper
CodeThinker, Enjoy your reading code