CGAtomic.cpp source code [clang_source_code/lib/CodeGen/CGAtomic.cpp]

1	//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file contains the code for emitting atomic operations.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "CGCall.h"
14	#include "CGRecordLayout.h"
15	#include "CodeGenFunction.h"
16	#include "CodeGenModule.h"
17	#include "TargetInfo.h"
18	#include "clang/AST/ASTContext.h"
19	#include "clang/CodeGen/CGFunctionInfo.h"
20	#include "clang/Frontend/FrontendDiagnostic.h"
21	#include "llvm/ADT/DenseMap.h"
22	#include "llvm/IR/DataLayout.h"
23	#include "llvm/IR/Intrinsics.h"
24	#include "llvm/IR/Operator.h"
25
26	using namespace clang;
27	using namespace CodeGen;
28
29	namespace {
30	class AtomicInfo {
31	CodeGenFunction &CGF;
32	QualType AtomicTy;
33	QualType ValueTy;
34	uint64_t AtomicSizeInBits;
35	uint64_t ValueSizeInBits;
36	CharUnits AtomicAlign;
37	CharUnits ValueAlign;
38	TypeEvaluationKind EvaluationKind;
39	bool UseLibcall;
40	LValue LVal;
41	CGBitFieldInfo BFI;
42	public:
43	AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44	: CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45	EvaluationKind(TEK_Scalar), UseLibcall(true) {
46	assert(!lvalue.isGlobalReg());
47	ASTContext &C = CGF.getContext();
48	if (lvalue.isSimple()) {
49	AtomicTy = lvalue.getType();
50	if (auto *ATy = AtomicTy->getAs<AtomicType>())
51	ValueTy = ATy->getValueType();
52	else
53	ValueTy = AtomicTy;
54	EvaluationKind = CGF.getEvaluationKind(ValueTy);
55
56	uint64_t ValueAlignInBits;
57	uint64_t AtomicAlignInBits;
58	TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59	ValueSizeInBits = ValueTI.Width;
60	ValueAlignInBits = ValueTI.Align;
61
62	TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63	AtomicSizeInBits = AtomicTI.Width;
64	AtomicAlignInBits = AtomicTI.Align;
65
66	assert(ValueSizeInBits <= AtomicSizeInBits);
67	assert(ValueAlignInBits <= AtomicAlignInBits);
68
69	AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70	ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71	if (lvalue.getAlignment().isZero())
72	lvalue.setAlignment(AtomicAlign);
73
74	LVal = lvalue;
75	} else if (lvalue.isBitField()) {
76	ValueTy = lvalue.getType();
77	ValueSizeInBits = C.getTypeSize(ValueTy);
78	auto &OrigBFI = lvalue.getBitFieldInfo();
79	auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80	AtomicSizeInBits = C.toBits(
81	C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82	.alignTo(lvalue.getAlignment()));
83	auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84	auto OffsetInChars =
85	(C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86	lvalue.getAlignment();
87	VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88	VoidPtrAddr, OffsetInChars.getQuantity());
89	auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
90	VoidPtrAddr,
91	CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92	"atomic_bitfield_base");
93	BFI = OrigBFI;
94	BFI.Offset = Offset;
95	BFI.StorageSize = AtomicSizeInBits;
96	BFI.StorageOffset += OffsetInChars;
97	LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
98	BFI, lvalue.getType(), lvalue.getBaseInfo(),
99	lvalue.getTBAAInfo());
100	AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
101	if (AtomicTy.isNull()) {
102	llvm::APInt Size(
103	/numBits=/32,
104	C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
105	AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
106	/IndexTypeQuals=/0);
107	}
108	AtomicAlign = ValueAlign = lvalue.getAlignment();
109	} else if (lvalue.isVectorElt()) {
110	ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
111	ValueSizeInBits = C.getTypeSize(ValueTy);
112	AtomicTy = lvalue.getType();
113	AtomicSizeInBits = C.getTypeSize(AtomicTy);
114	AtomicAlign = ValueAlign = lvalue.getAlignment();
115	LVal = lvalue;
116	} else {
117	assert(lvalue.isExtVectorElt());
118	ValueTy = lvalue.getType();
119	ValueSizeInBits = C.getTypeSize(ValueTy);
120	AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
121	lvalue.getType(), lvalue.getExtVectorAddress()
122	.getElementType()->getVectorNumElements());
123	AtomicSizeInBits = C.getTypeSize(AtomicTy);
124	AtomicAlign = ValueAlign = lvalue.getAlignment();
125	LVal = lvalue;
126	}
127	UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
128	AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
129	}
130
131	QualType getAtomicType() const { return AtomicTy; }
132	QualType getValueType() const { return ValueTy; }
133	CharUnits getAtomicAlignment() const { return AtomicAlign; }
134	uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
135	uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
136	TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
137	bool shouldUseLibcall() const { return UseLibcall; }
138	const LValue &getAtomicLValue() const { return LVal; }
139	llvm::Value *getAtomicPointer() const {
140	if (LVal.isSimple())
141	return LVal.getPointer();
142	else if (LVal.isBitField())
143	return LVal.getBitFieldPointer();
144	else if (LVal.isVectorElt())
145	return LVal.getVectorPointer();
146	assert(LVal.isExtVectorElt());
147	return LVal.getExtVectorPointer();
148	}
149	Address getAtomicAddress() const {
150	return Address(getAtomicPointer(), getAtomicAlignment());
151	}
152
153	Address getAtomicAddressAsAtomicIntPointer() const {
154	return emitCastToAtomicIntPointer(getAtomicAddress());
155	}
156
157	/// Is the atomic size larger than the underlying value type?
158	///
159	/// Note that the absence of padding does not mean that atomic
160	/// objects are completely interchangeable with non-atomic
161	/// objects: we might have promoted the alignment of a type
162	/// without making it bigger.
163	bool hasPadding() const {
164	return (ValueSizeInBits != AtomicSizeInBits);
165	}
166
167	bool emitMemSetZeroIfNecessary() const;
168
169	llvm::Value *getAtomicSizeValue() const {
170	CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
171	return CGF.CGM.getSize(size);
172	}
173
174	/// Cast the given pointer to an integer pointer suitable for atomic
175	/// operations if the source.
176	Address emitCastToAtomicIntPointer(Address Addr) const;
177
178	/// If Addr is compatible with the iN that will be used for an atomic
179	/// operation, bitcast it. Otherwise, create a temporary that is suitable
180	/// and copy the value across.
181	Address convertToAtomicIntPointer(Address Addr) const;
182
183	/// Turn an atomic-layout object into an r-value.
184	RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
185	SourceLocation loc, bool AsValue) const;
186
187	/// Converts a rvalue to integer value.
188	llvm::Value *convertRValueToInt(RValue RVal) const;
189
190	RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
191	AggValueSlot ResultSlot,
192	SourceLocation Loc, bool AsValue) const;
193
194	/// Copy an atomic r-value into atomic-layout memory.
195	void emitCopyIntoMemory(RValue rvalue) const;
196
197	/// Project an l-value down to the value field.
198	LValue projectValue() const {
199	assert(LVal.isSimple());
200	Address addr = getAtomicAddress();
201	if (hasPadding())
202	addr = CGF.Builder.CreateStructGEP(addr, 0);
203
204	return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
205	LVal.getBaseInfo(), LVal.getTBAAInfo());
206	}
207
208	/// Emits atomic load.
209	/// \returns Loaded value.
210	RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
211	bool AsValue, llvm::AtomicOrdering AO,
212	bool IsVolatile);
213
214	/// Emits atomic compare-and-exchange sequence.
215	/// \param Expected Expected value.
216	/// \param Desired Desired value.
217	/// \param Success Atomic ordering for success operation.
218	/// \param Failure Atomic ordering for failed operation.
219	/// \param IsWeak true if atomic operation is weak, false otherwise.
220	/// \returns Pair of values: previous value from storage (value type) and
221	/// boolean flag (i1 type) with true if success and false otherwise.
222	std::pair<RValue, llvm::Value *>
223	EmitAtomicCompareExchange(RValue Expected, RValue Desired,
224	llvm::AtomicOrdering Success =
225	llvm::AtomicOrdering::SequentiallyConsistent,
226	llvm::AtomicOrdering Failure =
227	llvm::AtomicOrdering::SequentiallyConsistent,
228	bool IsWeak = false);
229
230	/// Emits atomic update.
231	/// \param AO Atomic ordering.
232	/// \param UpdateOp Update operation for the current lvalue.
233	void EmitAtomicUpdate(llvm::AtomicOrdering AO,
234	const llvm::function_ref<RValue(RValue)> &UpdateOp,
235	bool IsVolatile);
236	/// Emits atomic update.
237	/// \param AO Atomic ordering.
238	void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
239	bool IsVolatile);
240
241	/// Materialize an atomic r-value in atomic-layout memory.
242	Address materializeRValue(RValue rvalue) const;
243
244	/// Creates temp alloca for intermediate operations on atomic value.
245	Address CreateTempAlloca() const;
246	private:
247	bool requiresMemSetZero(llvm::Type *type) const;
248
249
250	/// Emits atomic load as a libcall.
251	void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
252	llvm::AtomicOrdering AO, bool IsVolatile);
253	/// Emits atomic load as LLVM instruction.
254	llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
255	/// Emits atomic compare-and-exchange op as a libcall.
256	llvm::Value *EmitAtomicCompareExchangeLibcall(
257	llvm::Value ExpectedAddr, llvm::Value DesiredAddr,
258	llvm::AtomicOrdering Success =
259	llvm::AtomicOrdering::SequentiallyConsistent,
260	llvm::AtomicOrdering Failure =
261	llvm::AtomicOrdering::SequentiallyConsistent);
262	/// Emits atomic compare-and-exchange op as LLVM instruction.
263	std::pair<llvm::Value , llvm::Value > EmitAtomicCompareExchangeOp(
264	llvm::Value ExpectedVal, llvm::Value DesiredVal,
265	llvm::AtomicOrdering Success =
266	llvm::AtomicOrdering::SequentiallyConsistent,
267	llvm::AtomicOrdering Failure =
268	llvm::AtomicOrdering::SequentiallyConsistent,
269	bool IsWeak = false);
270	/// Emit atomic update as libcalls.
271	void
272	EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
273	const llvm::function_ref<RValue(RValue)> &UpdateOp,
274	bool IsVolatile);
275	/// Emit atomic update as LLVM instructions.
276	void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
277	const llvm::function_ref<RValue(RValue)> &UpdateOp,
278	bool IsVolatile);
279	/// Emit atomic update as libcalls.
280	void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
281	bool IsVolatile);
282	/// Emit atomic update as LLVM instructions.
283	void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
284	bool IsVolatile);
285	};
286	}
287
288	Address AtomicInfo::CreateTempAlloca() const {
289	Address TempAlloca = CGF.CreateMemTemp(
290	(LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
291	: AtomicTy,
292	getAtomicAlignment(),
293	"atomic-temp");
294	// Cast to pointer to value type for bitfields.
295	if (LVal.isBitField())
296	return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
297	TempAlloca, getAtomicAddress().getType());
298	return TempAlloca;
299	}
300
301	static RValue emitAtomicLibcall(CodeGenFunction &CGF,
302	StringRef fnName,
303	QualType resultType,
304	CallArgList &args) {
305	const CGFunctionInfo &fnInfo =
306	CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
307	llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
308	llvm::FunctionCallee fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
309	auto callee = CGCallee::forDirect(fn);
310	return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
311	}
312
313	/// Does a store of the given IR type modify the full expected width?
314	static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
315	uint64_t expectedSize) {
316	return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
317	}
318
319	/// Does the atomic type require memsetting to zero before initialization?
320	///
321	/// The IR type is provided as a way of making certain queries faster.
322	bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
323	// If the atomic type has size padding, we definitely need a memset.
324	if (hasPadding()) return true;
325
326	// Otherwise, do some simple heuristics to try to avoid it:
327	switch (getEvaluationKind()) {
328	// For scalars and complexes, check whether the store size of the
329	// type uses the full size.
330	case TEK_Scalar:
331	return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
332	case TEK_Complex:
333	return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
334	AtomicSizeInBits / 2);
335
336	// Padding in structs has an undefined bit pattern. User beware.
337	case TEK_Aggregate:
338	return false;
339	}
340	llvm_unreachable("bad evaluation kind");
341	}
342
343	bool AtomicInfo::emitMemSetZeroIfNecessary() const {
344	assert(LVal.isSimple());
345	llvm::Value *addr = LVal.getPointer();
346	if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
347	return false;
348
349	CGF.Builder.CreateMemSet(
350	addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
351	CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
352	LVal.getAlignment().getQuantity());
353	return true;
354	}
355
356	static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
357	Address Dest, Address Ptr,
358	Address Val1, Address Val2,
359	uint64_t Size,
360	llvm::AtomicOrdering SuccessOrder,
361	llvm::AtomicOrdering FailureOrder,
362	llvm::SyncScope::ID Scope) {
363	// Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
364	llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
365	llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
366
367	llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
368	Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
369	Scope);
370	Pair->setVolatile(E->isVolatile());
371	Pair->setWeak(IsWeak);
372
373	// Cmp holds the result of the compare-exchange operation: true on success,
374	// false on failure.
375	llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
376	llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
377
378	// This basic block is used to hold the store instruction if the operation
379	// failed.
380	llvm::BasicBlock *StoreExpectedBB =
381	CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
382
383	// This basic block is the exit point of the operation, we should end up
384	// here regardless of whether or not the operation succeeded.
385	llvm::BasicBlock *ContinueBB =
386	CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
387
388	// Update Expected if Expected isn't equal to Old, otherwise branch to the
389	// exit point.
390	CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
391
392	CGF.Builder.SetInsertPoint(StoreExpectedBB);
393	// Update the memory at Expected with Old's value.
394	CGF.Builder.CreateStore(Old, Val1);
395	// Finally, branch to the exit point.
396	CGF.Builder.CreateBr(ContinueBB);
397
398	CGF.Builder.SetInsertPoint(ContinueBB);
399	// Update the memory at Dest with Cmp's value.
400	CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
401	}
402
403	/// Given an ordering required on success, emit all possible cmpxchg
404	/// instructions to cope with the provided (but possibly only dynamically known)
405	/// FailureOrder.
406	static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
407	bool IsWeak, Address Dest, Address Ptr,
408	Address Val1, Address Val2,
409	llvm::Value *FailureOrderVal,
410	uint64_t Size,
411	llvm::AtomicOrdering SuccessOrder,
412	llvm::SyncScope::ID Scope) {
413	llvm::AtomicOrdering FailureOrder;
414	if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
415	auto FOS = FO->getSExtValue();
416	if (!llvm::isValidAtomicOrderingCABI(FOS))
417	FailureOrder = llvm::AtomicOrdering::Monotonic;
418	else
419	switch ((llvm::AtomicOrderingCABI)FOS) {
420	case llvm::AtomicOrderingCABI::relaxed:
421	case llvm::AtomicOrderingCABI::release:
422	case llvm::AtomicOrderingCABI::acq_rel:
423	FailureOrder = llvm::AtomicOrdering::Monotonic;
424	break;
425	case llvm::AtomicOrderingCABI::consume:
426	case llvm::AtomicOrderingCABI::acquire:
427	FailureOrder = llvm::AtomicOrdering::Acquire;
428	break;
429	case llvm::AtomicOrderingCABI::seq_cst:
430	FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
431	break;
432	}
433	if (isStrongerThan(FailureOrder, SuccessOrder)) {
434	// Don't assert on undefined behavior "failure argument shall be no
435	// stronger than the success argument".
436	FailureOrder =
437	llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
438	}
439	emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
440	FailureOrder, Scope);
441	return;
442	}
443
444	// Create all the relevant BB's
445	llvm::BasicBlock MonotonicBB = nullptr, AcquireBB = nullptr,
446	*SeqCstBB = nullptr;
447	MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
448	if (SuccessOrder != llvm::AtomicOrdering::Monotonic &&
449	SuccessOrder != llvm::AtomicOrdering::Release)
450	AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
451	if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent)
452	SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
453
454	llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
455
456	llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
457
458	// Emit all the different atomics
459
460	// MonotonicBB is arbitrarily chosen as the default case; in practice, this
461	// doesn't matter unless someone is crazy enough to use something that
462	// doesn't fold to a constant for the ordering.
463	CGF.Builder.SetInsertPoint(MonotonicBB);
464	emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
465	Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
466	CGF.Builder.CreateBr(ContBB);
467
468	if (AcquireBB) {
469	CGF.Builder.SetInsertPoint(AcquireBB);
470	emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
471	Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope);
472	CGF.Builder.CreateBr(ContBB);
473	SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
474	AcquireBB);
475	SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
476	AcquireBB);
477	}
478	if (SeqCstBB) {
479	CGF.Builder.SetInsertPoint(SeqCstBB);
480	emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
481	llvm::AtomicOrdering::SequentiallyConsistent, Scope);
482	CGF.Builder.CreateBr(ContBB);
483	SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
484	SeqCstBB);
485	}
486
487	CGF.Builder.SetInsertPoint(ContBB);
488	}
489
490	static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
491	Address Ptr, Address Val1, Address Val2,
492	llvm::Value IsWeak, llvm::Value FailureOrder,
493	uint64_t Size, llvm::AtomicOrdering Order,
494	llvm::SyncScope::ID Scope) {
495	llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
496	llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
497
498	switch (E->getOp()) {
499	case AtomicExpr::AO__c11_atomic_init:
500	case AtomicExpr::AO__opencl_atomic_init:
501	llvm_unreachable("Already handled!");
502
503	case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
504	case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
505	emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
506	FailureOrder, Size, Order, Scope);
507	return;
508	case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
509	case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
510	emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
511	FailureOrder, Size, Order, Scope);
512	return;
513	case AtomicExpr::AO__atomic_compare_exchange:
514	case AtomicExpr::AO__atomic_compare_exchange_n: {
515	if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
516	emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
517	Val1, Val2, FailureOrder, Size, Order, Scope);
518	} else {
519	// Create all the relevant BB's
520	llvm::BasicBlock *StrongBB =
521	CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
522	llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
523	llvm::BasicBlock *ContBB =
524	CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
525
526	llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
527	SI->addCase(CGF.Builder.getInt1(false), StrongBB);
528
529	CGF.Builder.SetInsertPoint(StrongBB);
530	emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
531	FailureOrder, Size, Order, Scope);
532	CGF.Builder.CreateBr(ContBB);
533
534	CGF.Builder.SetInsertPoint(WeakBB);
535	emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
536	FailureOrder, Size, Order, Scope);
537	CGF.Builder.CreateBr(ContBB);
538
539	CGF.Builder.SetInsertPoint(ContBB);
540	}
541	return;
542	}
543	case AtomicExpr::AO__c11_atomic_load:
544	case AtomicExpr::AO__opencl_atomic_load:
545	case AtomicExpr::AO__atomic_load_n:
546	case AtomicExpr::AO__atomic_load: {
547	llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
548	Load->setAtomic(Order, Scope);
549	Load->setVolatile(E->isVolatile());
550	CGF.Builder.CreateStore(Load, Dest);
551	return;
552	}
553
554	case AtomicExpr::AO__c11_atomic_store:
555	case AtomicExpr::AO__opencl_atomic_store:
556	case AtomicExpr::AO__atomic_store:
557	case AtomicExpr::AO__atomic_store_n: {
558	llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
559	llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
560	Store->setAtomic(Order, Scope);
561	Store->setVolatile(E->isVolatile());
562	return;
563	}
564
565	case AtomicExpr::AO__c11_atomic_exchange:
566	case AtomicExpr::AO__opencl_atomic_exchange:
567	case AtomicExpr::AO__atomic_exchange_n:
568	case AtomicExpr::AO__atomic_exchange:
569	Op = llvm::AtomicRMWInst::Xchg;
570	break;
571
572	case AtomicExpr::AO__atomic_add_fetch:
573	PostOp = llvm::Instruction::Add;
574	LLVM_FALLTHROUGH;
575	case AtomicExpr::AO__c11_atomic_fetch_add:
576	case AtomicExpr::AO__opencl_atomic_fetch_add:
577	case AtomicExpr::AO__atomic_fetch_add:
578	Op = llvm::AtomicRMWInst::Add;
579	break;
580
581	case AtomicExpr::AO__atomic_sub_fetch:
582	PostOp = llvm::Instruction::Sub;
583	LLVM_FALLTHROUGH;
584	case AtomicExpr::AO__c11_atomic_fetch_sub:
585	case AtomicExpr::AO__opencl_atomic_fetch_sub:
586	case AtomicExpr::AO__atomic_fetch_sub:
587	Op = llvm::AtomicRMWInst::Sub;
588	break;
589
590	case AtomicExpr::AO__opencl_atomic_fetch_min:
591	case AtomicExpr::AO__atomic_fetch_min:
592	Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min
593	: llvm::AtomicRMWInst::UMin;
594	break;
595
596	case AtomicExpr::AO__opencl_atomic_fetch_max:
597	case AtomicExpr::AO__atomic_fetch_max:
598	Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max
599	: llvm::AtomicRMWInst::UMax;
600	break;
601
602	case AtomicExpr::AO__atomic_and_fetch:
603	PostOp = llvm::Instruction::And;
604	LLVM_FALLTHROUGH;
605	case AtomicExpr::AO__c11_atomic_fetch_and:
606	case AtomicExpr::AO__opencl_atomic_fetch_and:
607	case AtomicExpr::AO__atomic_fetch_and:
608	Op = llvm::AtomicRMWInst::And;
609	break;
610
611	case AtomicExpr::AO__atomic_or_fetch:
612	PostOp = llvm::Instruction::Or;
613	LLVM_FALLTHROUGH;
614	case AtomicExpr::AO__c11_atomic_fetch_or:
615	case AtomicExpr::AO__opencl_atomic_fetch_or:
616	case AtomicExpr::AO__atomic_fetch_or:
617	Op = llvm::AtomicRMWInst::Or;
618	break;
619
620	case AtomicExpr::AO__atomic_xor_fetch:
621	PostOp = llvm::Instruction::Xor;
622	LLVM_FALLTHROUGH;
623	case AtomicExpr::AO__c11_atomic_fetch_xor:
624	case AtomicExpr::AO__opencl_atomic_fetch_xor:
625	case AtomicExpr::AO__atomic_fetch_xor:
626	Op = llvm::AtomicRMWInst::Xor;
627	break;
628
629	case AtomicExpr::AO__atomic_nand_fetch:
630	PostOp = llvm::Instruction::And; // the NOT is special cased below
631	LLVM_FALLTHROUGH;
632	case AtomicExpr::AO__atomic_fetch_nand:
633	Op = llvm::AtomicRMWInst::Nand;
634	break;
635	}
636
637	llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
638	llvm::AtomicRMWInst *RMWI =
639	CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
640	RMWI->setVolatile(E->isVolatile());
641
642	// For __atomic_*_fetch operations, perform the operation again to
643	// determine the value which was written.
644	llvm::Value *Result = RMWI;
645	if (PostOp)
646	Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
647	if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
648	Result = CGF.Builder.CreateNot(Result);
649	CGF.Builder.CreateStore(Result, Dest);
650	}
651
652	// This function emits any expression (scalar, complex, or aggregate)
653	// into a temporary alloca.
654	static Address
655	EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
656	Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
657	CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
658	/Init/ true);
659	return DeclPtr;
660	}
661
662	static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
663	Address Ptr, Address Val1, Address Val2,
664	llvm::Value IsWeak, llvm::Value FailureOrder,
665	uint64_t Size, llvm::AtomicOrdering Order,
666	llvm::Value *Scope) {
667	auto ScopeModel = Expr->getScopeModel();
668
669	// LLVM atomic instructions always have synch scope. If clang atomic
670	// expression has no scope operand, use default LLVM synch scope.
671	if (!ScopeModel) {
672	EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
673	Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
674	return;
675	}
676
677	// Handle constant scope.
678	if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
679	auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
680	CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
681	Order, CGF.CGM.getLLVMContext());
682	EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
683	Order, SCID);
684	return;
685	}
686
687	// Handle non-constant scope.
688	auto &Builder = CGF.Builder;
689	auto Scopes = ScopeModel->getRuntimeValues();
690	llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
691	for (auto S : Scopes)
692	BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
693
694	llvm::BasicBlock *ContBB =
695	CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
696
697	auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
698	// If unsupported synch scope is encountered at run time, assume a fallback
699	// synch scope value.
700	auto FallBack = ScopeModel->getFallBackValue();
701	llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
702	for (auto S : Scopes) {
703	auto *B = BB[S];
704	if (S != FallBack)
705	SI->addCase(Builder.getInt32(S), B);
706
707	Builder.SetInsertPoint(B);
708	EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
709	Order,
710	CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
711	ScopeModel->map(S),
712	Order,
713	CGF.getLLVMContext()));
714	Builder.CreateBr(ContBB);
715	}
716
717	Builder.SetInsertPoint(ContBB);
718	}
719
720	static void
721	AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
722	bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
723	SourceLocation Loc, CharUnits SizeInChars) {
724	if (UseOptimizedLibcall) {
725	// Load value and pass it to the function directly.
726	CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
727	int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
728	ValTy =
729	CGF.getContext().getIntTypeForBitwidth(SizeInBits, /Signed=/false);
730	llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
731	SizeInBits)->getPointerTo();
732	Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
733	Val = CGF.EmitLoadOfScalar(Ptr, false,
734	CGF.getContext().getPointerType(ValTy),
735	Loc);
736	// Coerce the value into an appropriately sized integer type.
737	Args.add(RValue::get(Val), ValTy);
738	} else {
739	// Non-optimized functions always take a reference.
740	Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
741	CGF.getContext().VoidPtrTy);
742	}
743	}
744
745	RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
746	QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
747	QualType MemTy = AtomicTy;
748	if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
749	MemTy = AT->getValueType();
750	llvm::Value IsWeak = nullptr, OrderFail = nullptr;
751
752	Address Val1 = Address::invalid();
753	Address Val2 = Address::invalid();
754	Address Dest = Address::invalid();
755	Address Ptr = EmitPointerWithAlignment(E->getPtr());
756
757	if (E->getOp() == AtomicExpr::AO__c11_atomic_init \|\|
758	E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
759	LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
760	EmitAtomicInit(E->getVal1(), lvalue);
761	return RValue::get(nullptr);
762	}
763
764	CharUnits sizeChars, alignChars;
765	std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
766	uint64_t Size = sizeChars.getQuantity();
767	unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
768
769	bool Oversized = getContext().toBits(sizeChars) > MaxInlineWidthInBits;
770	bool Misaligned = (Ptr.getAlignment() % sizeChars) != 0;
771	bool UseLibcall = Misaligned \| Oversized;
772
773	if (UseLibcall) {
774	CGM.getDiags().Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
775	<< !Oversized;
776	}
777
778	llvm::Value *Order = EmitScalarExpr(E->getOrder());
779	llvm::Value *Scope =
780	E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
781
782	switch (E->getOp()) {
783	case AtomicExpr::AO__c11_atomic_init:
784	case AtomicExpr::AO__opencl_atomic_init:
785	llvm_unreachable("Already handled above with EmitAtomicInit!");
786
787	case AtomicExpr::AO__c11_atomic_load:
788	case AtomicExpr::AO__opencl_atomic_load:
789	case AtomicExpr::AO__atomic_load_n:
790	break;
791
792	case AtomicExpr::AO__atomic_load:
793	Dest = EmitPointerWithAlignment(E->getVal1());
794	break;
795
796	case AtomicExpr::AO__atomic_store:
797	Val1 = EmitPointerWithAlignment(E->getVal1());
798	break;
799
800	case AtomicExpr::AO__atomic_exchange:
801	Val1 = EmitPointerWithAlignment(E->getVal1());
802	Dest = EmitPointerWithAlignment(E->getVal2());
803	break;
804
805	case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
806	case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
807	case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
808	case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
809	case AtomicExpr::AO__atomic_compare_exchange_n:
810	case AtomicExpr::AO__atomic_compare_exchange:
811	Val1 = EmitPointerWithAlignment(E->getVal1());
812	if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
813	Val2 = EmitPointerWithAlignment(E->getVal2());
814	else
815	Val2 = EmitValToTemp(*this, E->getVal2());
816	OrderFail = EmitScalarExpr(E->getOrderFail());
817	if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n \|\|
818	E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
819	IsWeak = EmitScalarExpr(E->getWeak());
820	break;
821
822	case AtomicExpr::AO__c11_atomic_fetch_add:
823	case AtomicExpr::AO__c11_atomic_fetch_sub:
824	case AtomicExpr::AO__opencl_atomic_fetch_add:
825	case AtomicExpr::AO__opencl_atomic_fetch_sub:
826	if (MemTy->isPointerType()) {
827	// For pointer arithmetic, we're required to do a bit of math:
828	// adding 1 to an int* is not the same as adding 1 to a uintptr_t.
829	// ... but only for the C11 builtins. The GNU builtins expect the
830	// user to multiply by sizeof(T).
831	QualType Val1Ty = E->getVal1()->getType();
832	llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
833	CharUnits PointeeIncAmt =
834	getContext().getTypeSizeInChars(MemTy->getPointeeType());
835	Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
836	auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
837	Val1 = Temp;
838	EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
839	break;
840	}
841	LLVM_FALLTHROUGH;
842	case AtomicExpr::AO__atomic_fetch_add:
843	case AtomicExpr::AO__atomic_fetch_sub:
844	case AtomicExpr::AO__atomic_add_fetch:
845	case AtomicExpr::AO__atomic_sub_fetch:
846	case AtomicExpr::AO__c11_atomic_store:
847	case AtomicExpr::AO__c11_atomic_exchange:
848	case AtomicExpr::AO__opencl_atomic_store:
849	case AtomicExpr::AO__opencl_atomic_exchange:
850	case AtomicExpr::AO__atomic_store_n:
851	case AtomicExpr::AO__atomic_exchange_n:
852	case AtomicExpr::AO__c11_atomic_fetch_and:
853	case AtomicExpr::AO__c11_atomic_fetch_or:
854	case AtomicExpr::AO__c11_atomic_fetch_xor:
855	case AtomicExpr::AO__opencl_atomic_fetch_and:
856	case AtomicExpr::AO__opencl_atomic_fetch_or:
857	case AtomicExpr::AO__opencl_atomic_fetch_xor:
858	case AtomicExpr::AO__opencl_atomic_fetch_min:
859	case AtomicExpr::AO__opencl_atomic_fetch_max:
860	case AtomicExpr::AO__atomic_fetch_and:
861	case AtomicExpr::AO__atomic_fetch_or:
862	case AtomicExpr::AO__atomic_fetch_xor:
863	case AtomicExpr::AO__atomic_fetch_nand:
864	case AtomicExpr::AO__atomic_and_fetch:
865	case AtomicExpr::AO__atomic_or_fetch:
866	case AtomicExpr::AO__atomic_xor_fetch:
867	case AtomicExpr::AO__atomic_nand_fetch:
868	case AtomicExpr::AO__atomic_fetch_min:
869	case AtomicExpr::AO__atomic_fetch_max:
870	Val1 = EmitValToTemp(*this, E->getVal1());
871	break;
872	}
873
874	QualType RValTy = E->getType().getUnqualifiedType();
875
876	// The inlined atomics only function on iN types, where N is a power of 2. We
877	// need to make sure (via temporaries if necessary) that all incoming values
878	// are compatible.
879	LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
880	AtomicInfo Atomics(*this, AtomicVal);
881
882	Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
883	if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
884	if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
885	if (Dest.isValid())
886	Dest = Atomics.emitCastToAtomicIntPointer(Dest);
887	else if (E->isCmpXChg())
888	Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
889	else if (!RValTy->isVoidType())
890	Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
891
892	// Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
893	if (UseLibcall) {
894	bool UseOptimizedLibcall = false;
895	switch (E->getOp()) {
896	case AtomicExpr::AO__c11_atomic_init:
897	case AtomicExpr::AO__opencl_atomic_init:
898	llvm_unreachable("Already handled above with EmitAtomicInit!");
899
900	case AtomicExpr::AO__c11_atomic_fetch_add:
901	case AtomicExpr::AO__opencl_atomic_fetch_add:
902	case AtomicExpr::AO__atomic_fetch_add:
903	case AtomicExpr::AO__c11_atomic_fetch_and:
904	case AtomicExpr::AO__opencl_atomic_fetch_and:
905	case AtomicExpr::AO__atomic_fetch_and:
906	case AtomicExpr::AO__c11_atomic_fetch_or:
907	case AtomicExpr::AO__opencl_atomic_fetch_or:
908	case AtomicExpr::AO__atomic_fetch_or:
909	case AtomicExpr::AO__atomic_fetch_nand:
910	case AtomicExpr::AO__c11_atomic_fetch_sub:
911	case AtomicExpr::AO__opencl_atomic_fetch_sub:
912	case AtomicExpr::AO__atomic_fetch_sub:
913	case AtomicExpr::AO__c11_atomic_fetch_xor:
914	case AtomicExpr::AO__opencl_atomic_fetch_xor:
915	case AtomicExpr::AO__opencl_atomic_fetch_min:
916	case AtomicExpr::AO__opencl_atomic_fetch_max:
917	case AtomicExpr::AO__atomic_fetch_xor:
918	case AtomicExpr::AO__atomic_add_fetch:
919	case AtomicExpr::AO__atomic_and_fetch:
920	case AtomicExpr::AO__atomic_nand_fetch:
921	case AtomicExpr::AO__atomic_or_fetch:
922	case AtomicExpr::AO__atomic_sub_fetch:
923	case AtomicExpr::AO__atomic_xor_fetch:
924	case AtomicExpr::AO__atomic_fetch_min:
925	case AtomicExpr::AO__atomic_fetch_max:
926	// For these, only library calls for certain sizes exist.
927	UseOptimizedLibcall = true;
928	break;
929
930	case AtomicExpr::AO__atomic_load:
931	case AtomicExpr::AO__atomic_store:
932	case AtomicExpr::AO__atomic_exchange:
933	case AtomicExpr::AO__atomic_compare_exchange:
934	// Use the generic version if we don't know that the operand will be
935	// suitably aligned for the optimized version.
936	if (Misaligned)
937	break;
938	LLVM_FALLTHROUGH;
939	case AtomicExpr::AO__c11_atomic_load:
940	case AtomicExpr::AO__c11_atomic_store:
941	case AtomicExpr::AO__c11_atomic_exchange:
942	case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
943	case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
944	case AtomicExpr::AO__opencl_atomic_load:
945	case AtomicExpr::AO__opencl_atomic_store:
946	case AtomicExpr::AO__opencl_atomic_exchange:
947	case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
948	case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
949	case AtomicExpr::AO__atomic_load_n:
950	case AtomicExpr::AO__atomic_store_n:
951	case AtomicExpr::AO__atomic_exchange_n:
952	case AtomicExpr::AO__atomic_compare_exchange_n:
953	// Only use optimized library calls for sizes for which they exist.
954	// FIXME: Size == 16 optimized library functions exist too.
955	if (Size == 1 \|\| Size == 2 \|\| Size == 4 \|\| Size == 8)
956	UseOptimizedLibcall = true;
957	break;
958	}
959
960	CallArgList Args;
961	if (!UseOptimizedLibcall) {
962	// For non-optimized library calls, the size is the first parameter
963	Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
964	getContext().getSizeType());
965	}
966	// Atomic address is the first or second parameter
967	// The OpenCL atomic library functions only accept pointer arguments to
968	// generic address space.
969	auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
970	if (!E->isOpenCL())
971	return V;
972	auto AS = PT->getAs<PointerType>()->getPointeeType().getAddressSpace();
973	if (AS == LangAS::opencl_generic)
974	return V;
975	auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
976	auto T = V->getType();
977	auto *DestType = T->getPointerElementType()->getPointerTo(DestAS);
978
979	return getTargetHooks().performAddrSpaceCast(
980	*this, V, AS, LangAS::opencl_generic, DestType, false);
981	};
982
983	Args.add(RValue::get(CastToGenericAddrSpace(
984	EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
985	getContext().VoidPtrTy);
986
987	std::string LibCallName;
988	QualType LoweredMemTy =
989	MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
990	QualType RetTy;
991	bool HaveRetTy = false;
992	llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
993	switch (E->getOp()) {
994	case AtomicExpr::AO__c11_atomic_init:
995	case AtomicExpr::AO__opencl_atomic_init:
996	llvm_unreachable("Already handled!");
997
998	// There is only one libcall for compare an exchange, because there is no
999	// optimisation benefit possible from a libcall version of a weak compare
1000	// and exchange.
1001	// bool __atomic_compare_exchange(size_t size, void mem, void expected,
1002	// void *desired, int success, int failure)
1003	// bool __atomic_compare_exchange_N(T mem, T expected, T desired,
1004	// int success, int failure)
1005	case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1006	case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1007	case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1008	case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1009	case AtomicExpr::AO__atomic_compare_exchange:
1010	case AtomicExpr::AO__atomic_compare_exchange_n:
1011	LibCallName = "__atomic_compare_exchange";
1012	RetTy = getContext().BoolTy;
1013	HaveRetTy = true;
1014	Args.add(
1015	RValue::get(CastToGenericAddrSpace(
1016	EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1017	getContext().VoidPtrTy);
1018	AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1019	MemTy, E->getExprLoc(), sizeChars);
1020	Args.add(RValue::get(Order), getContext().IntTy);
1021	Order = OrderFail;
1022	break;
1023	// void __atomic_exchange(size_t size, void mem, void val, void *return,
1024	// int order)
1025	// T __atomic_exchange_N(T *mem, T val, int order)
1026	case AtomicExpr::AO__c11_atomic_exchange:
1027	case AtomicExpr::AO__opencl_atomic_exchange:
1028	case AtomicExpr::AO__atomic_exchange_n:
1029	case AtomicExpr::AO__atomic_exchange:
1030	LibCallName = "__atomic_exchange";
1031	AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1032	MemTy, E->getExprLoc(), sizeChars);
1033	break;
1034	// void __atomic_store(size_t size, void mem, void val, int order)
1035	// void __atomic_store_N(T *mem, T val, int order)
1036	case AtomicExpr::AO__c11_atomic_store:
1037	case AtomicExpr::AO__opencl_atomic_store:
1038	case AtomicExpr::AO__atomic_store:
1039	case AtomicExpr::AO__atomic_store_n:
1040	LibCallName = "__atomic_store";
1041	RetTy = getContext().VoidTy;
1042	HaveRetTy = true;
1043	AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1044	MemTy, E->getExprLoc(), sizeChars);
1045	break;
1046	// void __atomic_load(size_t size, void mem, void return, int order)
1047	// T __atomic_load_N(T *mem, int order)
1048	case AtomicExpr::AO__c11_atomic_load:
1049	case AtomicExpr::AO__opencl_atomic_load:
1050	case AtomicExpr::AO__atomic_load:
1051	case AtomicExpr::AO__atomic_load_n:
1052	LibCallName = "__atomic_load";
1053	break;
1054	// T __atomic_add_fetch_N(T *mem, T val, int order)
1055	// T __atomic_fetch_add_N(T *mem, T val, int order)
1056	case AtomicExpr::AO__atomic_add_fetch:
1057	PostOp = llvm::Instruction::Add;
1058	LLVM_FALLTHROUGH;
1059	case AtomicExpr::AO__c11_atomic_fetch_add:
1060	case AtomicExpr::AO__opencl_atomic_fetch_add:
1061	case AtomicExpr::AO__atomic_fetch_add:
1062	LibCallName = "__atomic_fetch_add";
1063	AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1064	LoweredMemTy, E->getExprLoc(), sizeChars);
1065	break;
1066	// T __atomic_and_fetch_N(T *mem, T val, int order)
1067	// T __atomic_fetch_and_N(T *mem, T val, int order)
1068	case AtomicExpr::AO__atomic_and_fetch:
1069	PostOp = llvm::Instruction::And;
1070	LLVM_FALLTHROUGH;
1071	case AtomicExpr::AO__c11_atomic_fetch_and:
1072	case AtomicExpr::AO__opencl_atomic_fetch_and:
1073	case AtomicExpr::AO__atomic_fetch_and:
1074	LibCallName = "__atomic_fetch_and";
1075	AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1076	MemTy, E->getExprLoc(), sizeChars);
1077	break;
1078	// T __atomic_or_fetch_N(T *mem, T val, int order)
1079	// T __atomic_fetch_or_N(T *mem, T val, int order)
1080	case AtomicExpr::AO__atomic_or_fetch:
1081	PostOp = llvm::Instruction::Or;
1082	LLVM_FALLTHROUGH;
1083	case AtomicExpr::AO__c11_atomic_fetch_or:
1084	case AtomicExpr::AO__opencl_atomic_fetch_or:
1085	case AtomicExpr::AO__atomic_fetch_or:
1086	LibCallName = "__atomic_fetch_or";
1087	AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1088	MemTy, E->getExprLoc(), sizeChars);
1089	break;
1090	// T __atomic_sub_fetch_N(T *mem, T val, int order)
1091	// T __atomic_fetch_sub_N(T *mem, T val, int order)
1092	case AtomicExpr::AO__atomic_sub_fetch:
1093	PostOp = llvm::Instruction::Sub;
1094	LLVM_FALLTHROUGH;
1095	case AtomicExpr::AO__c11_atomic_fetch_sub:
1096	case AtomicExpr::AO__opencl_atomic_fetch_sub:
1097	case AtomicExpr::AO__atomic_fetch_sub:
1098	LibCallName = "__atomic_fetch_sub";
1099	AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1100	LoweredMemTy, E->getExprLoc(), sizeChars);
1101	break;
1102	// T __atomic_xor_fetch_N(T *mem, T val, int order)
1103	// T __atomic_fetch_xor_N(T *mem, T val, int order)
1104	case AtomicExpr::AO__atomic_xor_fetch:
1105	PostOp = llvm::Instruction::Xor;
1106	LLVM_FALLTHROUGH;
1107	case AtomicExpr::AO__c11_atomic_fetch_xor:
1108	case AtomicExpr::AO__opencl_atomic_fetch_xor:
1109	case AtomicExpr::AO__atomic_fetch_xor:
1110	LibCallName = "__atomic_fetch_xor";
1111	AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1112	MemTy, E->getExprLoc(), sizeChars);
1113	break;
1114	case AtomicExpr::AO__atomic_fetch_min:
1115	case AtomicExpr::AO__opencl_atomic_fetch_min:
1116	LibCallName = E->getValueType()->isSignedIntegerType()
1117	? "__atomic_fetch_min"
1118	: "__atomic_fetch_umin";
1119	AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1120	LoweredMemTy, E->getExprLoc(), sizeChars);
1121	break;
1122	case AtomicExpr::AO__atomic_fetch_max:
1123	case AtomicExpr::AO__opencl_atomic_fetch_max:
1124	LibCallName = E->getValueType()->isSignedIntegerType()
1125	? "__atomic_fetch_max"
1126	: "__atomic_fetch_umax";
1127	AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1128	LoweredMemTy, E->getExprLoc(), sizeChars);
1129	break;
1130	// T __atomic_nand_fetch_N(T *mem, T val, int order)
1131	// T __atomic_fetch_nand_N(T *mem, T val, int order)
1132	case AtomicExpr::AO__atomic_nand_fetch:
1133	PostOp = llvm::Instruction::And; // the NOT is special cased below
1134	LLVM_FALLTHROUGH;
1135	case AtomicExpr::AO__atomic_fetch_nand:
1136	LibCallName = "__atomic_fetch_nand";
1137	AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1138	MemTy, E->getExprLoc(), sizeChars);
1139	break;
1140	}
1141
1142	if (E->isOpenCL()) {
1143	LibCallName = std::string("__opencl") +
1144	StringRef(LibCallName).drop_front(1).str();
1145
1146	}
1147	// Optimized functions have the size in their name.
1148	if (UseOptimizedLibcall)
1149	LibCallName += "_" + llvm::utostr(Size);
1150	// By default, assume we return a value of the atomic type.
1151	if (!HaveRetTy) {
1152	if (UseOptimizedLibcall) {
1153	// Value is returned directly.
1154	// The function returns an appropriately sized integer type.
1155	RetTy = getContext().getIntTypeForBitwidth(
1156	getContext().toBits(sizeChars), /Signed=/false);
1157	} else {
1158	// Value is returned through parameter before the order.
1159	RetTy = getContext().VoidTy;
1160	Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1161	getContext().VoidPtrTy);
1162	}
1163	}
1164	// order is always the last parameter
1165	Args.add(RValue::get(Order),
1166	getContext().IntTy);
1167	if (E->isOpenCL())
1168	Args.add(RValue::get(Scope), getContext().IntTy);
1169
1170	// PostOp is only needed for the atomic_*_fetch operations, and
1171	// thus is only needed for and implemented in the
1172	// UseOptimizedLibcall codepath.
1173	assert(UseOptimizedLibcall \|\| !PostOp);
1174
1175	RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1176	// The value is returned directly from the libcall.
1177	if (E->isCmpXChg())
1178	return Res;
1179
1180	// The value is returned directly for optimized libcalls but the expr
1181	// provided an out-param.
1182	if (UseOptimizedLibcall && Res.getScalarVal()) {
1183	llvm::Value *ResVal = Res.getScalarVal();
1184	if (PostOp) {
1185	llvm::Value LoadVal1 = Args[1].getRValue(this).getScalarVal();
1186	ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1187	}
1188	if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1189	ResVal = Builder.CreateNot(ResVal);
1190
1191	Builder.CreateStore(
1192	ResVal,
1193	Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1194	}
1195
1196	if (RValTy->isVoidType())
1197	return RValue::get(nullptr);
1198
1199	return convertTempToRValue(
1200	Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1201	RValTy, E->getExprLoc());
1202	}
1203
1204	bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store \|\|
1205	E->getOp() == AtomicExpr::AO__opencl_atomic_store \|\|
1206	E->getOp() == AtomicExpr::AO__atomic_store \|\|
1207	E->getOp() == AtomicExpr::AO__atomic_store_n;
1208	bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load \|\|
1209	E->getOp() == AtomicExpr::AO__opencl_atomic_load \|\|
1210	E->getOp() == AtomicExpr::AO__atomic_load \|\|
1211	E->getOp() == AtomicExpr::AO__atomic_load_n;
1212
1213	if (isa<llvm::ConstantInt>(Order)) {
1214	auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1215	// We should not ever get to a case where the ordering isn't a valid C ABI
1216	// value, but it's hard to enforce that in general.
1217	if (llvm::isValidAtomicOrderingCABI(ord))
1218	switch ((llvm::AtomicOrderingCABI)ord) {
1219	case llvm::AtomicOrderingCABI::relaxed:
1220	EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1221	llvm::AtomicOrdering::Monotonic, Scope);
1222	break;
1223	case llvm::AtomicOrderingCABI::consume:
1224	case llvm::AtomicOrderingCABI::acquire:
1225	if (IsStore)
1226	break; // Avoid crashing on code with undefined behavior
1227	EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1228	llvm::AtomicOrdering::Acquire, Scope);
1229	break;
1230	case llvm::AtomicOrderingCABI::release:
1231	if (IsLoad)
1232	break; // Avoid crashing on code with undefined behavior
1233	EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1234	llvm::AtomicOrdering::Release, Scope);
1235	break;
1236	case llvm::AtomicOrderingCABI::acq_rel:
1237	if (IsLoad \|\| IsStore)
1238	break; // Avoid crashing on code with undefined behavior
1239	EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1240	llvm::AtomicOrdering::AcquireRelease, Scope);
1241	break;
1242	case llvm::AtomicOrderingCABI::seq_cst:
1243	EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1244	llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1245	break;
1246	}
1247	if (RValTy->isVoidType())
1248	return RValue::get(nullptr);
1249
1250	return convertTempToRValue(
1251	Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1252	Dest.getAddressSpace())),
1253	RValTy, E->getExprLoc());
1254	}
1255
1256	// Long case, when Order isn't obviously constant.
1257
1258	// Create all the relevant BB's
1259	llvm::BasicBlock MonotonicBB = nullptr, AcquireBB = nullptr,
1260	ReleaseBB = nullptr, AcqRelBB = nullptr,
1261	*SeqCstBB = nullptr;
1262	MonotonicBB = createBasicBlock("monotonic", CurFn);
1263	if (!IsStore)
1264	AcquireBB = createBasicBlock("acquire", CurFn);
1265	if (!IsLoad)
1266	ReleaseBB = createBasicBlock("release", CurFn);
1267	if (!IsLoad && !IsStore)
1268	AcqRelBB = createBasicBlock("acqrel", CurFn);
1269	SeqCstBB = createBasicBlock("seqcst", CurFn);
1270	llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1271
1272	// Create the switch for the split
1273	// MonotonicBB is arbitrarily chosen as the default case; in practice, this
1274	// doesn't matter unless someone is crazy enough to use something that
1275	// doesn't fold to a constant for the ordering.
1276	Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1277	llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1278
1279	// Emit all the different atomics
1280	Builder.SetInsertPoint(MonotonicBB);
1281	EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1282	llvm::AtomicOrdering::Monotonic, Scope);
1283	Builder.CreateBr(ContBB);
1284	if (!IsStore) {
1285	Builder.SetInsertPoint(AcquireBB);
1286	EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1287	llvm::AtomicOrdering::Acquire, Scope);
1288	Builder.CreateBr(ContBB);
1289	SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1290	AcquireBB);
1291	SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1292	AcquireBB);
1293	}
1294	if (!IsLoad) {
1295	Builder.SetInsertPoint(ReleaseBB);
1296	EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1297	llvm::AtomicOrdering::Release, Scope);
1298	Builder.CreateBr(ContBB);
1299	SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1300	ReleaseBB);
1301	}
1302	if (!IsLoad && !IsStore) {
1303	Builder.SetInsertPoint(AcqRelBB);
1304	EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1305	llvm::AtomicOrdering::AcquireRelease, Scope);
1306	Builder.CreateBr(ContBB);
1307	SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1308	AcqRelBB);
1309	}
1310	Builder.SetInsertPoint(SeqCstBB);
1311	EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1312	llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1313	Builder.CreateBr(ContBB);
1314	SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1315	SeqCstBB);
1316
1317	// Cleanup and return
1318	Builder.SetInsertPoint(ContBB);
1319	if (RValTy->isVoidType())
1320	return RValue::get(nullptr);
1321
1322	assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1323	return convertTempToRValue(
1324	Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1325	Dest.getAddressSpace())),
1326	RValTy, E->getExprLoc());
1327	}
1328
1329	Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1330	unsigned addrspace =
1331	cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1332	llvm::IntegerType *ty =
1333	llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1334	return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1335	}
1336
1337	Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1338	llvm::Type *Ty = Addr.getElementType();
1339	uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1340	if (SourceSizeInBits != AtomicSizeInBits) {
1341	Address Tmp = CreateTempAlloca();
1342	CGF.Builder.CreateMemCpy(Tmp, Addr,
1343	std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1344	Addr = Tmp;
1345	}
1346
1347	return emitCastToAtomicIntPointer(Addr);
1348	}
1349
1350	RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1351	AggValueSlot resultSlot,
1352	SourceLocation loc,
1353	bool asValue) const {
1354	if (LVal.isSimple()) {
1355	if (EvaluationKind == TEK_Aggregate)
1356	return resultSlot.asRValue();
1357
1358	// Drill into the padding structure if we have one.
1359	if (hasPadding())
1360	addr = CGF.Builder.CreateStructGEP(addr, 0);
1361
1362	// Otherwise, just convert the temporary to an r-value using the
1363	// normal conversion routine.
1364	return CGF.convertTempToRValue(addr, getValueType(), loc);
1365	}
1366	if (!asValue)
1367	// Get RValue from temp memory as atomic for non-simple lvalues
1368	return RValue::get(CGF.Builder.CreateLoad(addr));
1369	if (LVal.isBitField())
1370	return CGF.EmitLoadOfBitfieldLValue(
1371	LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1372	LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1373	if (LVal.isVectorElt())
1374	return CGF.EmitLoadOfLValue(
1375	LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1376	LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1377	assert(LVal.isExtVectorElt());
1378	return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1379	addr, LVal.getExtVectorElts(), LVal.getType(),
1380	LVal.getBaseInfo(), TBAAAccessInfo()));
1381	}
1382
1383	RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1384	AggValueSlot ResultSlot,
1385	SourceLocation Loc,
1386	bool AsValue) const {
1387	// Try not to in some easy cases.
1388	(0) . __assert_fail ("IntVal->getType()->isIntegerTy() && \"Expected integer value\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGAtomic.cpp", 1388, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1389	if (getEvaluationKind() == TEK_Scalar &&
1390	(((!LVal.isBitField() \|\|
1391	LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1392	!hasPadding()) \|\|
1393	!AsValue)) {
1394	auto *ValTy = AsValue
1395	? CGF.ConvertTypeForMem(ValueTy)
1396	: getAtomicAddress().getType()->getPointerElementType();
1397	if (ValTy->isIntegerTy()) {
1398	(0) . __assert_fail ("IntVal->getType() == ValTy && \"Different integer types.\"", "/home/seafit/code_projects/clang_source/clang/lib/CodeGen/CGAtomic.cpp", 1398, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(IntVal->getType() == ValTy && "Different integer types.");
1399	return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1400	} else if (ValTy->isPointerTy())
1401	return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1402	else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1403	return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1404	}
1405
1406	// Create a temporary. This needs to be big enough to hold the
1407	// atomic integer.
1408	Address Temp = Address::invalid();
1409	bool TempIsVolatile = false;
1410	if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1411	assert(!ResultSlot.isIgnored());
1412	Temp = ResultSlot.getAddress();
1413	TempIsVolatile = ResultSlot.isVolatile();
1414	} else {
1415	Temp = CreateTempAlloca();
1416	}
1417
1418	// Slam the integer into the temporary.
1419	Address CastTemp = emitCastToAtomicIntPointer(Temp);
1420	CGF.Builder.CreateStore(IntVal, CastTemp)
1421	->setVolatile(TempIsVolatile);
1422
1423	return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1424	}
1425
1426	void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1427	llvm::AtomicOrdering AO, bool) {
1428	// void __atomic_load(size_t size, void mem, void return, int order);
1429	CallArgList Args;
1430	Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1431	Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1432	CGF.getContext().VoidPtrTy);
1433	Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1434	CGF.getContext().VoidPtrTy);
1435	Args.add(
1436	RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1437	CGF.getContext().IntTy);
1438	emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1439	}
1440
1441	llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1442	bool IsVolatile) {
1443	// Okay, we're doing this natively.
1444	Address Addr = getAtomicAddressAsAtomicIntPointer();
1445	llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1446	Load->setAtomic(AO);
1447
1448	// Other decoration.
1449	if (IsVolatile)
1450	Load->setVolatile(true);
1451	CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1452	return Load;
1453	}
1454
1455	/// An LValue is a candidate for having its loads and stores be made atomic if
1456	/// we are operating under /volatile:ms and the LValue itself is volatile and
1457	/// performing such an operation can be performed without a libcall.
1458	bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1459	if (!CGM.getCodeGenOpts().MSVolatile) return false;
1460	AtomicInfo AI(*this, LV);
1461	bool IsVolatile = LV.isVolatile() \|\| hasVolatileMember(LV.getType());
1462	// An atomic is inline if we don't need to use a libcall.
1463	bool AtomicIsInline = !AI.shouldUseLibcall();
1464	// MSVC doesn't seem to do this for types wider than a pointer.
1465	if (getContext().getTypeSize(LV.getType()) >
1466	getContext().getTypeSize(getContext().getIntPtrType()))
1467	return false;
1468	return IsVolatile && AtomicIsInline;
1469	}
1470
1471	RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1472	AggValueSlot Slot) {
1473	llvm::AtomicOrdering AO;
1474	bool IsVolatile = LV.isVolatileQualified();
1475	if (LV.getType()->isAtomicType()) {
1476	AO = llvm::AtomicOrdering::SequentiallyConsistent;
1477	} else {
1478	AO = llvm::AtomicOrdering::Acquire;
1479	IsVolatile = true;
1480	}
1481	return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1482	}
1483
1484	RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1485	bool AsValue, llvm::AtomicOrdering AO,
1486	bool IsVolatile) {
1487	// Check whether we should use a library call.
1488	if (shouldUseLibcall()) {
1489	Address TempAddr = Address::invalid();
1490	if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1491	assert(getEvaluationKind() == TEK_Aggregate);
1492	TempAddr = ResultSlot.getAddress();
1493	} else
1494	TempAddr = CreateTempAlloca();
1495
1496	EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1497
1498	// Okay, turn that back into the original value or whole atomic (for
1499	// non-simple lvalues) type.
1500	return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1501	}
1502
1503	// Okay, we're doing this natively.
1504	auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1505
1506	// If we're ignoring an aggregate return, don't do anything.
1507	if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1508	return RValue::getAggregate(Address::invalid(), false);
1509
1510	// Okay, turn that back into the original value or atomic (for non-simple
1511	// lvalues) type.
1512	return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1513	}
1514
1515	/// Emit a load from an l-value of atomic type. Note that the r-value
1516	/// we produce is an r-value of the atomic value type.
1517	RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1518	llvm::AtomicOrdering AO, bool IsVolatile,
1519	AggValueSlot resultSlot) {
1520	AtomicInfo Atomics(*this, src);
1521	return Atomics.EmitAtomicLoad(resultSlot, loc, /AsValue=/true, AO,
1522	IsVolatile);
1523	}
1524
1525	/// Copy an r-value into memory as part of storing to an atomic type.
1526	/// This needs to create a bit-pattern suitable for atomic operations.
1527	void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1528	assert(LVal.isSimple());
1529	// If we have an r-value, the rvalue should be of the atomic type,
1530	// which means that the caller is responsible for having zeroed
1531	// any padding. Just do an aggregate copy of that type.
1532	if (rvalue.isAggregate()) {
1533	LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1534	LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1535	getAtomicType());
1536	bool IsVolatile = rvalue.isVolatileQualified() \|\|
1537	LVal.isVolatileQualified();
1538	CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1539	AggValueSlot::DoesNotOverlap, IsVolatile);
1540	return;
1541	}
1542
1543	// Okay, otherwise we're copying stuff.
1544
1545	// Zero out the buffer if necessary.
1546	emitMemSetZeroIfNecessary();
1547
1548	// Drill past the padding if present.
1549	LValue TempLVal = projectValue();
1550
1551	// Okay, store the rvalue in.
1552	if (rvalue.isScalar()) {
1553	CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /init/ true);
1554	} else {
1555	CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /init/ true);
1556	}
1557	}
1558
1559
1560	/// Materialize an r-value into memory for the purposes of storing it
1561	/// to an atomic type.
1562	Address AtomicInfo::materializeRValue(RValue rvalue) const {
1563	// Aggregate r-values are already in memory, and EmitAtomicStore
1564	// requires them to be values of the atomic type.
1565	if (rvalue.isAggregate())
1566	return rvalue.getAggregateAddress();
1567
1568	// Otherwise, make a temporary and materialize into it.
1569	LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1570	AtomicInfo Atomics(CGF, TempLV);
1571	Atomics.emitCopyIntoMemory(rvalue);
1572	return TempLV.getAddress();
1573	}
1574
1575	llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1576	// If we've got a scalar value of the right size, try to avoid going
1577	// through memory.
1578	if (RVal.isScalar() && (!hasPadding() \|\| !LVal.isSimple())) {
1579	llvm::Value *Value = RVal.getScalarVal();
1580	if (isa<llvm::IntegerType>(Value->getType()))
1581	return CGF.EmitToMemory(Value, ValueTy);
1582	else {
1583	llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1584	CGF.getLLVMContext(),
1585	LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1586	if (isa<llvm::PointerType>(Value->getType()))
1587	return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1588	else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1589	return CGF.Builder.CreateBitCast(Value, InputIntTy);
1590	}
1591	}
1592	// Otherwise, we need to go through memory.
1593	// Put the r-value in memory.
1594	Address Addr = materializeRValue(RVal);
1595
1596	// Cast the temporary to the atomic int type and pull a value out.
1597	Addr = emitCastToAtomicIntPointer(Addr);
1598	return CGF.Builder.CreateLoad(Addr);
1599	}
1600
1601	std::pair<llvm::Value , llvm::Value > AtomicInfo::EmitAtomicCompareExchangeOp(
1602	llvm::Value ExpectedVal, llvm::Value DesiredVal,
1603	llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1604	// Do the atomic store.
1605	Address Addr = getAtomicAddressAsAtomicIntPointer();
1606	auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1607	ExpectedVal, DesiredVal,
1608	Success, Failure);
1609	// Other decoration.
1610	Inst->setVolatile(LVal.isVolatileQualified());
1611	Inst->setWeak(IsWeak);
1612
1613	// Okay, turn that back into the original value type.
1614	auto PreviousVal = CGF.Builder.CreateExtractValue(Inst, /Idxs=*/0);
1615	auto SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /Idxs=*/1);
1616	return std::make_pair(PreviousVal, SuccessFailureVal);
1617	}
1618
1619	llvm::Value *
1620	AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1621	llvm::Value *DesiredAddr,
1622	llvm::AtomicOrdering Success,
1623	llvm::AtomicOrdering Failure) {
1624	// bool __atomic_compare_exchange(size_t size, void obj, void expected,
1625	// void *desired, int success, int failure);
1626	CallArgList Args;
1627	Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1628	Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1629	CGF.getContext().VoidPtrTy);
1630	Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1631	CGF.getContext().VoidPtrTy);
1632	Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1633	CGF.getContext().VoidPtrTy);
1634	Args.add(RValue::get(
1635	llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1636	CGF.getContext().IntTy);
1637	Args.add(RValue::get(
1638	llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1639	CGF.getContext().IntTy);
1640	auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1641	CGF.getContext().BoolTy, Args);
1642
1643	return SuccessFailureRVal.getScalarVal();
1644	}
1645
1646	std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1647	RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1648	llvm::AtomicOrdering Failure, bool IsWeak) {
1649	if (isStrongerThan(Failure, Success))
1650	// Don't assert on undefined behavior "failure argument shall be no stronger
1651	// than the success argument".
1652	Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1653
1654	// Check whether we should use a library call.
1655	if (shouldUseLibcall()) {
1656	// Produce a source address.
1657	Address ExpectedAddr = materializeRValue(Expected);
1658	Address DesiredAddr = materializeRValue(Desired);
1659	auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1660	DesiredAddr.getPointer(),
1661	Success, Failure);
1662	return std::make_pair(
1663	convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1664	SourceLocation(), /AsValue=/false),
1665	Res);
1666	}
1667
1668	// If we've got a scalar value of the right size, try to avoid going
1669	// through memory.
1670	auto *ExpectedVal = convertRValueToInt(Expected);
1671	auto *DesiredVal = convertRValueToInt(Desired);
1672	auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1673	Failure, IsWeak);
1674	return std::make_pair(
1675	ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1676	SourceLocation(), /AsValue=/false),
1677	Res.second);
1678	}
1679
1680	static void
1681	EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1682	const llvm::function_ref<RValue(RValue)> &UpdateOp,
1683	Address DesiredAddr) {
1684	RValue UpRVal;
1685	LValue AtomicLVal = Atomics.getAtomicLValue();
1686	LValue DesiredLVal;
1687	if (AtomicLVal.isSimple()) {
1688	UpRVal = OldRVal;
1689	DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1690	} else {
1691	// Build new lvalue for temp address
1692	Address Ptr = Atomics.materializeRValue(OldRVal);
1693	LValue UpdateLVal;
1694	if (AtomicLVal.isBitField()) {
1695	UpdateLVal =
1696	LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1697	AtomicLVal.getType(),
1698	AtomicLVal.getBaseInfo(),
1699	AtomicLVal.getTBAAInfo());
1700	DesiredLVal =
1701	LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1702	AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1703	AtomicLVal.getTBAAInfo());
1704	} else if (AtomicLVal.isVectorElt()) {
1705	UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1706	AtomicLVal.getType(),
1707	AtomicLVal.getBaseInfo(),
1708	AtomicLVal.getTBAAInfo());
1709	DesiredLVal = LValue::MakeVectorElt(
1710	DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1711	AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1712	} else {
1713	assert(AtomicLVal.isExtVectorElt());
1714	UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1715	AtomicLVal.getType(),
1716	AtomicLVal.getBaseInfo(),
1717	AtomicLVal.getTBAAInfo());
1718	DesiredLVal = LValue::MakeExtVectorElt(
1719	DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1720	AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1721	}
1722	UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1723	}
1724	// Store new value in the corresponding memory area
1725	RValue NewRVal = UpdateOp(UpRVal);
1726	if (NewRVal.isScalar()) {
1727	CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1728	} else {
1729	assert(NewRVal.isComplex());
1730	CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1731	/isInit=/false);
1732	}
1733	}
1734
1735	void AtomicInfo::EmitAtomicUpdateLibcall(
1736	llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1737	bool IsVolatile) {
1738	auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1739
1740	Address ExpectedAddr = CreateTempAlloca();
1741
1742	EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1743	auto *ContBB = CGF.createBasicBlock("atomic_cont");
1744	auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1745	CGF.EmitBlock(ContBB);
1746	Address DesiredAddr = CreateTempAlloca();
1747	if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) \|\|
1748	requiresMemSetZero(getAtomicAddress().getElementType())) {
1749	auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1750	CGF.Builder.CreateStore(OldVal, DesiredAddr);
1751	}
1752	auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1753	AggValueSlot::ignored(),
1754	SourceLocation(), /AsValue=/false);
1755	EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1756	auto *Res =
1757	EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1758	DesiredAddr.getPointer(),
1759	AO, Failure);
1760	CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1761	CGF.EmitBlock(ExitBB, /IsFinished=/true);
1762	}
1763
1764	void AtomicInfo::EmitAtomicUpdateOp(
1765	llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1766	bool IsVolatile) {
1767	auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1768
1769	// Do the atomic load.
1770	auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1771	// For non-simple lvalues perform compare-and-swap procedure.
1772	auto *ContBB = CGF.createBasicBlock("atomic_cont");
1773	auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1774	auto *CurBB = CGF.Builder.GetInsertBlock();
1775	CGF.EmitBlock(ContBB);
1776	llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1777	/NumReservedValues=/2);
1778	PHI->addIncoming(OldVal, CurBB);
1779	Address NewAtomicAddr = CreateTempAlloca();
1780	Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1781	if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) \|\|
1782	requiresMemSetZero(getAtomicAddress().getElementType())) {
1783	CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1784	}
1785	auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1786	SourceLocation(), /AsValue=/false);
1787	EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1788	auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1789	// Try to write new value using cmpxchg operation
1790	auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1791	PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1792	CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1793	CGF.EmitBlock(ExitBB, /IsFinished=/true);
1794	}
1795
1796	static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1797	RValue UpdateRVal, Address DesiredAddr) {
1798	LValue AtomicLVal = Atomics.getAtomicLValue();
1799	LValue DesiredLVal;
1800	// Build new lvalue for temp address
1801	if (AtomicLVal.isBitField()) {
1802	DesiredLVal =
1803	LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1804	AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1805	AtomicLVal.getTBAAInfo());
1806	} else if (AtomicLVal.isVectorElt()) {
1807	DesiredLVal =
1808	LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1809	AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1810	AtomicLVal.getTBAAInfo());
1811	} else {
1812	assert(AtomicLVal.isExtVectorElt());
1813	DesiredLVal = LValue::MakeExtVectorElt(
1814	DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1815	AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1816	}
1817	// Store new value in the corresponding memory area
1818	assert(UpdateRVal.isScalar());
1819	CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1820	}
1821
1822	void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1823	RValue UpdateRVal, bool IsVolatile) {
1824	auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1825
1826	Address ExpectedAddr = CreateTempAlloca();
1827
1828	EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1829	auto *ContBB = CGF.createBasicBlock("atomic_cont");
1830	auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1831	CGF.EmitBlock(ContBB);
1832	Address DesiredAddr = CreateTempAlloca();
1833	if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) \|\|
1834	requiresMemSetZero(getAtomicAddress().getElementType())) {
1835	auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1836	CGF.Builder.CreateStore(OldVal, DesiredAddr);
1837	}
1838	EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1839	auto *Res =
1840	EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1841	DesiredAddr.getPointer(),
1842	AO, Failure);
1843	CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1844	CGF.EmitBlock(ExitBB, /IsFinished=/true);
1845	}
1846
1847	void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1848	bool IsVolatile) {
1849	auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1850
1851	// Do the atomic load.
1852	auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1853	// For non-simple lvalues perform compare-and-swap procedure.
1854	auto *ContBB = CGF.createBasicBlock("atomic_cont");
1855	auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1856	auto *CurBB = CGF.Builder.GetInsertBlock();
1857	CGF.EmitBlock(ContBB);
1858	llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1859	/NumReservedValues=/2);
1860	PHI->addIncoming(OldVal, CurBB);
1861	Address NewAtomicAddr = CreateTempAlloca();
1862	Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1863	if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) \|\|
1864	requiresMemSetZero(getAtomicAddress().getElementType())) {
1865	CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1866	}
1867	EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1868	auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1869	// Try to write new value using cmpxchg operation
1870	auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1871	PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1872	CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1873	CGF.EmitBlock(ExitBB, /IsFinished=/true);
1874	}
1875
1876	void AtomicInfo::EmitAtomicUpdate(
1877	llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1878	bool IsVolatile) {
1879	if (shouldUseLibcall()) {
1880	EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1881	} else {
1882	EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1883	}
1884	}
1885
1886	void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1887	bool IsVolatile) {
1888	if (shouldUseLibcall()) {
1889	EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1890	} else {
1891	EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1892	}
1893	}
1894
1895	void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1896	bool isInit) {
1897	bool IsVolatile = lvalue.isVolatileQualified();
1898	llvm::AtomicOrdering AO;
1899	if (lvalue.getType()->isAtomicType()) {
1900	AO = llvm::AtomicOrdering::SequentiallyConsistent;
1901	} else {
1902	AO = llvm::AtomicOrdering::Release;
1903	IsVolatile = true;
1904	}
1905	return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1906	}
1907
1908	/// Emit a store to an l-value of atomic type.
1909	///
1910	/// Note that the r-value is expected to be an r-value *of the atomic
1911	/// type*; this means that for aggregate r-values, it should include
1912	/// storage for any padding that was necessary.
1913	void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1914	llvm::AtomicOrdering AO, bool IsVolatile,
1915	bool isInit) {
1916	// If this is an aggregate r-value, it should agree in type except
1917	// maybe for address-space qualification.
1918	assert(!rvalue.isAggregate() \|\|
1919	rvalue.getAggregateAddress().getElementType()
1920	== dest.getAddress().getElementType());
1921
1922	AtomicInfo atomics(*this, dest);
1923	LValue LVal = atomics.getAtomicLValue();
1924
1925	// If this is an initialization, just put the value there normally.
1926	if (LVal.isSimple()) {
1927	if (isInit) {
1928	atomics.emitCopyIntoMemory(rvalue);
1929	return;
1930	}
1931
1932	// Check whether we should use a library call.
1933	if (atomics.shouldUseLibcall()) {
1934	// Produce a source address.
1935	Address srcAddr = atomics.materializeRValue(rvalue);
1936
1937	// void __atomic_store(size_t size, void mem, void val, int order)
1938	CallArgList args;
1939	args.add(RValue::get(atomics.getAtomicSizeValue()),
1940	getContext().getSizeType());
1941	args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
1942	getContext().VoidPtrTy);
1943	args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
1944	getContext().VoidPtrTy);
1945	args.add(
1946	RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
1947	getContext().IntTy);
1948	emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1949	return;
1950	}
1951
1952	// Okay, we're doing this natively.
1953	llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1954
1955	// Do the atomic store.
1956	Address addr =
1957	atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1958	intValue = Builder.CreateIntCast(
1959	intValue, addr.getElementType(), /isSigned=/false);
1960	llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1961
1962	// Initializations don't need to be atomic.
1963	if (!isInit)
1964	store->setAtomic(AO);
1965
1966	// Other decoration.
1967	if (IsVolatile)
1968	store->setVolatile(true);
1969	CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
1970	return;
1971	}
1972
1973	// Emit simple atomic update operation.
1974	atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1975	}
1976
1977	/// Emit a compare-and-exchange op for atomic type.
1978	///
1979	std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1980	LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1981	llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1982	AggValueSlot Slot) {
1983	// If this is an aggregate r-value, it should agree in type except
1984	// maybe for address-space qualification.
1985	assert(!Expected.isAggregate() \|\|
1986	Expected.getAggregateAddress().getElementType() ==
1987	Obj.getAddress().getElementType());
1988	assert(!Desired.isAggregate() \|\|
1989	Desired.getAggregateAddress().getElementType() ==
1990	Obj.getAddress().getElementType());
1991	AtomicInfo Atomics(*this, Obj);
1992
1993	return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1994	IsWeak);
1995	}
1996
1997	void CodeGenFunction::EmitAtomicUpdate(
1998	LValue LVal, llvm::AtomicOrdering AO,
1999	const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2000	AtomicInfo Atomics(*this, LVal);
2001	Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2002	}
2003
2004	void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2005	AtomicInfo atomics(*this, dest);
2006
2007	switch (atomics.getEvaluationKind()) {
2008	case TEK_Scalar: {
2009	llvm::Value *value = EmitScalarExpr(init);
2010	atomics.emitCopyIntoMemory(RValue::get(value));
2011	return;
2012	}
2013
2014	case TEK_Complex: {
2015	ComplexPairTy value = EmitComplexExpr(init);
2016	atomics.emitCopyIntoMemory(RValue::getComplex(value));
2017	return;
2018	}
2019
2020	case TEK_Aggregate: {
2021	// Fix up the destination if the initializer isn't an expression
2022	// of atomic type.
2023	bool Zeroed = false;
2024	if (!init->getType()->isAtomicType()) {
2025	Zeroed = atomics.emitMemSetZeroIfNecessary();
2026	dest = atomics.projectValue();
2027	}
2028
2029	// Evaluate the expression directly into the destination.
2030	AggValueSlot slot = AggValueSlot::forLValue(dest,
2031	AggValueSlot::IsNotDestructed,
2032	AggValueSlot::DoesNotNeedGCBarriers,
2033	AggValueSlot::IsNotAliased,
2034	AggValueSlot::DoesNotOverlap,
2035	Zeroed ? AggValueSlot::IsZeroed :
2036	AggValueSlot::IsNotZeroed);
2037
2038	EmitAggExpr(init, slot);
2039	return;
2040	}
2041	}
2042	llvm_unreachable("bad evaluation kind");
2043	}
2044

clang::CodeGen::CodeGenFunction::EmitAtomicExpr

clang::CodeGen::CodeGenFunction::LValueIsSuitableForInlineAtomic

clang::CodeGen::CodeGenFunction::EmitAtomicLoad

clang::CodeGen::CodeGenFunction::EmitAtomicStore

clang::CodeGen::CodeGenFunction::EmitAtomicCompareExchange

clang::CodeGen::CodeGenFunction::EmitAtomicUpdate

clang::CodeGen::CodeGenFunction::EmitAtomicInit

Clang Project