Cold-Air-Networks/mirror-llvm-project
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 7

[WebAssembly] MC support for acquire-release atomics (#183656)

Browse source 
Initial support for acquire-release atomics, specified as part of
https://github.com/WebAssembly/shared-everything-threads

This adds an ordering operand to atomic loads, stores, RMWs,
wait/notify,
and fences. It currently defaults to 0 and ISel is not updated yet, so
atomics produced by the compiler will still always be seqcst.

Asm parsing and printing, binary emission and disassembly are all
updated. Binary emission will always use the old encoding because the
encoding is smaller, and to get backwards compatibility for free.
This commit is contained in:
Derek Schuff 2026-03-04 12:10:14 -08:00 committed by GitHub
parent bb78a0a470
commit ade43a54d4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 449 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
llvm/include/llvm/BinaryFormat/Wasm.h
View file

@ -81,6 +81,15 @@ enum : unsigned {
WASM_TYPE_NORESULT = 0x40, // for blocks with no result values
};
// Memory ordering encodings for atomic instructions.
enum : unsigned {
WASM_MEM_ORDER_SEQ_CST = 0x00,
WASM_MEM_ORDER_ACQ_REL = 0x01,
// RMW/CMPXCHG operations have 2 orderings but they must currently match.
WASM_MEM_ORDER_RMW_ACQ_REL = 0x11,
};
const unsigned WASM_MEMARG_HAS_MEM_ORDER = 0x20;
// Kinds of externals (for imports and exports).
enum : unsigned {
WASM_EXTERNAL_FUNCTION = 0x0,

44
llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
View file

@ -472,6 +472,29 @@ public:
return false;
}
bool addMemOrderOrDefault(OperandVector &Operands) {
auto &Tok = Lexer.getTok();
int64_t Order = wasm::WASM_MEM_ORDER_SEQ_CST;
if (Tok.is(AsmToken::Identifier)) {
StringRef S = Tok.getString();
Order = StringSwitch<int64_t>(S)
.Case("acqrel", wasm::WASM_MEM_ORDER_ACQ_REL)
.Case("seqcst", wasm::WASM_MEM_ORDER_SEQ_CST)
.Default(-1);
if (Order != -1) {
if (!STI->checkFeatures("+relaxed-atomics"))
return error("memory ordering requires relaxed-atomics feature: ",
Tok);
Parser.Lex();
} else {
Order = wasm::WASM_MEM_ORDER_SEQ_CST;
}
}
Operands.push_back(std::make_unique<WebAssemblyOperand>(
Tok.getLoc(), Tok.getEndLoc(), WebAssemblyOperand::IntOp{Order}));
return false;
}
bool checkForP2AlignIfLoadStore(OperandVector &Operands, StringRef InstName) {
// FIXME: there is probably a cleaner way to do this.
auto IsLoadStore = InstName.contains(".load") ||
@ -674,6 +697,11 @@ public:
ExpectFuncType = true;
}
if (Name.contains("atomic.")) {
if (addMemOrderOrDefault(Operands))
return true;
}
// Returns true if the next tokens are a catch clause
auto PeekCatchList = [&]() {
if (Lexer.isNot(AsmToken::LParen))
@ -1167,11 +1195,21 @@ public:
case Match_Success: {
ensureLocals(Out);
// Fix unknown p2align operands.
const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
auto Align = WebAssembly::GetDefaultP2AlignAny(Inst.getOpcode());
if (Align != -1U) {
auto &Op0 = Inst.getOperand(0);
if (Op0.getImm() == -1)
Op0.setImm(Align);
unsigned I = 0;
// It's operand 0 for regular memory ops and 1 for atomics.
for (unsigned E = Desc.getNumOperands(); I < E; ++I) {
if (Desc.operands()[I].OperandType == WebAssembly::OPERAND_P2ALIGN) {
auto &Op = Inst.getOperand(I);
if (Op.getImm() == -1) {
Op.setImm(Align);
}
break;
}
}
assert(I < 2 && "Default p2align set but operand not found");
}
if (Is64) {
// Upgrade 32-bit loads/stores to 64-bit. These mostly differ by having

50
llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
View file

@ -210,6 +210,34 @@ MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction(
case MCOI::OPERAND_IMMEDIATE: {
if (!parseLEBImmediate(MI, Size, Bytes, false))
return MCDisassembler::Fail;
if (OT == WebAssembly::OPERAND_P2ALIGN) {
// The byte that encodes P2align also encodes whether a memory index
// and/or ordering is present.
int64_t Val = MI.getOperand(MI.getNumOperands() - 1).getImm();
if (Val & wasm::WASM_MEMARG_HAS_MEM_ORDER) {
// Clear the order so we just have the alignment in this operand.
MI.getOperand(MI.getNumOperands() - 1)
.setImm(Val & ~wasm::WASM_MEMARG_HAS_MEM_ORDER);
if (Size >= Bytes.size())
return MCDisassembler::Fail;
uint8_t Order = Bytes[Size++];
// If we have a memory ordering, it must have been preceded by a
// MEMORDER operand which was added as a placeholder (because we are
// iterating in MI operand order which has mem order first, but this
// byte is encoded first).
assert(OPI > 0 &&
OperandTable[WasmInst->OperandStart + OPI - 1] ==
WebAssembly::OPERAND_MEMORDER &&
"P2ALIGN with memory order not preceded by MEMORDER");
if (Order == wasm::WASM_MEM_ORDER_RMW_ACQ_REL ||
Order == wasm::WASM_MEM_ORDER_ACQ_REL)
MI.getOperand(MI.getNumOperands() - 2)
.setImm(wasm::WASM_MEM_ORDER_ACQ_REL);
else
MI.getOperand(MI.getNumOperands() - 2)
.setImm(wasm::WASM_MEM_ORDER_SEQ_CST);
}
}
break;
}
// SLEB operands:
@ -261,6 +289,28 @@ MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction(
return MCDisassembler::Fail;
break;
}
case WebAssembly::OPERAND_MEMORDER: {
uint8_t Val;
if (OPI + 1 < WasmInst->NumOperands &&
OperandTable[WasmInst->OperandStart + OPI + 1] ==
WebAssembly::OPERAND_P2ALIGN) {
// If we have P2ALIGN next, it will be encoded as part of the memarg,
// which has not been parsed yet. Default to SEQ_CST
// and we will update it when we parse P2ALIGN if necessary.
Val = wasm::WASM_MEM_ORDER_SEQ_CST;
} else {
// atomic.fence instructions have no p2align operand.
if (Size >= Bytes.size())
return MCDisassembler::Fail;
Val = Bytes[Size++];
}
if (Val == wasm::WASM_MEM_ORDER_RMW_ACQ_REL ||
Val == wasm::WASM_MEM_ORDER_ACQ_REL)
MI.addOperand(MCOperand::createImm(wasm::WASM_MEM_ORDER_ACQ_REL));
else
MI.addOperand(MCOperand::createImm(wasm::WASM_MEM_ORDER_SEQ_CST));
break;
}
case WebAssembly::OPERAND_VEC_I16IMM: {
if (!parseImmediate<uint16_t>(MI, Size, Bytes))
return MCDisassembler::Fail;

42
llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
View file

@ -69,19 +69,19 @@ void WebAssemblyInstPrinter::printInst(const MCInst *MI, uint64_t Address,
OS << getMnemonic(*MI).first;
OS << " ";
if (MI->getOperand(TableOperand).isExpr()) {
printOperand(MI, TableOperand, OS);
printOperand(MI, TableOperand, STI, OS);
OS << ", ";
} else {
assert(MI->getOperand(TableOperand).getImm() == 0);
}
printOperand(MI, TypeOperand, OS);
printOperand(MI, TypeOperand, STI, OS);
if (MI->getOpcode() == WebAssembly::CALL_INDIRECT)
OS << ", ";
break;
}
default:
// Print the instruction (this uses the AsmStrings from the .td files).
printInstruction(MI, Address, OS);
printInstruction(MI, Address, STI, OS);
break;
}
@ -108,7 +108,7 @@ void WebAssemblyInstPrinter::printInst(const MCInst *MI, uint64_t Address,
}
if (NeedsComma)
OS << ", ";
printOperand(MI, I, OS, I - Start < NumVariadicDefs);
printOperand(MI, I, STI, OS, I - Start < NumVariadicDefs);
NeedsComma = true;
}
}
@ -320,6 +320,7 @@ static std::string toString(const APFloat &FP) {
}
void WebAssemblyInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
const MCSubtargetInfo &STI,
raw_ostream &O, bool IsVariadicDef) {
const MCOperand &Op = MI->getOperand(OpNo);
if (Op.isReg()) {
@ -358,6 +359,7 @@ void WebAssemblyInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
}
void WebAssemblyInstPrinter::printBrList(const MCInst *MI, unsigned OpNo,
const MCSubtargetInfo &STI,
raw_ostream &O) {
O << "{";
for (unsigned I = OpNo, E = MI->getNumOperands(); I != E; ++I) {
@ -368,18 +370,37 @@ void WebAssemblyInstPrinter::printBrList(const MCInst *MI, unsigned OpNo,
O << "}";
}
void WebAssemblyInstPrinter::printWebAssemblyP2AlignOperand(const MCInst *MI,
unsigned OpNo,
raw_ostream &O) {
void WebAssemblyInstPrinter::printWebAssemblyP2AlignOperand(
const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
raw_ostream &O) {
int64_t Imm = MI->getOperand(OpNo).getImm();
if (Imm == WebAssembly::GetDefaultP2Align(MI->getOpcode()))
return;
O << ":p2align=" << Imm;
}
void WebAssemblyInstPrinter::printWebAssemblySignatureOperand(const MCInst *MI,
unsigned OpNo,
raw_ostream &O) {
void WebAssemblyInstPrinter::printWebAssemblyMemOrderOperand(
const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
raw_ostream &O) {
int64_t Imm = MI->getOperand(OpNo).getImm();
switch (Imm) {
case wasm::WASM_MEM_ORDER_RMW_ACQ_REL:
case wasm::WASM_MEM_ORDER_ACQ_REL:
O << "acqrel";
break;
case wasm::WASM_MEM_ORDER_SEQ_CST:
if (STI.getFeatureBits()[WebAssembly::FeatureRelaxedAtomics])
O << "seqcst";
break;
default:
llvm_unreachable("Unknown memory ordering");
}
}
void WebAssemblyInstPrinter::printWebAssemblySignatureOperand(
const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
raw_ostream &O) {
const MCOperand &Op = MI->getOperand(OpNo);
if (Op.isImm()) {
auto Imm = static_cast<unsigned>(Op.getImm());
@ -398,6 +419,7 @@ void WebAssemblyInstPrinter::printWebAssemblySignatureOperand(const MCInst *MI,
}
void WebAssemblyInstPrinter::printCatchList(const MCInst *MI, unsigned OpNo,
const MCSubtargetInfo &STI,
raw_ostream &O) {
unsigned OpIdx = OpNo;
const MCOperand &Op = MI->getOperand(OpIdx++);

18
llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h
View file

@ -40,19 +40,27 @@ public:
const MCSubtargetInfo &STI, raw_ostream &OS) override;
// Used by tblegen code.
void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O,
bool IsVariadicDef = false);
void printBrList(const MCInst *MI, unsigned OpNo, raw_ostream &O);
void printOperand(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
raw_ostream &O, bool IsVariadicDef = false);
void printBrList(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
raw_ostream &O);
void printWebAssemblyP2AlignOperand(const MCInst *MI, unsigned OpNo,
const MCSubtargetInfo &STI,
raw_ostream &O);
void printWebAssemblyMemOrderOperand(const MCInst *MI, unsigned OpNo,
const MCSubtargetInfo &STI,
raw_ostream &O);
void printWebAssemblySignatureOperand(const MCInst *MI, unsigned OpNo,
const MCSubtargetInfo &STI,
raw_ostream &O);
void printCatchList(const MCInst *MI, unsigned OpNo, raw_ostream &O);
void printCatchList(const MCInst *MI, unsigned OpNo,
const MCSubtargetInfo &STI, raw_ostream &O);
// Autogenerated by tblgen.
std::pair<const char *, uint64_t>
getMnemonic(const MCInst &MI) const override;
void printInstruction(const MCInst *MI, uint64_t Address, raw_ostream &O);
void printInstruction(const MCInst *MI, uint64_t Address,
const MCSubtargetInfo &STI, raw_ostream &O);
static const char *getRegisterName(MCRegister Reg);
};

68
llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
View file

@ -47,6 +47,14 @@ class WebAssemblyMCCodeEmitter final : public MCCodeEmitter {
SmallVectorImpl<MCFixup> &Fixups,
const MCSubtargetInfo &STI) const override;
void encodeP2AlignAndMemOrder(const MCInst &MI, unsigned P2AlignIdx,
const MCInstrDesc &Desc,
const MCSubtargetInfo &STI, raw_ostream &OS,
SmallVectorImpl<MCFixup> &Fixups,
uint64_t Start) const;
uint8_t getEncodedMemOrder(uint8_t Order, unsigned Opcode) const;
public:
WebAssemblyMCCodeEmitter(const MCInstrInfo &MCII, MCContext &Ctx)
: MCII(MCII), Ctx{Ctx} {}
@ -58,6 +66,47 @@ MCCodeEmitter *llvm::createWebAssemblyMCCodeEmitter(const MCInstrInfo &MCII,
return new WebAssemblyMCCodeEmitter(MCII, Ctx);
}
uint8_t WebAssemblyMCCodeEmitter::getEncodedMemOrder(uint8_t Order,
unsigned Opcode) const {
if (Order == wasm::WASM_MEM_ORDER_ACQ_REL) {
StringRef Name = MCII.getName(Opcode);
if (Name.contains("RMW") || Name.contains("CMPXCHG"))
return wasm::WASM_MEM_ORDER_RMW_ACQ_REL;
}
return Order;
}
void WebAssemblyMCCodeEmitter::encodeP2AlignAndMemOrder(
const MCInst &MI, unsigned P2AlignIdx, const MCInstrDesc &Desc,
const MCSubtargetInfo &STI, raw_ostream &OS,
SmallVectorImpl<MCFixup> &Fixups, uint64_t Start) const {
uint64_t P2Align = MI.getOperand(P2AlignIdx).getImm();
uint8_t Order = wasm::WASM_MEM_ORDER_SEQ_CST;
// Atomic instructions always have an ordering, but if it's SEQ_CST then we
// don't use the relaxed-atomics encoding (even if relaxed-atomics is
// enabled) because the original encoding is smaller.
if (P2AlignIdx > 0 && Desc.operands()[P2AlignIdx - 1].OperandType ==
WebAssembly::OPERAND_MEMORDER) {
Order = MI.getOperand(P2AlignIdx - 1).getImm();
if (Order != wasm::WASM_MEM_ORDER_SEQ_CST) {
assert(STI.getFeatureBits()[WebAssembly::FeatureRelaxedAtomics] &&
"Non-default atomic ordering but feature not enabled");
P2Align |= wasm::WASM_MEMARG_HAS_MEM_ORDER;
}
}
encodeULEB128(P2Align, OS);
// Memory index will go here once we support multi-memory.
if (P2Align & wasm::WASM_MEMARG_HAS_MEM_ORDER) {
support::endian::write<uint8_t>(OS,
getEncodedMemOrder(Order, MI.getOpcode()),
llvm::endianness::little);
}
}
void WebAssemblyMCCodeEmitter::encodeInstruction(
const MCInst &MI, SmallVectorImpl<char> &CB,
SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
@ -103,6 +152,9 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
case WebAssembly::OPERAND_I32IMM:
encodeSLEB128(int32_t(MO.getImm()), OS);
break;
case WebAssembly::OPERAND_P2ALIGN:
encodeP2AlignAndMemOrder(MI, I, Desc, STI, OS, Fixups, Start);
break;
case WebAssembly::OPERAND_OFFSET32:
encodeULEB128(uint32_t(MO.getImm()), OS);
break;
@ -114,6 +166,22 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
support::endian::write<uint8_t>(OS, MO.getImm(),
llvm::endianness::little);
break;
case WebAssembly::OPERAND_MEMORDER: {
// If there is a p2align operand (everything but fence) it is encoded
// together with the mem ordering (in the next iteration).
if (I + 1 < Desc.getNumOperands() &&
Desc.operands()[I + 1].OperandType ==
WebAssembly::OPERAND_P2ALIGN)
break;
uint8_t Val = getEncodedMemOrder(MO.getImm(), Opcode);
if (STI.getFeatureBits()[WebAssembly::FeatureRelaxedAtomics]) {
support::endian::write<uint8_t>(OS, Val, llvm::endianness::little);
} else {
assert(Opcode == WebAssembly::ATOMIC_FENCE_S);
support::endian::write<uint8_t>(OS, 0, llvm::endianness::little);
}
break;
}
case WebAssembly::OPERAND_VEC_I16IMM:
support::endian::write<uint16_t>(OS, MO.getImm(),
llvm::endianness::little);

2
llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
View file

@ -82,6 +82,8 @@ enum OperandType {
OPERAND_TABLE,
/// A list of catch clauses for try_table.
OPERAND_CATCH_LIST,
/// Memory ordering immediate for atomic instructions.
OPERAND_MEMORDER,
};
} // end namespace WebAssembly

6
llvm/lib/Target/WebAssembly/WebAssembly.td
View file

@ -73,6 +73,10 @@ def FeatureReferenceTypes :
SubtargetFeature<"reference-types", "HasReferenceTypes", "true",
"Enable reference types">;
def FeatureRelaxedAtomics :
SubtargetFeature<"relaxed-atomics", "HasRelaxedAtomics", "true",
"Enable relaxed-atomics proposal">;
def FeatureRelaxedSIMD :
SubtargetFeature<"relaxed-simd", "SIMDLevel", "RelaxedSIMD",
"Enable relaxed-simd instructions">;
@ -166,7 +170,7 @@ def WebAssemblyAsmParser : AsmParser {
def WebAssemblyAsmWriter : AsmWriter {
string AsmWriterClassName = "InstPrinter";
int PassSubtarget = 0;
int PassSubtarget = 1;
int Variant = 0;
bit isMCAsmWriter = 1;
}

201
llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
View file

@ -35,66 +35,74 @@ multiclass ATOMIC_NRI<dag oops, dag iops, list<dag> pattern, string asmstr = "",
let hasSideEffects = 1 in {
defm MEMORY_ATOMIC_NOTIFY_A32 :
ATOMIC_I<(outs I32:$dst),
(ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count),
(outs), (ins P2Align:$p2align, offset32_op:$off), [],
"memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count",
"memory.atomic.notify \t${off}${p2align}", 0x00, false>;
(ins MemOrder:$order, P2Align:$p2align, offset32_op:$off, I32:$addr,
I32:$count),
(outs), (ins MemOrder:$order, P2Align:$p2align, offset32_op:$off),
[], "memory.atomic.notify \t$dst, "
"${order} ${off}(${addr})${p2align}, $count",
"memory.atomic.notify \t${order} ${off}${p2align}", 0x00, false>;
defm MEMORY_ATOMIC_NOTIFY_A64 :
ATOMIC_I<(outs I32:$dst),
(ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$count),
(outs), (ins P2Align:$p2align, offset64_op:$off), [],
"memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count",
"memory.atomic.notify \t${off}${p2align}", 0x00, true>;
(ins MemOrder:$order, P2Align:$p2align, offset64_op:$off, I64:$addr,
I32:$count),
(outs), (ins MemOrder:$order, P2Align:$p2align, offset64_op:$off),
[], "memory.atomic.notify \t$dst, "
"${order} ${off}(${addr})${p2align}, $count",
"memory.atomic.notify \t${order} ${off}${p2align}", 0x00, true>;
let mayLoad = 1 in {
defm MEMORY_ATOMIC_WAIT32_A32 :
ATOMIC_I<(outs I32:$dst),
(ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp,
I64:$timeout),
(outs), (ins P2Align:$p2align, offset32_op:$off), [],
"memory.atomic.wait32 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout",
"memory.atomic.wait32 \t${off}${p2align}", 0x01, false>;
(ins MemOrder:$order, P2Align:$p2align, offset32_op:$off, I32:$addr,
I32:$exp, I64:$timeout),
(outs), (ins MemOrder:$order, P2Align:$p2align, offset32_op:$off),
[], "memory.atomic.wait32 \t$dst, "
"${order} ${off}(${addr})${p2align}, $exp, $timeout",
"memory.atomic.wait32 \t${order} ${off}${p2align}", 0x01, false>;
defm MEMORY_ATOMIC_WAIT32_A64 :
ATOMIC_I<(outs I32:$dst),
(ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$exp,
I64:$timeout),
(outs), (ins P2Align:$p2align, offset64_op:$off), [],
"memory.atomic.wait32 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout",
"memory.atomic.wait32 \t${off}${p2align}", 0x01, true>;
(ins MemOrder:$order, P2Align:$p2align, offset64_op:$off, I64:$addr,
I32:$exp, I64:$timeout),
(outs), (ins MemOrder:$order, P2Align:$p2align, offset64_op:$off),
[], "memory.atomic.wait32 \t$dst, "
"${order} ${off}(${addr})${p2align}, $exp, $timeout",
"memory.atomic.wait32 \t${order} ${off}${p2align}", 0x01, true>;
defm MEMORY_ATOMIC_WAIT64_A32 :
ATOMIC_I<(outs I32:$dst),
(ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp,
I64:$timeout),
(outs), (ins P2Align:$p2align, offset32_op:$off), [],
"memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout",
"memory.atomic.wait64 \t${off}${p2align}", 0x02, false>;
(ins MemOrder:$order, P2Align:$p2align, offset32_op:$off, I32:$addr,
I64:$exp, I64:$timeout),
(outs), (ins MemOrder:$order, P2Align:$p2align, offset32_op:$off),
[], "memory.atomic.wait64 \t$dst, "
"${order} ${off}(${addr})${p2align}, $exp, $timeout",
"memory.atomic.wait64 \t${order} ${off}${p2align}", 0x02, false>;
defm MEMORY_ATOMIC_WAIT64_A64 :
ATOMIC_I<(outs I32:$dst),
(ins P2Align:$p2align, offset64_op:$off, I64:$addr, I64:$exp,
I64:$timeout),
(outs), (ins P2Align:$p2align, offset64_op:$off), [],
"memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout",
"memory.atomic.wait64 \t${off}${p2align}", 0x02, true>;
(ins MemOrder:$order, P2Align:$p2align, offset64_op:$off, I64:$addr,
I64:$exp, I64:$timeout),
(outs), (ins MemOrder:$order, P2Align:$p2align, offset64_op:$off),
[], "memory.atomic.wait64 \t$dst, "
"${order} ${off}(${addr})${p2align}, $exp, $timeout",
"memory.atomic.wait64 \t${order} ${off}${p2align}", 0x02, true>;
} // mayLoad = 1
} // hasSideEffects = 1
def NotifyPat_A32 :
Pat<(i32 (int_wasm_memory_atomic_notify (AddrOps32 offset32_op:$offset, I32:$addr), I32:$count)),
(MEMORY_ATOMIC_NOTIFY_A32 0, $offset, $addr, $count)>,
(MEMORY_ATOMIC_NOTIFY_A32 0, 0, $offset, $addr, $count)>,
Requires<[HasAddr32, HasAtomics]>;
def NotifyPat_A64 :
Pat<(i32 (int_wasm_memory_atomic_notify (AddrOps64 offset64_op:$offset, I64:$addr), I32:$count)),
(MEMORY_ATOMIC_NOTIFY_A64 0, $offset, $addr, $count)>,
(MEMORY_ATOMIC_NOTIFY_A64 0, 0, $offset, $addr, $count)>,
Requires<[HasAddr64, HasAtomics]>;
multiclass WaitPat<ValueType ty, Intrinsic kind, string inst> {
def WaitPat_A32 :
Pat<(i32 (kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$exp, I64:$timeout)),
(!cast<NI>(inst#_A32) 0, $offset, $addr, $exp, $timeout)>,
(!cast<NI>(inst#_A32) 0, 0, $offset, $addr, $exp, $timeout)>,
Requires<[HasAddr32, HasAtomics]>;
def WaitPat_A64 :
Pat<(i32 (kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$exp, I64:$timeout)),
(!cast<NI>(inst#_A64) 0, $offset, $addr, $exp, $timeout)>,
(!cast<NI>(inst#_A64) 0, 0, $offset, $addr, $exp, $timeout)>,
Requires<[HasAddr64, HasAtomics]>;
}
@ -110,8 +118,8 @@ let Defs = [ARGUMENTS] in {
let isPseudo = 1, hasSideEffects = 1 in
defm COMPILER_FENCE : ATOMIC_NRI<(outs), (ins), [], "compiler_fence">;
let hasSideEffects = 1 in
defm ATOMIC_FENCE : ATOMIC_NRI<(outs), (ins i8imm:$flags), [], "atomic.fence",
0x03>;
defm ATOMIC_FENCE : ATOMIC_NRI<(outs), (ins MemOrder:$order), [],
"atomic.fence\t${order}", 0x03>;
} // Defs = [ARGUMENTS]
//===----------------------------------------------------------------------===//
@ -119,16 +127,43 @@ defm ATOMIC_FENCE : ATOMIC_NRI<(outs), (ins i8imm:$flags), [], "atomic.fence",
//===----------------------------------------------------------------------===//
multiclass AtomicLoad<WebAssemblyRegClass rc, string name, int atomic_op> {
defm "" : WebAssemblyLoad<rc, name, !or(0xfe00, !and(0xff, atomic_op)),
[HasAtomics]>;
let mayLoad = 1, UseNamedOperandTable = 1 in {
defm "_A32": ATOMIC_I<(outs rc:$dst),
(ins MemOrder:$order, P2Align:$p2align,
offset32_op:$off, I32:$addr),
(outs), (ins MemOrder:$order, P2Align:$p2align,
offset32_op:$off),
[], !strconcat(name, "\t$dst, ${order} "
"${off}(${addr})${p2align}"),
!strconcat(name, "\t${order} ${off}${p2align}"),
atomic_op, false>;
defm "_A64": ATOMIC_I<(outs rc:$dst),
(ins MemOrder:$order, P2Align:$p2align,
offset64_op:$off, I64:$addr),
(outs), (ins MemOrder:$order, P2Align:$p2align,
offset64_op:$off),
[], !strconcat(name, "\t$dst, ${order} "
"${off}(${addr})${p2align}"),
!strconcat(name, "\t${order} ${off}${p2align}"),
atomic_op, true>;
}
}
multiclass AtomicLoadPat<ValueType ty, SDPatternOperator kind, string Name> {
def : Pat<(ty (kind (AddrOps32 offset32_op:$offset, I32:$addr))),
(!cast<NI>(Name # "_A32") 0, 0, offset32_op:$offset, I32:$addr)>,
Requires<[HasAddr32, HasAtomics]>;
def : Pat<(ty (kind (AddrOps64 offset64_op:$offset, I64:$addr))),
(!cast<NI>(Name # "_A64") 0, 0, offset64_op:$offset, I64:$addr)>,
Requires<[HasAddr64, HasAtomics]>;
}
defm ATOMIC_LOAD_I32 : AtomicLoad<I32, "i32.atomic.load", 0x10>;
defm ATOMIC_LOAD_I64 : AtomicLoad<I64, "i64.atomic.load", 0x11>;
// Select loads
defm : LoadPat<i32, atomic_load_nonext_32, "ATOMIC_LOAD_I32">;
defm : LoadPat<i64, atomic_load_nonext_64, "ATOMIC_LOAD_I64">;
defm : AtomicLoadPat<i32, atomic_load_nonext_32, "ATOMIC_LOAD_I32">;
defm : AtomicLoadPat<i64, atomic_load_nonext_64, "ATOMIC_LOAD_I64">;
// Extending loads. Note that there are only zero-extending atomic loads, no
// sign-extending loads.
@ -168,15 +203,15 @@ def sext_aload_16_64 :
PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_azext_16 node:$addr)))>;
// Select zero-extending loads
defm : LoadPat<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">;
defm : LoadPat<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">;
defm : LoadPat<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">;
defm : AtomicLoadPat<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">;
defm : AtomicLoadPat<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">;
defm : AtomicLoadPat<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">;
// Select sign-extending loads
defm : LoadPat<i32, atomic_load_zext_8, "ATOMIC_LOAD8_U_I32">;
defm : LoadPat<i32, atomic_load_zext_16, "ATOMIC_LOAD16_U_I32">;
defm : LoadPat<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">;
defm : LoadPat<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">;
defm : AtomicLoadPat<i32, atomic_load_zext_8, "ATOMIC_LOAD8_U_I32">;
defm : AtomicLoadPat<i32, atomic_load_zext_16, "ATOMIC_LOAD16_U_I32">;
defm : AtomicLoadPat<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">;
defm : AtomicLoadPat<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">;
// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s
@ -185,8 +220,26 @@ defm : LoadPat<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">;
//===----------------------------------------------------------------------===//
multiclass AtomicStore<WebAssemblyRegClass rc, string name, int atomic_op> {
defm "" : WebAssemblyStore<rc, name, !or(0xfe00, !and(0xff, atomic_op)),
[HasAtomics]>;
let mayStore = 1, UseNamedOperandTable = 1 in {
defm "_A32" : ATOMIC_I<(outs),
(ins MemOrder:$order, P2Align:$p2align,
offset32_op:$off, I32:$addr, rc:$val),
(outs), (ins MemOrder:$order, P2Align:$p2align,
offset32_op:$off), [],
!strconcat(name, "\t${order} ${off}"
"(${addr})${p2align}, $val"),
!strconcat(name, "\t${order} ${off}${p2align}"),
atomic_op, false>;
defm "_A64" : ATOMIC_I<(outs),
(ins MemOrder:$order, P2Align:$p2align,
offset64_op:$off, I64:$addr, rc:$val),
(outs), (ins MemOrder:$order, P2Align:$p2align,
offset64_op:$off), [],
!strconcat(name, "\t${order} ${off}"
"(${addr})${p2align}, $val"),
!strconcat(name, "\t${order} ${off}${p2align}"),
atomic_op, true>;
}
}
defm ATOMIC_STORE_I32 : AtomicStore<I32, "i32.atomic.store", 0x17>;
@ -200,10 +253,10 @@ defm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>;
multiclass AStorePat<ValueType ty, PatFrag kind, string inst> {
def : Pat<(kind ty:$val, (AddrOps32 offset32_op:$offset, I32:$addr)),
(!cast<NI>(inst#_A32) 0, $offset, $addr, $val)>,
(!cast<NI>(inst#_A32) 0, 0, $offset, $addr, $val)>,
Requires<[HasAddr32, HasAtomics]>;
def : Pat<(kind ty:$val, (AddrOps64 offset64_op:$offset, I64:$addr)),
(!cast<NI>(inst#_A64) 0, $offset, $addr, $val)>,
(!cast<NI>(inst#_A64) 0, 0, $offset, $addr, $val)>,
Requires<[HasAddr64, HasAtomics]>;
}
defm : AStorePat<i32, atomic_store_32, "ATOMIC_STORE_I32">;
@ -243,16 +296,20 @@ multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string name,
int atomic_op> {
defm "_A32" :
ATOMIC_I<(outs rc:$dst),
(ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val),
(outs), (ins P2Align:$p2align, offset32_op:$off), [],
!strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"),
!strconcat(name, "\t${off}${p2align}"), atomic_op, false>;
(ins MemOrder:$order, P2Align:$p2align, offset32_op:$off,
I32:$addr, rc:$val),
(outs), (ins MemOrder:$order, P2Align:$p2align, offset32_op:$off),
[], !strconcat(name, "\t$dst, ${order} "
"${off}(${addr})${p2align}, $val"),
!strconcat(name, "\t${order} ${off}${p2align}"), atomic_op, false>;
defm "_A64" :
ATOMIC_I<(outs rc:$dst),
(ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$val),
(outs), (ins P2Align:$p2align, offset64_op:$off), [],
!strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"),
!strconcat(name, "\t${off}${p2align}"), atomic_op, true>;
(ins MemOrder:$order, P2Align:$p2align, offset64_op:$off,
I64:$addr, rc:$val),
(outs), (ins MemOrder:$order, P2Align:$p2align, offset64_op:$off),
[], !strconcat(name, "\t$dst, ${order} "
"${off}(${addr})${p2align}, $val"),
!strconcat(name, "\t${order} ${off}${p2align}"), atomic_op, true>;
}
defm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0x1e>;
@ -337,10 +394,10 @@ defm ATOMIC_RMW32_U_XCHG_I64 :
multiclass BinRMWPat<ValueType ty, PatFrag kind, string inst> {
def : Pat<(ty (kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$val)),
(!cast<NI>(inst#_A32) 0, $offset, $addr, $val)>,
(!cast<NI>(inst#_A32) 0, 0, $offset, $addr, $val)>,
Requires<[HasAddr32, HasAtomics]>;
def : Pat<(ty (kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$val)),
(!cast<NI>(inst#_A64) 0, $offset, $addr, $val)>,
(!cast<NI>(inst#_A64) 0, 0, $offset, $addr, $val)>,
Requires<[HasAddr64, HasAtomics]>;
}
@ -447,18 +504,20 @@ multiclass WebAssemblyTerRMW<WebAssemblyRegClass rc, string name,
int atomic_op> {
defm "_A32" :
ATOMIC_I<(outs rc:$dst),
(ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp,
rc:$new_),
(outs), (ins P2Align:$p2align, offset32_op:$off), [],
!strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"),
!strconcat(name, "\t${off}${p2align}"), atomic_op, false>;
(ins MemOrder:$order, P2Align:$p2align, offset32_op:$off,
I32:$addr, rc:$exp, rc:$new_),
(outs), (ins MemOrder:$order, P2Align:$p2align, offset32_op:$off),
[], !strconcat(name, "\t$dst, ${order} "
"${off}(${addr})${p2align}, $exp, $new_"),
!strconcat(name, "\t${order} ${off}${p2align}"), atomic_op, false>;
defm "_A64" :
ATOMIC_I<(outs rc:$dst),
(ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$exp,
rc:$new_),
(outs), (ins P2Align:$p2align, offset64_op:$off), [],
!strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"),
!strconcat(name, "\t${off}${p2align}"), atomic_op, true>;
(ins MemOrder:$order, P2Align:$p2align, offset64_op:$off,
I64:$addr, rc:$exp, rc:$new_),
(outs), (ins MemOrder:$order, P2Align:$p2align, offset64_op:$off),
[], !strconcat(name, "\t$dst, ${order} "
"${off}(${addr})${p2align}, $exp, $new_"),
!strconcat(name, "\t${order} ${off}${p2align}"), atomic_op, true>;
}
defm ATOMIC_RMW_CMPXCHG_I32 :
@ -478,10 +537,10 @@ defm ATOMIC_RMW32_U_CMPXCHG_I64 :
multiclass TerRMWPat<ValueType ty, PatFrag kind, string inst> {
def : Pat<(ty (kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$exp, ty:$new)),
(!cast<NI>(inst#_A32) 0, $offset, $addr, $exp, $new)>,
(!cast<NI>(inst#_A32) 0, 0, $offset, $addr, $exp, $new)>,
Requires<[HasAddr32, HasAtomics]>;
def : Pat<(ty (kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$exp, ty:$new)),
(!cast<NI>(inst#_A64) 0, $offset, $addr, $exp, $new)>,
(!cast<NI>(inst#_A64) 0, 0, $offset, $addr, $exp, $new)>,
Requires<[HasAddr64, HasAtomics]>;
}

10
llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
View file

@ -79,6 +79,10 @@ def HasReferenceTypes :
Predicate<"Subtarget->hasReferenceTypes()">,
AssemblerPredicate<(all_of FeatureReferenceTypes), "reference-types">;
def HasRelaxedAtomics :
Predicate<"Subtarget->hasRelaxedAtomics()">,
AssemblerPredicate<(all_of FeatureRelaxedAtomics), "relaxed-atomics">;
def HasRelaxedSIMD :
Predicate<"Subtarget->hasRelaxedSIMD()">,
AssemblerPredicate<(all_of FeatureRelaxedSIMD), "relaxed-simd">;
@ -230,6 +234,12 @@ def tag_op : Operand<i32>;
} // OperandType = "OPERAND_P2ALIGN"
let OperandType = "OPERAND_MEMORDER" in {
def MemOrder : Operand<i32> {
let PrintMethod = "printWebAssemblyMemOrderOperand";
}
} // OperandType = "OPERAND_MEMORDER"
let OperandType = "OPERAND_SIGNATURE" in
def Signature : Operand<i32> {
let PrintMethod = "printWebAssemblySignatureOperand";

4
llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h
View file

@ -57,6 +57,7 @@ class WebAssemblySubtarget final : public WebAssemblyGenSubtargetInfo {
bool HasMutableGlobals = false;
bool HasNontrappingFPToInt = false;
bool HasReferenceTypes = false;
bool HasRelaxedAtomics = false;
bool HasSignExt = false;
bool HasTailCall = false;
bool HasWideArithmetic = false;
@ -114,12 +115,13 @@ public:
bool hasExceptionHandling() const { return HasExceptionHandling; }
bool hasExtendedConst() const { return HasExtendedConst; }
bool hasFP16() const { return HasFP16; }
bool hasGC() const { return HasGC; }
bool hasMultiMemory() const { return HasMultiMemory; }
bool hasMultivalue() const { return HasMultivalue; }
bool hasMutableGlobals() const { return HasMutableGlobals; }
bool hasNontrappingFPToInt() const { return HasNontrappingFPToInt; }
bool hasReferenceTypes() const { return HasReferenceTypes; }
bool hasGC() const { return HasGC; }
bool hasRelaxedAtomics() const { return HasRelaxedAtomics; }
bool hasRelaxedSIMD() const { return SIMDLevel >= RelaxedSIMD; }
bool hasSignExt() const { return HasSignExt; }
bool hasSIMD128() const { return SIMDLevel >= SIMD128; }

4
llvm/test/CodeGen/WebAssembly/atomic-fence.mir
View file

@ -34,7 +34,7 @@ body: |
liveins: $arguments
%0:i32 = CONST_I32 0, implicit-def $arguments
%1:i32 = MEMORY_ATOMIC_NOTIFY_A32 2, 0, %0:i32, %0:i32, implicit-def $arguments
%1:i32 = MEMORY_ATOMIC_NOTIFY_A32 0, 2, 0, %0:i32, %0:i32, implicit-def $arguments
COMPILER_FENCE implicit-def $arguments
%2:i32 = ADD_I32 %0:i32, %0:i32, implicit-def $arguments
CALL @foo, %2:i32, %1:i32, implicit-def $arguments
@ -58,7 +58,7 @@ body: |
liveins: $arguments
%0:i32 = CONST_I32 0, implicit-def $arguments
%1:i32 = MEMORY_ATOMIC_NOTIFY_A32 2, 0, %0:i32, %0:i32, implicit-def $arguments
%1:i32 = MEMORY_ATOMIC_NOTIFY_A32 0, 2, 0, %0:i32, %0:i32, implicit-def $arguments
ATOMIC_FENCE 0, implicit-def $arguments
%2:i32 = ADD_I32 %0:i32, %0:i32, implicit-def $arguments
CALL @foo, %2:i32, %1:i32, implicit-def $arguments

15
llvm/test/MC/WebAssembly/atomics-orderings-errors.s Normal file
View file

@ -0,0 +1,15 @@
# RUN: not llvm-mc -triple=wasm32-unknown-unknown -mattr=+atomics < %s 2>&1 | FileCheck %s
main:
.functype main () -> ()
# CHECK: :[[@LINE+1]]:16: error: memory ordering requires relaxed-atomics feature: acqrel
atomic.fence acqrel
# CHECK: :[[@LINE+1]]:19: error: memory ordering requires relaxed-atomics feature: acqrel
i32.atomic.load acqrel 0
# CHECK: :[[@LINE+1]]:22: error: memory ordering requires relaxed-atomics feature: seqcst
i32.atomic.rmw.add seqcst 0
end_function

69
llvm/test/MC/WebAssembly/atomics-orderings.s Normal file
View file

@ -0,0 +1,69 @@
# RUN: llvm-mc -no-type-check -show-encoding -triple=wasm32-unknown-unknown -mattr=+atomics,+relaxed-atomics < %s | FileCheck %s
# RUN: llvm-mc -no-type-check -triple=wasm32-unknown-unknown -mattr=+atomics,+relaxed-atomics %s -filetype=obj -o - | llvm-objdump --mattr=+atomics,+relaxed-atomics --no-print-imm-hex -d - | FileCheck %s --check-prefix=DISASM
# Ensure we can disassemble even when not explicitly enabling the feature in the disassembler.
# In that case we will print nothing instead of "seqcst" but will still print "acqrel"
# RUN: llvm-mc -no-type-check -triple=wasm32-unknown-unknown -mattr=+atomics,+relaxed-atomics %s -filetype=obj -o - | llvm-objdump --no-print-imm-hex -d - | FileCheck %s --check-prefix=DISASM-NOATTR
.section .text.main,"",@
main:
.functype main () -> ()
atomic.fence seqcst
# CHECK: atomic.fence seqcst # encoding: [0xfe,0x03,0x00]
# DISASM: atomic.fence seqcst
# DISASM-NOATTR: atomic.fence {{$}}
atomic.fence acqrel
# CHECK: atomic.fence acqrel # encoding: [0xfe,0x03,0x01]
# DISASM: atomic.fence acqrel
# DISASM-NOATTR: atomic.fence acqrel
atomic.fence
# CHECK: atomic.fence seqcst # encoding: [0xfe,0x03,0x00]
# DISASM: atomic.fence seqcst
# DISASM-NOATTR: atomic.fence {{$}}
# CHECK: i32.atomic.load seqcst 0 # encoding: [0xfe,0x10,0x02,0x00]
# DISASM: i32.atomic.load seqcst 0
i32.atomic.load 0
# CHECK: i32.atomic.load seqcst 0 # encoding: [0xfe,0x10,0x02,0x00]
# DISASM: i32.atomic.load seqcst 0
i32.atomic.load seqcst 0
# CHECK: i32.atomic.load acqrel 0 # encoding: [0xfe,0x10,0x22,0x01,0x00]
# DISASM: i32.atomic.load acqrel 0
i32.atomic.load acqrel 0
# CHECK: i64.atomic.load acqrel 0 # encoding: [0xfe,0x11,0x23,0x01,0x00]
# DISASM: i64.atomic.load acqrel 0
i64.atomic.load acqrel 0
# CHECK: i32.atomic.store acqrel 0 # encoding: [0xfe,0x17,0x22,0x01,0x00]
# DISASM: i32.atomic.store acqrel 0
i32.atomic.store acqrel 0
# CHECK: i64.atomic.store acqrel 8 # encoding: [0xfe,0x18,0x23,0x01,0x08]
# DISASM: i64.atomic.store acqrel 8
i64.atomic.store acqrel 8
# CHECK: i32.atomic.rmw.add acqrel 0 # encoding: [0xfe,0x1e,0x22,0x11,0x00]
# DISASM: i32.atomic.rmw.add acqrel 0
i32.atomic.rmw.add acqrel 0
# CHECK: i64.atomic.rmw.cmpxchg acqrel 0 # encoding: [0xfe,0x49,0x23,0x11,0x00]
# DISASM: i64.atomic.rmw.cmpxchg acqrel 0
i64.atomic.rmw.cmpxchg acqrel 0
# CHECK: i32.atomic.load8_u seqcst 0 # encoding: [0xfe,0x12,0x00,0x00]
# DISASM: i32.atomic.load8_u seqcst 0
i32.atomic.load8_u seqcst 0:p2align=0
# CHECK: i64.atomic.rmw32.xchg_u acqrel 0 # encoding: [0xfe,0x47,0x22,0x11,0x00]
# DISASM: i64.atomic.rmw32.xchg_u acqrel 0
i64.atomic.rmw32.xchg_u acqrel 0
end_function