From 67fea8c7f9bdde801ecc721938ea5798c36dd7e4 Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Mon, 5 Oct 2026 21:46:37 -0700 Subject: [PATCH] x64: add APX NDD lowering for simple integer binary ops --- .../meta/src/instructions/and.rs | 6 +- .../assembler-x64/meta/src/instructions/or.rs | 6 +- .../meta/src/instructions/sub.rs | 11 +- .../meta/src/instructions/xor.rs | 6 +- cranelift/assembler-x64/src/fuzz.rs | 118 ++++++++++++++++++ cranelift/codegen/src/isa/x64/lower.isle | 30 +++++ .../filetests/isa/x64/apx-arith.clif | 72 +++++++++++ .../filetests/filetests/isa/x64/apx-sub.clif | 26 ++++ 8 files changed, 267 insertions(+), 8 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/x64/apx-arith.clif create mode 100644 cranelift/filetests/filetests/isa/x64/apx-sub.clif diff --git a/cranelift/assembler-x64/meta/src/instructions/and.rs b/cranelift/assembler-x64/meta/src/instructions/and.rs index b22e7aa2fb15..7806d3128426 100644 --- a/cranelift/assembler-x64/meta/src/instructions/and.rs +++ b/cranelift/assembler-x64/meta/src/instructions/and.rs @@ -1,5 +1,5 @@ -use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*}; -use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w}; +use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*}; +use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w}; #[rustfmt::skip] // Keeps instructions on a single line. pub fn list() -> Vec { @@ -29,6 +29,8 @@ pub fn list() -> Vec { inst("andw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x23]).r(), _64b | compat), inst("andl", fmt("RM", [rw(r32), r(rm32)]), rex(0x23).r(), _64b | compat), inst("andq", fmt("RM", [rw(r64), r(rm64)]), rex(0x23).w().r(), _64b), + // APX: NDD form of `AND`, promoted into EVEX map 4. + inst("andq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x21).r(), _64b & apx), // BMI1 andn inst("andnl", fmt("RVM", [w(r32a), r(r32b), r(rm32)]), vex(LZ)._0f38().w0().op(0xF2), (_64b | compat) & bmi1), inst("andnq", fmt("RVM", [w(r64a), r(r64b), r(rm64)]), vex(LZ)._0f38().w1().op(0xF2), _64b & bmi1), diff --git a/cranelift/assembler-x64/meta/src/instructions/or.rs b/cranelift/assembler-x64/meta/src/instructions/or.rs index a99ba36a9830..af56d3274b53 100644 --- a/cranelift/assembler-x64/meta/src/instructions/or.rs +++ b/cranelift/assembler-x64/meta/src/instructions/or.rs @@ -1,5 +1,5 @@ -use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*}; -use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w}; +use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*}; +use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w}; #[rustfmt::skip] // Keeps instructions on a single line. pub fn list() -> Vec { @@ -22,6 +22,8 @@ pub fn list() -> Vec { inst("orw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x0B]).r(), _64b | compat), inst("orl", fmt("RM", [rw(r32), r(rm32)]), rex(0x0B).r(), _64b | compat), inst("orq", fmt("RM", [rw(r64), r(rm64)]), rex(0x0B).w().r(), _64b), + // APX: NDD form of `OR`, promoted into EVEX map 4. + inst("orq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x09).r(), _64b & apx), // `LOCK`-prefixed memory-writing instructions. inst("lock_orb", fmt("MI", [rw(m8), r(imm8)]), rex([0xf0, 0x80]).digit(1).ib(), _64b | compat).custom(Mnemonic), inst("lock_orw", fmt("MI", [rw(m16), r(imm16)]), rex([0xf0, 0x66, 0x81]).digit(1).iw(), _64b | compat).custom(Mnemonic), diff --git a/cranelift/assembler-x64/meta/src/instructions/sub.rs b/cranelift/assembler-x64/meta/src/instructions/sub.rs index e476f7292a0a..8244725dc874 100644 --- a/cranelift/assembler-x64/meta/src/instructions/sub.rs +++ b/cranelift/assembler-x64/meta/src/instructions/sub.rs @@ -1,5 +1,5 @@ -use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*}; -use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w}; +use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*}; +use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w}; #[rustfmt::skip] // Keeps instructions on a single line. pub fn list() -> Vec { @@ -22,6 +22,13 @@ pub fn list() -> Vec { inst("subw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x2B]).r(), _64b | compat), inst("subl", fmt("RM", [rw(r32), r(rm32)]), rex(0x2B).r(), _64b | compat), inst("subq", fmt("RM", [rw(r64), r(rm64)]), rex(0x2B).w().r(), _64b), + // APX: NDD (new data destination) form of `SUB`, promoted into EVEX + // "map 4" via the extended-EVEX prefix. With `ND = 1` the architectural + // destination is the `vvvv`-encoded register, so the written operand is + // placed in the `V` slot (the middle operand of the `RVM` format) while + // the two sources occupy ModRM.reg and ModRM.rm. `subq` is 64-bit so + // the `W` bit is set (`.w1()`); use `.w0()` for the 32-bit `subl` form. + inst("subq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x29).r(), _64b & apx), // Subtract with borrow. inst("sbbb", fmt("I", [rw(al), r(imm8)]), rex(0x1C).ib(), _64b | compat), inst("sbbw", fmt("I", [rw(ax), r(imm16)]), rex([0x66, 0x1D]).iw(), _64b | compat), diff --git a/cranelift/assembler-x64/meta/src/instructions/xor.rs b/cranelift/assembler-x64/meta/src/instructions/xor.rs index 08ac5a9f0f62..52b93ddc2cf4 100644 --- a/cranelift/assembler-x64/meta/src/instructions/xor.rs +++ b/cranelift/assembler-x64/meta/src/instructions/xor.rs @@ -1,5 +1,5 @@ -use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*}; -use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w}; +use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*}; +use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w}; #[rustfmt::skip] // Keeps instructions on a single line. pub fn list() -> Vec { @@ -22,6 +22,8 @@ pub fn list() -> Vec { inst("xorw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x33]).r(), _64b | compat), inst("xorl", fmt("RM", [rw(r32), r(rm32)]), rex(0x33).r(), _64b | compat), inst("xorq", fmt("RM", [rw(r64), r(rm64)]), rex(0x33).w().r(), _64b), + // APX: NDD form of `XOR`, promoted into EVEX map 4. + inst("xorq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x31).r(), _64b & apx), // `LOCK`-prefixed memory-writing instructions. inst("lock_xorb", fmt("MI", [rw(m8), r(imm8)]), rex([0xf0, 0x80]).digit(6).ib(), _64b | compat).custom(Mnemonic), inst("lock_xorw", fmt("MI", [rw(m16), r(imm16)]), rex([0xf0, 0x66, 0x81]).digit(6).iw(), _64b | compat).custom(Mnemonic), diff --git a/cranelift/assembler-x64/src/fuzz.rs b/cranelift/assembler-x64/src/fuzz.rs index 99166feca756..4d8f3a685ec5 100644 --- a/cranelift/assembler-x64/src/fuzz.rs +++ b/cranelift/assembler-x64/src/fuzz.rs @@ -654,4 +654,122 @@ mod test { let assembled = assemble(&inst.into()); assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418017F50"); } + + /// Byte-level encoding check for the APX NDD (new-data-destination) form of + /// `SUB`, promoted into EVEX "map 4" via the extended-EVEX prefix. + /// + /// Operand placement is the same as [`apx_addq_rvm_ndd_encoding`]: in `RVM` + /// format the constructor arguments are `[ModRM.reg source, + /// vvvv destination, ModRM.rm source]`. For `subq %rax, %rcx, %rdx` + /// (destination `%rax` = 0 in `vvvv`, source `%rcx` = 1 in ModRM.reg, + /// source `%rdx` = 2 in ModRM.rm), `W = 1`, `ND = 1`, `NF = 0`, and the + /// legacy `SUB r/m, reg` opcode byte is `0x29`. + #[test] + fn apx_subq_rvm_ndd_encoding() { + use crate::inst::subq_rvm; + + let inst = subq_rvm::::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2)); + let assembled = assemble(&inst.into()); + assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1829CA"); + } + + /// Companion to [`apx_subq_rvm_ndd_encoding`] covering a memory operand. + #[test] + fn apx_subq_rvm_ndd_disp8_is_unscaled() { + use crate::inst::subq_rvm; + use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem}; + + let mem: GprMem = GprMem::Mem(Amode::ImmReg { + base: FuzzReg::new(7), + simm32: AmodeOffsetPlusKnownOffset { + simm32: AmodeOffset::new(0x50), + offset: None, + }, + trap: None, + }); + let inst = subq_rvm::::new(FuzzReg::new(7), FuzzReg::new(7), mem); + let assembled = assemble(&inst.into()); + assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418297F50"); + } + + #[test] + fn apx_andq_rvm_ndd_encoding() { + use crate::inst::andq_rvm; + + let inst = andq_rvm::::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2)); + let assembled = assemble(&inst.into()); + assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1821CA"); + } + + #[test] + fn apx_andq_rvm_ndd_disp8_is_unscaled() { + use crate::inst::andq_rvm; + use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem}; + + let mem: GprMem = GprMem::Mem(Amode::ImmReg { + base: FuzzReg::new(7), + simm32: AmodeOffsetPlusKnownOffset { + simm32: AmodeOffset::new(0x50), + offset: None, + }, + trap: None, + }); + let inst = andq_rvm::::new(FuzzReg::new(7), FuzzReg::new(7), mem); + let assembled = assemble(&inst.into()); + assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418217F50"); + } + + #[test] + fn apx_orq_rvm_ndd_encoding() { + use crate::inst::orq_rvm; + + let inst = orq_rvm::::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2)); + let assembled = assemble(&inst.into()); + assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1809CA"); + } + + #[test] + fn apx_orq_rvm_ndd_disp8_is_unscaled() { + use crate::inst::orq_rvm; + use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem}; + + let mem: GprMem = GprMem::Mem(Amode::ImmReg { + base: FuzzReg::new(7), + simm32: AmodeOffsetPlusKnownOffset { + simm32: AmodeOffset::new(0x50), + offset: None, + }, + trap: None, + }); + let inst = orq_rvm::::new(FuzzReg::new(7), FuzzReg::new(7), mem); + let assembled = assemble(&inst.into()); + assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418097F50"); + } + + #[test] + fn apx_xorq_rvm_ndd_encoding() { + use crate::inst::xorq_rvm; + + let inst = xorq_rvm::::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2)); + let assembled = assemble(&inst.into()); + assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1831CA"); + } + + #[test] + fn apx_xorq_rvm_ndd_disp8_is_unscaled() { + use crate::inst::xorq_rvm; + use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem}; + + let mem: GprMem = GprMem::Mem(Amode::ImmReg { + base: FuzzReg::new(7), + simm32: AmodeOffsetPlusKnownOffset { + simm32: AmodeOffset::new(0x50), + offset: None, + }, + trap: None, + }); + let inst = xorq_rvm::::new(FuzzReg::new(7), FuzzReg::new(7), mem); + let assembled = assemble(&inst.into()); + assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418317F50"); + } } diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 10c885289ed7..4fde7e22208b 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -306,6 +306,18 @@ ;; `i64` and smaller. +;; APX: when the `has_apx` feature is enabled, use the NDD (new-data- +;; destination) form of `sub` for 64-bit subtracts. Unlike the legacy two- +;; operand `sub`, the NDD form writes its result to a fresh register encoded in +;; the EVEX `vvvv` field, so the register allocator does not need to insert a +;; move to preserve either input. +;; +;; Note that the NDD form computes `dest = rm - reg`, so the minuend goes in +;; ModRM.rm (the second argument here) and the subtrahend in ModRM.reg. +(rule isub_apx_ndd 1 (lower (isub $I64 x y)) + (if-let true (use_apx)) + (x64_subq_rvm (put_in_gpr y) x)) + ;; Sub two registers. (rule -3 (lower (isub (fits_in_64 ty) x y)) (x64_sub ty x y)) @@ -370,6 +382,12 @@ ;; `{i,b}64` and smaller. +;; APX: when the `has_apx` feature is enabled, use the NDD form of `and` for +;; 64-bit integer values. +(rule band_apx_ndd 3 (lower (band $I64 x y)) + (if-let true (use_apx)) + (x64_andq_rvm x y)) + ;; And two registers. (rule 0 (lower (band ty x y)) (if (ty_int_ref_scalar_64 ty)) @@ -499,6 +517,12 @@ ;; `{i,b}64` and smaller. +;; APX: when the `has_apx` feature is enabled, use the NDD form of `or` for +;; 64-bit integer values. +(rule bor_apx_ndd 3 (lower (bor $I64 x y)) + (if-let true (use_apx)) + (x64_orq_rvm x y)) + ;; Or two registers. (rule 0 (lower (bor ty x y)) (if (ty_int_ref_scalar_64 ty)) @@ -572,6 +596,12 @@ ;; `{i,b}64` and smaller. +;; APX: when the `has_apx` feature is enabled, use the NDD form of `xor` for +;; 64-bit integer values. +(rule bxor_apx_ndd 3 (lower (bxor $I64 x y)) + (if-let true (use_apx)) + (x64_xorq_rvm x y)) + ;; Xor two registers. (rule 0 (lower (bxor ty x y)) (if (ty_int_ref_scalar_64 ty)) diff --git a/cranelift/filetests/filetests/isa/x64/apx-arith.clif b/cranelift/filetests/filetests/isa/x64/apx-arith.clif new file mode 100644 index 000000000000..c72630051c06 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/apx-arith.clif @@ -0,0 +1,72 @@ +test compile precise-output +target x86_64 has_apx + +function %and_i64(i64, i64) -> i64 { +block0(v0: i64, v1: i64): + v2 = band v0, v1 + return v2 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; block0: +; andq %rdi, %rsi, %rax +; movq %rbp, %rsp +; popq %rbp +; retq +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; block1: ; offset 0x4 +; .byte 0x62, 0xf4, 0xfc, 0x18, 0x21, 0xfe, 0x48, 0x89 +; .byte 0xec, 0x5d, 0xc3 + +function %or_i64(i64, i64) -> i64 { +block0(v0: i64, v1: i64): + v2 = bor v0, v1 + return v2 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; block0: +; orq %rdi, %rsi, %rax +; movq %rbp, %rsp +; popq %rbp +; retq +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; block1: ; offset 0x4 +; .byte 0x62, 0xf4, 0xfc, 0x18, 0x09, 0xfe, 0x48, 0x89 +; .byte 0xec, 0x5d, 0xc3 + +function %xor_i64(i64, i64) -> i64 { +block0(v0: i64, v1: i64): + v2 = bxor v0, v1 + return v2 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; block0: +; xorq %rdi, %rsi, %rax +; movq %rbp, %rsp +; popq %rbp +; retq +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; block1: ; offset 0x4 +; .byte 0x62, 0xf4, 0xfc, 0x18, 0x31, 0xfe, 0x48, 0x89 +; .byte 0xec, 0x5d, 0xc3 + diff --git a/cranelift/filetests/filetests/isa/x64/apx-sub.clif b/cranelift/filetests/filetests/isa/x64/apx-sub.clif new file mode 100644 index 000000000000..69ee4c51f6fe --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/apx-sub.clif @@ -0,0 +1,26 @@ +test compile precise-output +target x86_64 has_apx + +function %sub_i64(i64, i64) -> i64 { +block0(v0: i64, v1: i64): + v2 = isub v0, v1 + return v2 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; block0: +; subq %rsi, %rdi, %rax +; movq %rbp, %rsp +; popq %rbp +; retq +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; block1: ; offset 0x4 +; .byte 0x62, 0xf4, 0xfc, 0x18, 0x29, 0xf7, 0x48, 0x89 +; .byte 0xec, 0x5d, 0xc3 +