x64: add APX NDD lowering for simple integer binary ops

cranelift/assembler-x64/meta/src/instructions/and.rs

@@ -1,5 +1,5 @@
-use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*};
-use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
+use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*};
+use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
 
 #[rustfmt::skip] // Keeps instructions on a single line.
 pub fn list() -> Vec<Inst> {
@@ -29,6 +29,8 @@ pub fn list() -> Vec<Inst> {
         inst("andw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x23]).r(), _64b | compat),
         inst("andl", fmt("RM", [rw(r32), r(rm32)]), rex(0x23).r(), _64b | compat),
         inst("andq", fmt("RM", [rw(r64), r(rm64)]), rex(0x23).w().r(), _64b),
+        // APX: NDD form of `AND`, promoted into EVEX map 4.
+        inst("andq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x21).r(), _64b & apx),
         // BMI1 andn
         inst("andnl", fmt("RVM", [w(r32a), r(r32b), r(rm32)]), vex(LZ)._0f38().w0().op(0xF2), (_64b | compat) & bmi1),
         inst("andnq", fmt("RVM", [w(r64a), r(r64b), r(rm64)]), vex(LZ)._0f38().w1().op(0xF2), _64b & bmi1),

cranelift/assembler-x64/meta/src/instructions/or.rs

@@ -1,5 +1,5 @@
-use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*};
-use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
+use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*};
+use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
 
 #[rustfmt::skip] // Keeps instructions on a single line.
 pub fn list() -> Vec<Inst> {
@@ -22,6 +22,8 @@ pub fn list() -> Vec<Inst> {
         inst("orw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x0B]).r(), _64b | compat),
         inst("orl", fmt("RM", [rw(r32), r(rm32)]), rex(0x0B).r(), _64b | compat),
         inst("orq", fmt("RM", [rw(r64), r(rm64)]), rex(0x0B).w().r(), _64b),
+        // APX: NDD form of `OR`, promoted into EVEX map 4.
+        inst("orq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x09).r(), _64b & apx),
         // `LOCK`-prefixed memory-writing instructions.
         inst("lock_orb", fmt("MI", [rw(m8), r(imm8)]), rex([0xf0, 0x80]).digit(1).ib(), _64b | compat).custom(Mnemonic),
         inst("lock_orw", fmt("MI", [rw(m16), r(imm16)]), rex([0xf0, 0x66, 0x81]).digit(1).iw(), _64b | compat).custom(Mnemonic),

cranelift/assembler-x64/meta/src/instructions/sub.rs

@@ -1,5 +1,5 @@
-use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*};
-use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
+use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*};
+use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
 
 #[rustfmt::skip] // Keeps instructions on a single line.
 pub fn list() -> Vec<Inst> {
@@ -22,6 +22,13 @@ pub fn list() -> Vec<Inst> {
         inst("subw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x2B]).r(), _64b | compat),
         inst("subl", fmt("RM", [rw(r32), r(rm32)]), rex(0x2B).r(), _64b | compat),
         inst("subq", fmt("RM", [rw(r64), r(rm64)]), rex(0x2B).w().r(), _64b),
+        // APX: NDD (new data destination) form of `SUB`, promoted into EVEX
+        // "map 4" via the extended-EVEX prefix. With `ND = 1` the architectural
+        // destination is the `vvvv`-encoded register, so the written operand is
+        // placed in the `V` slot (the middle operand of the `RVM` format) while
+        // the two sources occupy ModRM.reg and ModRM.rm. `subq` is 64-bit so
+        // the `W` bit is set (`.w1()`); use `.w0()` for the 32-bit `subl` form.
+        inst("subq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x29).r(), _64b & apx),
         // Subtract with borrow.
         inst("sbbb", fmt("I", [rw(al), r(imm8)]), rex(0x1C).ib(), _64b | compat),
         inst("sbbw", fmt("I", [rw(ax), r(imm16)]), rex([0x66, 0x1D]).iw(), _64b | compat),

cranelift/assembler-x64/meta/src/instructions/xor.rs

@@ -1,5 +1,5 @@
-use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*};
-use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
+use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*};
+use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
 
 #[rustfmt::skip] // Keeps instructions on a single line.
 pub fn list() -> Vec<Inst> {
@@ -22,6 +22,8 @@ pub fn list() -> Vec<Inst> {
         inst("xorw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x33]).r(), _64b | compat),
         inst("xorl", fmt("RM", [rw(r32), r(rm32)]), rex(0x33).r(), _64b | compat),
         inst("xorq", fmt("RM", [rw(r64), r(rm64)]), rex(0x33).w().r(), _64b),
+        // APX: NDD form of `XOR`, promoted into EVEX map 4.
+        inst("xorq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x31).r(), _64b & apx),
         // `LOCK`-prefixed memory-writing instructions.
         inst("lock_xorb", fmt("MI", [rw(m8), r(imm8)]), rex([0xf0, 0x80]).digit(6).ib(), _64b | compat).custom(Mnemonic),
         inst("lock_xorw", fmt("MI", [rw(m16), r(imm16)]), rex([0xf0, 0x66, 0x81]).digit(6).iw(), _64b | compat).custom(Mnemonic),

cranelift/assembler-x64/src/fuzz.rs

@@ -654,4 +654,122 @@ mod test {
         let assembled = assemble(&inst.into());
         assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418017F50");
     }
+
+    /// Byte-level encoding check for the APX NDD (new-data-destination) form of
+    /// `SUB`, promoted into EVEX "map 4" via the extended-EVEX prefix.
+    ///
+    /// Operand placement is the same as [`apx_addq_rvm_ndd_encoding`]: in `RVM`
+    /// format the constructor arguments are `[ModRM.reg source,
+    /// vvvv destination, ModRM.rm source]`. For `subq %rax, %rcx, %rdx`
+    /// (destination `%rax` = 0 in `vvvv`, source `%rcx` = 1 in ModRM.reg,
+    /// source `%rdx` = 2 in ModRM.rm), `W = 1`, `ND = 1`, `NF = 0`, and the
+    /// legacy `SUB r/m, reg` opcode byte is `0x29`.
+    #[test]
+    fn apx_subq_rvm_ndd_encoding() {
+        use crate::inst::subq_rvm;
+
+        let inst = subq_rvm::<FuzzRegs>::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));
+        let assembled = assemble(&inst.into());
+        assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1829CA");
+    }
+
+    /// Companion to [`apx_subq_rvm_ndd_encoding`] covering a memory operand.
+    #[test]
+    fn apx_subq_rvm_ndd_disp8_is_unscaled() {
+        use crate::inst::subq_rvm;
+        use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};
+
+        let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {
+            base: FuzzReg::new(7),
+            simm32: AmodeOffsetPlusKnownOffset {
+                simm32: AmodeOffset::new(0x50),
+                offset: None,
+            },
+            trap: None,
+        });
+        let inst = subq_rvm::<FuzzRegs>::new(FuzzReg::new(7), FuzzReg::new(7), mem);
+        let assembled = assemble(&inst.into());
+        assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418297F50");
+    }
+
+    #[test]
+    fn apx_andq_rvm_ndd_encoding() {
+        use crate::inst::andq_rvm;
+
+        let inst = andq_rvm::<FuzzRegs>::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));
+        let assembled = assemble(&inst.into());
+        assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1821CA");
+    }
+
+    #[test]
+    fn apx_andq_rvm_ndd_disp8_is_unscaled() {
+        use crate::inst::andq_rvm;
+        use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};
+
+        let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {
+            base: FuzzReg::new(7),
+            simm32: AmodeOffsetPlusKnownOffset {
+                simm32: AmodeOffset::new(0x50),
+                offset: None,
+            },
+            trap: None,
+        });
+        let inst = andq_rvm::<FuzzRegs>::new(FuzzReg::new(7), FuzzReg::new(7), mem);
+        let assembled = assemble(&inst.into());
+        assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418217F50");
+    }
+
+    #[test]
+    fn apx_orq_rvm_ndd_encoding() {
+        use crate::inst::orq_rvm;
+
+        let inst = orq_rvm::<FuzzRegs>::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));
+        let assembled = assemble(&inst.into());
+        assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1809CA");
+    }
+
+    #[test]
+    fn apx_orq_rvm_ndd_disp8_is_unscaled() {
+        use crate::inst::orq_rvm;
+        use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};
+
+        let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {
+            base: FuzzReg::new(7),
+            simm32: AmodeOffsetPlusKnownOffset {
+                simm32: AmodeOffset::new(0x50),
+                offset: None,
+            },
+            trap: None,
+        });
+        let inst = orq_rvm::<FuzzRegs>::new(FuzzReg::new(7), FuzzReg::new(7), mem);
+        let assembled = assemble(&inst.into());
+        assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418097F50");
+    }
+
+    #[test]
+    fn apx_xorq_rvm_ndd_encoding() {
+        use crate::inst::xorq_rvm;
+
+        let inst = xorq_rvm::<FuzzRegs>::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));
+        let assembled = assemble(&inst.into());
+        assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1831CA");
+    }
+
+    #[test]
+    fn apx_xorq_rvm_ndd_disp8_is_unscaled() {
+        use crate::inst::xorq_rvm;
+        use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};
+
+        let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {
+            base: FuzzReg::new(7),
+            simm32: AmodeOffsetPlusKnownOffset {
+                simm32: AmodeOffset::new(0x50),
+                offset: None,
+            },
+            trap: None,
+        });
+        let inst = xorq_rvm::<FuzzRegs>::new(FuzzReg::new(7), FuzzReg::new(7), mem);
+        let assembled = assemble(&inst.into());
+        assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418317F50");
+    }
 }

cranelift/codegen/src/isa/x64/lower.isle

@@ -306,6 +306,18 @@
 
 ;; `i64` and smaller.
 
+;; APX: when the `has_apx` feature is enabled, use the NDD (new-data-
+;; destination) form of `sub` for 64-bit subtracts. Unlike the legacy two-
+;; operand `sub`, the NDD form writes its result to a fresh register encoded in
+;; the EVEX `vvvv` field, so the register allocator does not need to insert a
+;; move to preserve either input.
+;;
+;; Note that the NDD form computes `dest = rm - reg`, so the minuend goes in
+;; ModRM.rm (the second argument here) and the subtrahend in ModRM.reg.
+(rule isub_apx_ndd 1 (lower (isub $I64 x y))
+      (if-let true (use_apx))
+      (x64_subq_rvm (put_in_gpr y) x))
+
 ;; Sub two registers.
 (rule -3 (lower (isub (fits_in_64 ty) x y))
       (x64_sub ty x y))
@@ -370,6 +382,12 @@
 
 ;; `{i,b}64` and smaller.
 
+;; APX: when the `has_apx` feature is enabled, use the NDD form of `and` for
+;; 64-bit integer values.
+(rule band_apx_ndd 3 (lower (band $I64 x y))
+      (if-let true (use_apx))
+      (x64_andq_rvm x y))
+
 ;; And two registers.
 (rule 0 (lower (band ty x y))
       (if (ty_int_ref_scalar_64 ty))
@@ -499,6 +517,12 @@
 
 ;; `{i,b}64` and smaller.
 
+;; APX: when the `has_apx` feature is enabled, use the NDD form of `or` for
+;; 64-bit integer values.
+(rule bor_apx_ndd 3 (lower (bor $I64 x y))
+      (if-let true (use_apx))
+      (x64_orq_rvm x y))
+
 ;; Or two registers.
 (rule 0 (lower (bor ty x y))
       (if (ty_int_ref_scalar_64 ty))
@@ -572,6 +596,12 @@
 
 ;; `{i,b}64` and smaller.
 
+;; APX: when the `has_apx` feature is enabled, use the NDD form of `xor` for
+;; 64-bit integer values.
+(rule bxor_apx_ndd 3 (lower (bxor $I64 x y))
+      (if-let true (use_apx))
+      (x64_xorq_rvm x y))
+
 ;; Xor two registers.
 (rule 0 (lower (bxor ty x y))
       (if (ty_int_ref_scalar_64 ty))

cranelift/filetests/filetests/isa/x64/apx-arith.clif

@@ -0,0 +1,72 @@
+test compile precise-output
+target x86_64 has_apx
+
+function %and_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = band v0, v1
+    return v2
+}
+
+; VCode:
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block0:
+;   andq %rdi, %rsi, %rax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   .byte 0x62, 0xf4, 0xfc, 0x18, 0x21, 0xfe, 0x48, 0x89
+;   .byte 0xec, 0x5d, 0xc3
+
+function %or_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = bor v0, v1
+    return v2
+}
+
+; VCode:
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block0:
+;   orq %rdi, %rsi, %rax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   .byte 0x62, 0xf4, 0xfc, 0x18, 0x09, 0xfe, 0x48, 0x89
+;   .byte 0xec, 0x5d, 0xc3
+
+function %xor_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = bxor v0, v1
+    return v2
+}
+
+; VCode:
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block0:
+;   xorq %rdi, %rsi, %rax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   .byte 0x62, 0xf4, 0xfc, 0x18, 0x31, 0xfe, 0x48, 0x89
+;   .byte 0xec, 0x5d, 0xc3
+

cranelift/filetests/filetests/isa/x64/apx-sub.clif

@@ -0,0 +1,26 @@
+test compile precise-output
+target x86_64 has_apx
+
+function %sub_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = isub v0, v1
+    return v2
+}
+
+; VCode:
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block0:
+;   subq %rsi, %rdi, %rax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   .byte 0x62, 0xf4, 0xfc, 0x18, 0x29, 0xf7, 0x48, 0x89
+;   .byte 0xec, 0x5d, 0xc3
+