Repository navigation
[FIX] Run NKI kernels written like the official samples in the NKI interpreter #539
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,140 @@ | ||
| """Kernels written in the style of the official NKI samples, run by TileLens and by | ||
| NKI's own CPU simulator (``nki.simulate_kernel``); both must give the same outputs.""" | ||
|
|
||
| import math | ||
|
|
||
| import numpy as np | ||
| import pytest | ||
|
|
||
| try: | ||
| from neuronxcc import nki | ||
| import neuronxcc.nki.language as nl | ||
| except ModuleNotFoundError: | ||
| pytest.skip( | ||
| "NeuronX dependencies are missing. Install tilelens[nki] to run these tests.", | ||
| allow_module_level=True, | ||
| ) | ||
|
|
||
| import tilelens | ||
| from tilelens.clients import Tracer | ||
|
|
||
| pytestmark = pytest.mark.nki # only run at "pytest -m nki" | ||
|
|
||
|
|
||
| def softmax_kernel(a_tensor): | ||
| # The output is allocated in shared HBM and returned; rows are tiled by 128 and | ||
| # the tail is masked with a [128, 1] mask that broadcasts over the tile. | ||
| out_tensor = nl.ndarray(a_tensor.shape, dtype=a_tensor.dtype, buffer=nl.shared_hbm) | ||
| ix = nl.arange(128)[:, None] | ||
| iy = nl.arange(a_tensor.shape[1])[None, :] | ||
| num_rows = a_tensor.shape[0] | ||
| for i in nl.affine_range(math.ceil(num_rows / 128)): | ||
| row_mask = i * 128 + ix < num_rows | ||
| a_tile = nl.load(a_tensor[i * 128 + ix, iy], mask=row_mask) | ||
| max_vals = nl.max(a_tile, axis=[1], keepdims=True, mask=row_mask) | ||
| shifted = nl.subtract(a_tile, max_vals, mask=row_mask) | ||
| numerator = nl.exp(shifted) | ||
| denominator = nl.sum(numerator, axis=[1]) # [128, 1], as in NKI | ||
| nl.store( | ||
| out_tensor[i * 128 + ix, iy], value=numerator / denominator, mask=row_mask | ||
| ) | ||
| return out_tensor | ||
|
|
||
|
|
||
| def copy_kernel(in_tensor): | ||
| # Whole-tensor load and store, and a load nested in a store. | ||
| out_tensor = nl.ndarray( | ||
| in_tensor.shape, dtype=in_tensor.dtype, buffer=nl.shared_hbm | ||
| ) | ||
| tmp = nl.ndarray(in_tensor.shape, dtype=in_tensor.dtype, buffer=nl.shared_hbm) | ||
| nl.store(tmp, value=nl.load(in_tensor)) | ||
| ix = nl.arange(in_tensor.shape[0])[:, None] | ||
| iy = nl.arange(in_tensor.shape[1])[None, :] | ||
| nl.store(out_tensor[ix, iy], value=nl.copy(nl.load(tmp[ix, iy]))) | ||
| return out_tensor | ||
|
|
||
|
|
||
| def spmd_add_kernel(a_input, b_input): | ||
| # One [128, 256] tile per program on a 2-D grid. | ||
| c_output = nl.ndarray(a_input.shape, dtype=a_input.dtype, buffer=nl.shared_hbm) | ||
| ix = nl.program_id(0) * 128 + nl.arange(128)[:, None] | ||
| iy = nl.program_id(1) * 256 + nl.arange(256)[None, :] | ||
| nl.store( | ||
| c_output[ix, iy], | ||
| value=nl.add(nl.load(a_input[ix, iy]), nl.load(b_input[ix, iy])), | ||
| ) | ||
| return c_output | ||
|
|
||
|
|
||
| def loops_and_casts_kernel(x): | ||
| # static_range / sequential_range loops, par_dim in a shape, static_cast, | ||
| # maximum / minimum / divide, and a reduction over two free axes. | ||
| out = nl.ndarray((x.shape[0], 1), dtype=nl.float32, buffer=nl.shared_hbm) | ||
| acc = nl.zeros((nl.par_dim(x.shape[0]), 1), dtype=nl.float32) | ||
| for k in nl.static_range(2): | ||
| t = nl.load(x[:, k]) | ||
| clipped = nl.minimum(nl.maximum(t, -1.0), 1.0) | ||
| acc[...] = nl.add(acc, nl.sum(clipped, axis=[1, 2])) | ||
| for _ in nl.sequential_range(1): | ||
| acc[...] = nl.divide(acc, 2.0) | ||
| nl.store(out, value=nl.static_cast(acc, nl.float32)) | ||
| return out | ||
|
|
||
|
|
||
| def _tilelens(kernel, grid, *args): | ||
| return tilelens.trace(client=Tracer(), frontend="nki")(kernel)[grid](*args) | ||
|
|
||
|
|
||
| def _reference(kernel, grid, *args): | ||
| jitted = nki.jit(kernel) | ||
| return nki.simulate_kernel(jitted[grid] if grid != (1,) else jitted, *args) | ||
|
|
||
|
|
||
| rng = np.random.default_rng(0) | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "kernel, grid, args", | ||
| [ | ||
| (softmax_kernel, (1,), (rng.standard_normal((200, 64)).astype(np.float32),)), | ||
| (copy_kernel, (1,), (rng.standard_normal((16, 32)).astype(np.float32),)), | ||
| ( | ||
| spmd_add_kernel, | ||
| (2, 2), | ||
| ( | ||
| rng.standard_normal((256, 512)).astype(np.float32), | ||
| rng.standard_normal((256, 512)).astype(np.float32), | ||
| ), | ||
| ), | ||
| ( | ||
| loops_and_casts_kernel, | ||
| (1,), | ||
| (rng.standard_normal((8, 2, 4, 3)).astype(np.float32),), | ||
| ), | ||
| ], | ||
| ids=["softmax", "copy", "spmd_add", "loops_and_casts"], | ||
| ) | ||
| def test_kernel_idioms_match_nki_simulator(kernel, grid, args): | ||
| tilelens.clear() | ||
| got = _tilelens(kernel, grid, *args) | ||
| want = _reference(kernel, grid, *args) | ||
| assert isinstance(got, np.ndarray) | ||
| np.testing.assert_allclose(got, want, rtol=1e-5, atol=1e-6) | ||
|
|
||
|
|
||
| class _StopAfterFirstProgram(Tracer): | ||
| def post_run_callback(self, fn): | ||
| return False | ||
|
|
||
|
|
||
| def test_outputs_are_returned_when_a_client_stops_the_grid_early(): | ||
| tilelens.clear() | ||
| a = rng.standard_normal((256, 512)).astype(np.float32) | ||
| b = rng.standard_normal((256, 512)).astype(np.float32) | ||
| traced = tilelens.trace(client=_StopAfterFirstProgram(), frontend="nki")( | ||
| spmd_add_kernel | ||
| ) | ||
| got = traced[(2, 2)](a, b) | ||
| assert isinstance(got, np.ndarray) | ||
| # Only program (0, 0) ran; its tile holds the sum. | ||
| np.testing.assert_allclose(got[:128, :256], a[:128, :256] + b[:128, :256]) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -8,35 +8,19 @@ class NKITransformer(ast.NodeTransformer): | |
|
|
||
| Transformations: | ||
| - `nl.<func>(x[...], ...)` -> `nl.<func>(x, slice_obj, ...)` | ||
| where <func> is one of {load, store, load_transpose2d} | ||
| where <func> is one of {load, store, load_transpose2d}, wherever the call | ||
| appears (e.g. nested in another call, as in `nl.store(o[i], nl.load(x[i]))`) | ||
| - strips out kernel decorators | ||
| """ | ||
|
|
||
| def visit_Expr(self, node: ast.Expr) -> ast.AST | list[ast.AST]: | ||
| if not isinstance(node.value, ast.Call): | ||
| return self.generic_visit(node) | ||
|
|
||
| new_call = self._rewrite_nl_slice_call(node.value) | ||
| if new_call is None: | ||
| return self.generic_visit(node) | ||
| return ast.Expr(value=new_call) | ||
|
|
||
| def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST: | ||
| node.decorator_list = [] | ||
| return self.generic_visit(node) | ||
|
|
||
| def visit_Assign(self, node: ast.Assign) -> ast.AST: | ||
| if not isinstance(node.value, ast.Call): | ||
| return self.generic_visit(node) | ||
|
|
||
| new_call = self._rewrite_nl_slice_call(node.value) | ||
| if new_call is None: | ||
| return self.generic_visit(node) | ||
| return ast.Assign( | ||
| targets=node.targets, | ||
| value=new_call, | ||
| type_comment=getattr(node, "type_comment", None), | ||
| ) | ||
| def visit_Call(self, node: ast.Call) -> ast.AST: | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The rewrite only looks at a positional first argument.
|
||
| node = cast(ast.Call, self.generic_visit(node)) | ||
| new_call = self._rewrite_nl_slice_call(node) | ||
| return node if new_call is None else new_call | ||
|
|
||
| def _rewrite_nl_slice_call(self, call_node: ast.Call) -> ast.Call | None: | ||
| if not ( | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -67,6 +67,30 @@ def _get_valid_indices( | |
| return valid_mask | ||
|
|
||
|
|
||
| def tile_shape(ndarray: np.ndarray, keys) -> tuple: | ||
| """Shape of ``ndarray[keys]``, also when some integer-array keys are out of bounds.""" | ||
| if not isinstance(keys, tuple): | ||
| keys = (keys,) | ||
| clipped = [] | ||
| for dim, key in enumerate(keys): | ||
| if ( | ||
| isinstance(key, np.ndarray) | ||
| and key.dtype.kind in "iu" | ||
| and dim < ndarray.ndim | ||
| ): | ||
| key = np.clip(key, 0, max(ndarray.shape[dim] - 1, 0)) | ||
| clipped.append(key) | ||
| return ndarray[tuple(clipped)].shape | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. tile_shape does not handle out-of-bounds scalar integer keys. It also lines keys up with array dims by key position, which is wrong once a A masked
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. tile_shape gets the shape by actually fancy-indexing the array, which copies the whole tile. It runs 2 to 3 times per load or store. Builder.store calls tile_shape and then tile_mask, which calls tile_shape again. The tracer's _nki_offsets_and_mask does it again on the offsets array. That is several extra full-tile gathers on every load/store in the hot per-program loop. Compute the shape from key shapes and slice lengths (np.broadcast_shapes for array keys), and pass the computed shape into tile_mask. |
||
|
|
||
|
|
||
| def tile_mask(ndarray: np.ndarray, keys, mask) -> np.ndarray | None: | ||
| """``mask`` broadcast to the shape of ``ndarray[keys]``, as NKI broadcasts masks; | ||
| None stays None.""" | ||
| if mask is None: | ||
| return None | ||
| return np.broadcast_to(np.asarray(mask, dtype=bool), tile_shape(ndarray, keys)) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. tile_mask/tile_shape only clip integer-array keys, so for an out-of-bounds slice key the 'tile shape' is numpy's truncated slice shape, and broadcasting the full-tile mask to it raises. Masked tail loads and stores that use slices now crash, where before this change they worked. a.shape == (100, 4), |
||
|
|
||
|
|
||
| def masked_load( | ||
| ndarray: np.ndarray, keys: tuple, mask: np.ndarray = None | ||
| ) -> np.ndarray: | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Whole-tensor access is patched by giving
keys=Nonedefaults in three places: Builder.load, Builder.store, and both adapters. Making NKITransformer always emit an explicit keys argument would fix it once.Because the default sits at the callee, the builder signature (dst, keys, value) no longer lines up with NKI's (dst, value), which causes the positional-store bug above. Keyword dst/src are also not covered. If the transformer normalized every nl.load/store/load_transpose2d call to (tensor, keys, ...), inserting full slices or handling keyword dst/src, the special-case defaults would not be needed.