One-launch LayerNorm: fp32 statistics and affine, output in the input dtype.
Functions:
-
layer_norm
–
LayerNorm over the last dim of a bf16 [T, H] tensor.
layer_norm(x, weight, bias, eps)
LayerNorm over the last dim of a bf16 [T, H] tensor.
Source code in vllm/models/mistral/amd/ops/layernorm.py
| def layer_norm(
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor, eps: float
) -> torch.Tensor:
"""LayerNorm over the last dim of a bf16 `[T, H]` tensor."""
assert x.ndim == 2 and x.stride(-1) == 1 and x.dtype == torch.bfloat16
num_tokens, hidden_size = x.shape
assert weight.shape == bias.shape == (hidden_size,)
out = torch.empty_like(x)
if num_tokens == 0:
return out
block = triton.next_power_of_2(hidden_size)
_layer_norm_kernel[(num_tokens,)](
x,
x.stride(0),
weight,
bias,
eps,
out,
out.stride(0),
hidden_size=hidden_size,
block=block,
num_warps=min(max(block // 256, 1), 8),
)
return out
|