sunnypilot v2026.002.000
version: sunnypilot v2026.002.000 (staging) date: 2026-07-21T07:44:02 master commit: 2c334ede443d7391d27575af8c854a095ba702a8
This commit is contained in:
commit
de77c89cf6
3995 files changed
+1260688
No files matched your search
@@ -0,0 +1,55 @@
|
||||
from __future__ import annotations
|
||||
import functools, pathlib
|
||||
from tinygrad import Tensor, dtypes
|
||||
from tinygrad.uop.ops import UOp, Ops, KernelInfo
|
||||
from tinygrad.renderer import Estimates
|
||||
from extra.llama_kernels import THREADS_PER_WG, dname_of, compile_hip
|
||||
|
||||
ELEMS_PER_THREAD = 8 # vectorized 16-byte load (uint4 = 8 bf16)
|
||||
|
||||
def _build_src(n_chunks:int) -> str:
|
||||
template = (pathlib.Path(__file__).parent/"fused_pad_grad_accum.cpp").read_text()
|
||||
params = "".join(f",\n const __hip_bfloat16* __restrict__ chunk{i}" for i in range(n_chunks))
|
||||
dispatch = "\n ".join(f"case {i}: chunk_ptr = chunk{i}; break;" for i in range(n_chunks))
|
||||
return (template.replace("__FUSED_PAD_GRAD_ACCUM_PARAMS", params)
|
||||
.replace("__FUSED_PAD_GRAD_ACCUM_DISPATCH", dispatch))
|
||||
|
||||
@functools.cache
|
||||
def _custom_fused_pad_grad_accum(grad_buf:UOp, *chunk_uops, dname:str, n_chunks:int, chunk_size:int) -> UOp:
|
||||
total = n_chunks * chunk_size
|
||||
elems_per_block = THREADS_PER_WG * ELEMS_PER_THREAD
|
||||
assert chunk_size % elems_per_block == 0, f"chunk_size {chunk_size} must be multiple of {elems_per_block}"
|
||||
num_wg = total // elems_per_block
|
||||
threads, workgroups = UOp.special(THREADS_PER_WG, "lidx0"), UOp.special(num_wg, "gidx0")
|
||||
mem = total * 2 * 3
|
||||
sink = UOp.sink(grad_buf.base, *(c.base for c in chunk_uops), threads, workgroups,
|
||||
arg=KernelInfo(f"fused_pad_grad_accum_n{n_chunks}_c{chunk_size}",
|
||||
estimates=Estimates(ops=2*total, mem=mem)))
|
||||
src = _build_src(n_chunks)
|
||||
defines = [f"-DCHUNK_SIZE={chunk_size}", f"-DTHREADS_PER_WG={THREADS_PER_WG}", f"-DELEMS_PER_THREAD={ELEMS_PER_THREAD}"]
|
||||
return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg=dname), UOp(Ops.LINEAR, src=(*sink.src, sink)),
|
||||
UOp(Ops.SOURCE, arg=src), UOp(Ops.BINARY, arg=compile_hip(src, defines))))
|
||||
|
||||
def can_fused_pad_grad_accum(grad_buf:Tensor, chunks:list[Tensor]) -> bool:
|
||||
if not chunks or grad_buf.dtype != dtypes.bfloat16: return False
|
||||
if any(c.dtype != dtypes.bfloat16 for c in chunks): return False
|
||||
chunk_shape = chunks[0].shape
|
||||
if any(c.shape != chunk_shape for c in chunks): return False
|
||||
chunk_size, total = 1, 1
|
||||
for d in chunk_shape: chunk_size *= d
|
||||
for d in grad_buf.shape: total *= d
|
||||
return total == len(chunks) * chunk_size and chunk_size % (THREADS_PER_WG * ELEMS_PER_THREAD) == 0
|
||||
|
||||
def fused_pad_grad_accum(grad_buf:Tensor, chunks:list[Tensor]) -> Tensor:
|
||||
# NOTE: grad_buf += cat(*chunks, dim=0) in one HBM pass (in-place add). Returns new grad_buf Tensor.
|
||||
# Requires uniform chunk shapes and chunk_size % (THREADS_PER_WG*ELEMS_PER_THREAD) == 0.
|
||||
assert chunks and grad_buf.dtype == dtypes.bfloat16
|
||||
for c in chunks: assert c.dtype == dtypes.bfloat16, f"chunk dtype must be bf16, got {c.dtype}"
|
||||
chunk_size, total = 1, 1
|
||||
for d in chunks[0].shape: chunk_size *= d
|
||||
for d in grad_buf.shape: total *= d
|
||||
assert total == len(chunks) * chunk_size, f"grad_buf size {total} != n_chunks {len(chunks)} * chunk_size {chunk_size}"
|
||||
fxn = functools.partial(_custom_fused_pad_grad_accum, dname=dname_of(grad_buf.device),
|
||||
n_chunks=len(chunks), chunk_size=chunk_size)
|
||||
out, *_ = Tensor.custom_kernel(grad_buf, *chunks, fxn=fxn)
|
||||
return out
|
||||
@@ -0,0 +1,63 @@
|
||||
// Fused custom kernel: grad_buf += cat(*chunks, dim=0) in one HBM pass.
|
||||
//
|
||||
// Template source — chunk parameter list and switch dispatch are filled by codegen
|
||||
// in cast_amax.py:_build_fused_pad_grad_accum_src to support arbitrary N.
|
||||
//
|
||||
// Defines required at compile time:
|
||||
// CHUNK_SIZE elements per chunk (must be multiple of THREADS_PER_WG * ELEMS_PER_THREAD)
|
||||
// THREADS_PER_WG
|
||||
// ELEMS_PER_THREAD (8 = one uint4 per thread = 16-byte vectorized load)
|
||||
//
|
||||
// Layout: one block-per-(slice-of-chunk) — blockIdx.x / BLOCKS_PER_CHUNK selects the chunk.
|
||||
// All threads in a block read the same chunk → switch is uniform → no warp divergence.
|
||||
|
||||
#include <hip/hip_runtime.h>
|
||||
#include <hip/hip_bf16.h>
|
||||
|
||||
#ifndef THREADS_PER_WG
|
||||
#define THREADS_PER_WG 256
|
||||
#endif
|
||||
#ifndef ELEMS_PER_THREAD
|
||||
#define ELEMS_PER_THREAD 8
|
||||
#endif
|
||||
|
||||
#define ELEMS_PER_BLOCK (THREADS_PER_WG * ELEMS_PER_THREAD)
|
||||
#define BLOCKS_PER_CHUNK (CHUNK_SIZE / ELEMS_PER_BLOCK)
|
||||
|
||||
extern "C" __attribute__((global))
|
||||
__attribute__((amdgpu_flat_work_group_size(1, THREADS_PER_WG)))
|
||||
void fused_pad_grad_accum(
|
||||
__hip_bfloat16* __restrict__ grad_buf
|
||||
__FUSED_PAD_GRAD_ACCUM_PARAMS
|
||||
) {
|
||||
const int bid = blockIdx.x;
|
||||
const int chunk_idx = bid / BLOCKS_PER_CHUNK;
|
||||
const int block_in_chunk = bid - chunk_idx * BLOCKS_PER_CHUNK;
|
||||
const int tid = threadIdx.x;
|
||||
|
||||
const __hip_bfloat16* chunk_ptr;
|
||||
switch (chunk_idx) {
|
||||
__FUSED_PAD_GRAD_ACCUM_DISPATCH
|
||||
default: chunk_ptr = (const __hip_bfloat16*)0; break; // unreachable
|
||||
}
|
||||
|
||||
// int64 for global_offset: at 32 chunks × 117M elements = 3.6B, int32 overflows → MEMVIOL.
|
||||
const int local_offset = block_in_chunk * ELEMS_PER_BLOCK + tid * ELEMS_PER_THREAD;
|
||||
const long long global_offset = (long long)chunk_idx * (long long)CHUNK_SIZE + (long long)local_offset;
|
||||
|
||||
// Vectorized 16-byte load (uint4 = 8 bf16). Requires CHUNK_SIZE % 8 == 0 and 16-byte alignment.
|
||||
const uint4 chunk_v = *reinterpret_cast<const uint4*>(&chunk_ptr[local_offset]);
|
||||
const uint4 grad_v = *reinterpret_cast<const uint4*>(&grad_buf[global_offset]);
|
||||
uint4 out_v;
|
||||
|
||||
const __hip_bfloat16* chunk_bf = reinterpret_cast<const __hip_bfloat16*>(&chunk_v);
|
||||
const __hip_bfloat16* grad_bf = reinterpret_cast<const __hip_bfloat16*>(&grad_v);
|
||||
__hip_bfloat16* out_bf = reinterpret_cast<__hip_bfloat16*>(&out_v);
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < ELEMS_PER_THREAD; i++) {
|
||||
out_bf[i] = (__hip_bfloat16)((float)grad_bf[i] + (float)chunk_bf[i]);
|
||||
}
|
||||
|
||||
*reinterpret_cast<uint4*>(&grad_buf[global_offset]) = out_v;
|
||||
}
|
||||
Reference in new issue
Block a user