sunnypilot v2026.002.000

version: sunnypilot v2026.002.000 (staging)
date: 2026-07-21T07:44:02
master commit: 2c334ede443d7391d27575af8c854a095ba702a8
This commit is contained in:
github-actions[bot] committed 2026-07-21 07:44:02 +00:00
commit de77c89cf6
3995 files changed
+1260688

No files matched your search

@@ -0,0 +1,96 @@
from __future__ import annotations
import functools, pathlib
from tinygrad import Tensor, dtypes
from tinygrad.uop.ops import UOp, Ops, KernelInfo
from tinygrad.renderer import Estimates
from tinygrad.runtime.support.compiler_amd import HIPCCCompiler
THREADS_PER_WG = 256
@functools.cache
def _custom_fused_ce_loss_fwd(loss_out:UOp, max_out:UOp, lse_out:UOp, logits:UOp, targets:UOp,
dname:str, vocab:int, rows:int, label_smoothing:float) -> UOp:
threads, workgroups = UOp.special(THREADS_PER_WG, "lidx0"), UOp.special(rows, "gidx0")
mem = rows * vocab * 2 + rows * 12 + rows * 4
sink = UOp.sink(loss_out.base, max_out.base, lse_out.base, logits.base, targets.base,
threads, workgroups,
arg=KernelInfo(f"fused_ce_loss_fwd", estimates=Estimates(ops=6*rows*vocab, mem=mem)))
src = (pathlib.Path(__file__).parent/"fused_ce_loss.cpp").read_text()
defines = [f"-DVOCAB={vocab}", f"-DTHREADS_PER_WG={THREADS_PER_WG}",
f"-DLABEL_SMOOTHING={label_smoothing}f"]
lib = HIPCCCompiler("gfx950", ["-std=c++20", "-ffast-math", *defines]).compile_cached(src)
return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg=dname), UOp(Ops.LINEAR, src=(*sink.src, sink)),
UOp(Ops.SOURCE, arg=src), UOp(Ops.BINARY, arg=lib)))
@functools.cache
def _custom_fused_ce_loss_bwd(d_logits:UOp, logits:UOp, lse:UOp, targets:UOp, scale:UOp,
dname:str, vocab:int, rows:int, label_smoothing:float) -> UOp:
threads, workgroups = UOp.special(THREADS_PER_WG, "lidx0"), UOp.special(rows, "gidx0")
mem = rows * vocab * 4 + rows * 8 + 4
sink = UOp.sink(d_logits.base, logits.base, lse.base, targets.base, scale.base,
threads, workgroups,
arg=KernelInfo(f"fused_ce_loss_bwd", estimates=Estimates(ops=4*rows*vocab, mem=mem)))
src = (pathlib.Path(__file__).parent/"fused_ce_loss_bwd.cpp").read_text()
defines = [f"-DVOCAB={vocab}", f"-DTHREADS_PER_WG={THREADS_PER_WG}",
f"-DLABEL_SMOOTHING={label_smoothing}f"]
lib = HIPCCCompiler("gfx950", ["-std=c++20", "-ffast-math", *defines]).compile_cached(src)
return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg=dname), UOp(Ops.LINEAR, src=(*sink.src, sink)),
UOp(Ops.SOURCE, arg=src), UOp(Ops.BINARY, arg=lib)))
def _fused_ce_loss_bwd(gradient:UOp, kernel:UOp, label_smoothing:float):
# NOTE: forward inputs are (loss_out, max_out, lse_out, logits, targets)
# gradient is the upstream grad w.r.t. per-row loss (shape: (rows,) fp32)
_, _, lse_u, logits_u, targets_u = kernel.src[1:]
device = logits_u.device
rows, VOCAB = logits_u.shape # (rows, VOCAB) after reshape
if isinstance(device, tuple):
axis = logits_u.axis
ndev = len(device)
d_logits = Tensor(Tensor.invalids(rows // ndev, VOCAB, dtype=dtypes.bfloat16, device=device).uop.multi(axis), device=device)
dname = device[0].split(":")[0]
rows_per_dev = rows // ndev
else:
d_logits = Tensor.invalids(rows, VOCAB, dtype=dtypes.bfloat16, device=device)
dname = device.split(":")[0] if isinstance(device, str) else device
rows_per_dev = rows
# NOTE: .mean() backward gives same grad per row (1/N), so broadcast is safe; take scalar
scale = Tensor(gradient, device=device).float().reshape(-1)[0:1].contiguous()
logits_t = Tensor(logits_u.after(kernel), device=device)
lse_t = Tensor(lse_u.after(kernel), device=device)
targets_t = Tensor(targets_u, device=device)
fxn = functools.partial(_custom_fused_ce_loss_bwd, dname=dname, vocab=VOCAB, rows=rows_per_dev, label_smoothing=label_smoothing)
d_logits, *_ = Tensor.custom_kernel(d_logits, logits_t, lse_t, targets_t, scale, fxn=fxn)
return (None, None, None, d_logits.uop, None)
def fused_ce_loss(logits:Tensor, targets:Tensor, label_smoothing:float=0.1) -> Tensor:
# NOTE: fused sparse_categorical_crossentropy with label smoothing, returns mean loss scalar
assert logits.dtype == dtypes.bfloat16, f"expected bf16, got {logits.dtype}"
assert logits.ndim == 3, f"expected (MBS, SEQ, VOCAB), got {logits.shape}"
MBS, SEQ, VOCAB = logits.shape
rows = MBS * SEQ
if isinstance(logits.device, tuple):
axis = logits.uop.axis
assert axis in (0, 1), f"unsupported sharding axis={axis} for CE loss"
ndev = len(logits.device)
loss_out = Tensor(Tensor.invalids(rows // ndev, dtype=dtypes.float32, device=logits.device).uop.multi(0),
device=logits.device)
max_out = Tensor(Tensor.invalids(rows // ndev, dtype=dtypes.float32, device=logits.device).uop.multi(0),
device=logits.device)
lse_out = Tensor(Tensor.invalids(rows // ndev, dtype=dtypes.float32, device=logits.device).uop.multi(0),
device=logits.device)
dname = logits.device[0].split(":")[0]
rows_per_dev = rows // ndev
else:
loss_out = Tensor.invalids(rows, dtype=dtypes.float32, device=logits.device)
max_out = Tensor.invalids(rows, dtype=dtypes.float32, device=logits.device)
lse_out = Tensor.invalids(rows, dtype=dtypes.float32, device=logits.device)
dname = logits.device.split(":")[0] if isinstance(logits.device, str) else logits.device
rows_per_dev = rows
logits_flat = logits.reshape(rows, VOCAB)
targets_flat = targets.reshape(-1).cast(dtypes.int32)
fxn = functools.partial(_custom_fused_ce_loss_fwd, dname=dname, vocab=VOCAB, rows=rows_per_dev,
label_smoothing=label_smoothing)
loss_out, max_out, lse_out, *_ = Tensor.custom_kernel(
loss_out, max_out, lse_out, logits_flat, targets_flat,
fxn=fxn, grad_fxn=functools.partial(_fused_ce_loss_bwd, label_smoothing=label_smoothing))
return loss_out.mean()
@@ -0,0 +1,104 @@
#include <hip/hip_runtime.h>
#include <hip/hip_bf16.h>
// Fused forward sparse-CE with label smoothing.
// SINGLE-PASS online softmax + vectorized 8-wide bf16 loads for HBM coalescing.
#ifndef VOCAB
#define VOCAB 128256
#endif
#ifndef THREADS_PER_WG
#define THREADS_PER_WG 256
#endif
#ifndef LABEL_SMOOTHING
#define LABEL_SMOOTHING 0.1f
#endif
constexpr int VEC = 8;
extern "C" __global__ __launch_bounds__(THREADS_PER_WG) void
fused_ce_loss_fwd(
float* __restrict__ loss_out, // out: fp32, ROWS
float* __restrict__ max_out, // out: fp32, ROWS
float* __restrict__ lse_out, // out: fp32, ROWS
const __hip_bfloat16* __restrict__ logits, // in: bf16, ROWS*VOCAB
const int* __restrict__ targets) // in: int32, ROWS
{
__shared__ float sdata_m[THREADS_PER_WG];
__shared__ float sdata_s[THREADS_PER_WG];
__shared__ float sdata_sumx[THREADS_PER_WG];
__shared__ float sdata_tgt[THREADS_PER_WG];
const int tid = threadIdx.x;
const int row = blockIdx.x;
const int target = targets[row];
const __hip_bfloat16* row_logits = logits + (size_t)row * VOCAB;
float m = -INFINITY;
float s = 0.0f;
float sum_x = 0.0f;
float target_logit = 0.0f;
constexpr bool needs_sum_x = (LABEL_SMOOTHING != 0.0f);
// Vectorized stride: each iter loads 8 bf16 = 16 bytes. Warp loads 32*16 = 512 bytes (4 cache lines).
const int VOCAB_VEC = VOCAB & ~(VEC - 1); // round down to multiple of VEC
for (int i = tid * VEC; i < VOCAB_VEC; i += THREADS_PER_WG * VEC) {
float4 raw = *reinterpret_cast<const float4*>(&row_logits[i]);
const __hip_bfloat16* xi = reinterpret_cast<const __hip_bfloat16*>(&raw);
#pragma unroll
for (int k = 0; k < VEC; k++) {
const float x = static_cast<float>(xi[k]);
if constexpr (needs_sum_x) sum_x += x;
if (i + k == target) target_logit = x;
if (x > m) {
s = s * __expf(m - x) + 1.0f;
m = x;
} else {
s += __expf(x - m);
}
}
}
// tail (VOCAB not divisible by VEC):
for (int i = VOCAB_VEC + tid; i < VOCAB; i += THREADS_PER_WG) {
const float x = static_cast<float>(row_logits[i]);
if constexpr (needs_sum_x) sum_x += x;
if (i == target) target_logit = x;
if (x > m) { s = s * __expf(m - x) + 1.0f; m = x; }
else { s += __expf(x - m); }
}
sdata_m[tid] = m;
sdata_s[tid] = s;
sdata_sumx[tid] = sum_x;
sdata_tgt[tid] = target_logit;
__syncthreads();
for (int step = THREADS_PER_WG / 2; step > 0; step >>= 1) {
if (tid < step) {
const float m1 = sdata_m[tid];
const float m2 = sdata_m[tid + step];
const float s1 = sdata_s[tid];
const float s2 = sdata_s[tid + step];
const float m_new = fmaxf(m1, m2);
const float s_new = s1 * __expf(m1 - m_new) + s2 * __expf(m2 - m_new);
sdata_m[tid] = m_new;
sdata_s[tid] = s_new;
sdata_sumx[tid] += sdata_sumx[tid + step];
sdata_tgt[tid] += sdata_tgt[tid + step];
}
__syncthreads();
}
if (tid == 0) {
const float row_max = sdata_m[0];
const float row_sum_exp = sdata_s[0];
const float row_sum_x = sdata_sumx[0];
const float tgt = sdata_tgt[0];
const float row_lse = logf(row_sum_exp) + row_max;
const float mean_logits = row_sum_x / static_cast<float>(VOCAB);
const float loss = row_lse - (1.0f - LABEL_SMOOTHING) * tgt - LABEL_SMOOTHING * mean_logits;
loss_out[row] = loss;
max_out[row] = row_max;
lse_out[row] = row_lse;
}
}
@@ -0,0 +1,58 @@
#include <hip/hip_runtime.h>
#include <hip/hip_bf16.h>
// Vectorized CE bwd: 8-wide bf16 loads + stores.
#ifndef VOCAB
#define VOCAB 128256
#endif
#ifndef THREADS_PER_WG
#define THREADS_PER_WG 256
#endif
#ifndef LABEL_SMOOTHING
#define LABEL_SMOOTHING 0.1f
#endif
constexpr int VEC = 8;
extern "C" __global__ __launch_bounds__(THREADS_PER_WG) void
fused_ce_loss_bwd(
__hip_bfloat16* __restrict__ d_logits,
const __hip_bfloat16* __restrict__ logits,
const float* __restrict__ lse,
const int* __restrict__ targets,
const float* __restrict__ scale_in)
{
const int tid = threadIdx.x;
const int row = blockIdx.x;
const int target = targets[row];
const float lse_r = lse[row];
const __hip_bfloat16* row_logits = logits + (size_t)row * VOCAB;
__hip_bfloat16* row_dlogits = d_logits + (size_t)row * VOCAB;
const float inv_vocab = 1.0f / static_cast<float>(VOCAB);
const float scale = *scale_in;
const float ls_term = LABEL_SMOOTHING * inv_vocab;
const int VOCAB_VEC = VOCAB & ~(VEC - 1);
for (int i = tid * VEC; i < VOCAB_VEC; i += THREADS_PER_WG * VEC) {
float4 raw = *reinterpret_cast<const float4*>(&row_logits[i]);
const __hip_bfloat16* xi = reinterpret_cast<const __hip_bfloat16*>(&raw);
__hip_bfloat16 out[VEC];
#pragma unroll
for (int k = 0; k < VEC; k++) {
const float x = static_cast<float>(xi[k]);
float g = __expf(x - lse_r);
if (i + k == target) g -= (1.0f - LABEL_SMOOTHING);
g -= ls_term;
out[k] = static_cast<__hip_bfloat16>(g * scale);
}
*reinterpret_cast<float4*>(&row_dlogits[i]) = *reinterpret_cast<float4*>(out);
}
for (int i = VOCAB_VEC + tid; i < VOCAB; i += THREADS_PER_WG) {
const float x = static_cast<float>(row_logits[i]);
float g = __expf(x - lse_r);
if (i == target) g -= (1.0f - LABEL_SMOOTHING);
g -= ls_term;
row_dlogits[i] = static_cast<__hip_bfloat16>(g * scale);
}
}