sunnypilot v2026.002.000
version: sunnypilot v2026.002.000 (staging) date: 2026-07-21T07:44:02 master commit: 2c334ede443d7391d27575af8c854a095ba702a8
This commit is contained in:
commit
de77c89cf6
3995 files changed
+1260688
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,131 @@
|
||||
from __future__ import annotations
|
||||
import time
|
||||
from typing import cast
|
||||
from tinygrad.device import Buffer, BufferSpec, Compiled, Device, MultiBuffer
|
||||
from tinygrad.dtype import dtypes
|
||||
from tinygrad.engine.jit import GraphRunner
|
||||
from tinygrad.engine.realize import get_call_outs_ins, get_runtime
|
||||
from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, graph_rewrite
|
||||
from extra.hcq2.hcq2 import HCQ2Compiled, HCQ2DeviceCtx, HCQ2LowerCtx, prep_runtime, pm_lower_kernargs, pm_lower_ops
|
||||
from extra.hcq2.hcq2 import pm_split_into_queues, pm_add_barriers, pm_add_signals, build_host_program
|
||||
|
||||
# **************** insert deps ****************
|
||||
|
||||
def insert_deps(ctx:HCQ2Graph, linear:UOp) -> UOp:
|
||||
src = []
|
||||
for j, call in enumerate(linear.src):
|
||||
call = call.replace(tag=j)
|
||||
_, _, bufs, _ = ctx.calls[j]
|
||||
outs, ins = get_call_outs_ins(call)
|
||||
deps = ctx._access_resources([bufs[i] for i in outs + ins], list(range(len(outs))), call)
|
||||
src.append(UOp(Ops.AFTER, call.dtype, (call, *deps), tag=call.tag))
|
||||
return linear.replace(src=tuple(src))
|
||||
pm_insert_deps = PatternMatcher([(UPat(Ops.LINEAR, name="linear"), insert_deps)])
|
||||
|
||||
def replace_params(ctx:HCQ2Graph, call:UOp) -> UOp|None:
|
||||
if not any(x.op is Ops.PARAM for x in call.src[1:]): return None
|
||||
return call.replace(src=tuple(ctx.input_addrs_uop[x.arg] if x.op is Ops.PARAM else x for x in call.src))
|
||||
pm_replace_params = PatternMatcher([(UPat(Ops.CALL, name="call", allow_any_len=True), replace_params)])
|
||||
|
||||
# **************** graph-only passes ****************
|
||||
|
||||
def alloc_queue_sig(ctx:HCQ2Graph, q:UOp) -> None:
|
||||
if q.arg in ctx.queue_sigs: return None
|
||||
buf = Buffer(q.arg[0], 0x100, dtypes.uint8, options=BufferSpec(host=True, uncached=True, cpu_access=True), preallocate=True)
|
||||
ctx.queue_sig_bufs.append(buf)
|
||||
ctx.queue_sigs[q.arg] = UOp.from_buffer(buf, q.arg[0])
|
||||
return None
|
||||
pm_alloc_queue_sigs = PatternMatcher([(UPat(Ops.LINEAR, src=UPat({Ops.PROGRAM, Ops.COPY}), name="q"), alloc_queue_sig)])
|
||||
|
||||
def lower_queue_deps(ctx:HCQ2Graph, after:UOp) -> UOp:
|
||||
wrapper, deps, call_idx = after.src[0], after.src[1:], after.tag
|
||||
def store(q_arg, v): return ctx.queue_sigs[q_arg].store(UOp.const(dtypes.uint32, v))
|
||||
waits = tuple(UOp(Ops.WAIT, dtypes.void, (ctx.queue_sigs[dep.src[0].arg], UOp.const(dtypes.uint32, dep.tag),
|
||||
store(dep.src[0].arg, dep.tag))) for dep in deps)
|
||||
return wrapper.replace(src=tuple(q.replace(src=(*waits, *q.src, store(q.arg, call_idx))) for q in wrapper.src))
|
||||
pm_lower_queue_deps = PatternMatcher([(UPat(Ops.AFTER, src=UPat(Ops.LINEAR), name="after"), lower_queue_deps)])
|
||||
|
||||
def optimize_queue_deps(ctx:HCQ2Graph, queue:UOp) -> UOp|None:
|
||||
src, seen, pending, queue_sig = [], {}, {}, ctx.queue_sigs[queue.arg]
|
||||
for x in queue.src:
|
||||
if x.op is Ops.WAIT:
|
||||
sig, val = x.src[0], x.src[1]
|
||||
if sig is queue_sig or seen.get(sig, -1) >= val.arg: continue
|
||||
if (old:=pending.get(sig)) is None or old.src[1].arg < val.arg: pending[sig] = x
|
||||
continue
|
||||
for wait in pending.values():
|
||||
src.append(wait)
|
||||
seen[wait.src[0]] = wait.src[1].arg
|
||||
pending.clear()
|
||||
src.append(x)
|
||||
src += pending.values()
|
||||
return queue.replace(src=tuple(src)) if tuple(src) != queue.src else None
|
||||
pm_optimize_queue_deps = PatternMatcher([
|
||||
(UPat(Ops.LINEAR, src=UPat({Ops.BARRIER, Ops.WAIT, Ops.STORE, Ops.PROGRAM, Ops.COPY}), name="queue"), optimize_queue_deps),
|
||||
])
|
||||
|
||||
def drop_dead_stores(ctx:HCQ2Graph, outer:UOp) -> UOp:
|
||||
live = {u.src[2] for u in outer.toposort() if u.op is Ops.WAIT}
|
||||
return outer.replace(src=tuple(q.replace(src=tuple(x for x in q.src if x.op is not Ops.STORE or x in live)) for q in outer.src))
|
||||
pm_drop_dead_stores = PatternMatcher([(UPat(Ops.LINEAR, src=UPat(Ops.LINEAR), name="outer"), drop_dead_stores)])
|
||||
|
||||
def add_queue_sig_resets(ctx:HCQ2Graph, outer:UOp) -> UOp|None:
|
||||
if not ctx.queue_sig_bufs: return None
|
||||
resets = tuple(ctx.hcq_ctx.host_param(sig).index(UOp.const(dtypes.int, 0), ptr=True).cast(dtypes.uint64.ptr())
|
||||
.store(UOp.const(dtypes.uint64, 0)) for sig in ctx.queue_sig_bufs)
|
||||
return outer.replace(src=tuple(c.replace(src=c.src + resets) if c.op is Ops.AFTER else c.after(*resets) for c in outer.src))
|
||||
pm_add_queue_sig_resets = PatternMatcher([(UPat(Ops.LINEAR, name="outer"), add_queue_sig_resets)])
|
||||
|
||||
# **************** Graph ****************
|
||||
|
||||
class HCQ2Graph(GraphRunner):
|
||||
def __init__(self, linear:UOp, input_uops:tuple[UOp, ...]=()):
|
||||
super().__init__(linear, input_uops)
|
||||
self.dev = cast(HCQ2Compiled, Device[self.device])
|
||||
self.hcq_ctx = HCQ2LowerCtx(name="hcq_graph")
|
||||
|
||||
self.input_addrs = Buffer("CPU", max(len(input_uops), 1), dtypes.uint64, preallocate=True)
|
||||
self.input_addrs_uop = self.hcq_ctx.host_param(self.input_addrs)
|
||||
|
||||
self.linear = graph_rewrite(self.linear, pm_insert_deps, ctx=self, name="hcq: insert deps", walk=True)
|
||||
self.linear, sizes = prep_runtime(self.hcq_ctx, self.linear)
|
||||
for dev_name, sz in sizes.items():
|
||||
buf = Buffer(dev_name, sz, dtypes.uint8, options=BufferSpec(cpu_access=True), preallocate=True)
|
||||
self.hcq_ctx.devs[dev_name] = HCQ2DeviceCtx(dev_name, UOp.from_buffer(buf, dev_name), UOp.const(dtypes.uint64, buf._buf.va_addr))
|
||||
|
||||
self.linear = graph_rewrite(self.linear, pm_replace_params, ctx=self, name="hcq: replace params", walk=True)
|
||||
self.linear = graph_rewrite(self.linear, pm_lower_kernargs + pm_lower_ops, ctx=self.hcq_ctx, name="hcq: lower ops")
|
||||
|
||||
# per-queue signal state — populated as a side-effect by pm_alloc_queue_sigs walking the lowered linear.
|
||||
self.queue_sig_bufs:list[Buffer] = []
|
||||
self.queue_sigs:dict[tuple[str, str], UOp] = {}
|
||||
graph_rewrite(self.linear, pm_alloc_queue_sigs, ctx=self, name="hcq: alloc queue sigs", walk=True)
|
||||
|
||||
self.linear = graph_rewrite(self.linear, pm_lower_queue_deps, ctx=self, name="hcq: lower queue deps")
|
||||
self.linear = graph_rewrite(self.linear, pm_split_into_queues, ctx=self.hcq_ctx, name="hcq: split into queues")
|
||||
self.linear = graph_rewrite(self.linear, pm_add_barriers, ctx=self.hcq_ctx, name="hcq: add barriers", walk=True)
|
||||
self.linear = graph_rewrite(self.linear, pm_optimize_queue_deps, ctx=self, name="hcq: optimize queue deps", walk=True)
|
||||
self.linear = graph_rewrite(self.linear, pm_drop_dead_stores, ctx=self, name="hcq: drop dead stores")
|
||||
self.linear = graph_rewrite(self.linear, pm_add_signals, ctx=self.hcq_ctx, name="hcq: add signals", walk=True)
|
||||
self.linear = graph_rewrite(self.linear, self.dev.pm_lower, ctx=self.hcq_ctx, name=f"hcq: encode cmdbuf {self.dev.device}", walk=True)
|
||||
self.linear = graph_rewrite(self.linear, pm_add_queue_sig_resets, ctx=self, name="hcq: add queue sig resets", walk=True)
|
||||
self.host_call = build_host_program(self.hcq_ctx, self.linear, None, self.dev)
|
||||
|
||||
self.host_rt, self.host_globals = get_runtime("CPU", self.host_call.src[0]), self.host_call.src[0].arg.globals
|
||||
|
||||
def __call__(self, input_uops:tuple[UOp, ...], var_vals:dict[str, int], wait=False) -> float|None:
|
||||
addrs = self.input_addrs.as_memoryview(force_zero_copy=True).cast('Q')
|
||||
for i, u in enumerate(input_uops):
|
||||
buf = next(b for b in u.buffer.bufs if b.device == self.dev.device) if isinstance(u.buffer, MultiBuffer) else u.buffer
|
||||
addrs[i] = buf._buf.va_addr
|
||||
self.host_rt(*[self.hcq_ctx.inputs[i].get_buf("CPU") for i in self.host_globals], vals=self.host_call.src[0].arg.vals(var_vals), wait=True)
|
||||
if wait:
|
||||
st = time.perf_counter()
|
||||
self.dev.synchronize()
|
||||
return time.perf_counter() - st
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def supports_uop(batch_devs:list[Compiled], new_call:UOp) -> bool:
|
||||
all_devs = GraphRunner._all_devs(batch_devs, new_call)
|
||||
return new_call.src[0].op in (Ops.PROGRAM, Ops.COPY) and len(all_devs) == 1 and isinstance(all_devs[0], HCQ2Compiled)
|
||||
@@ -0,0 +1,377 @@
|
||||
from __future__ import annotations
|
||||
from typing import cast, Callable, TypeVar, Generic, Any, TYPE_CHECKING
|
||||
import struct, functools, time, collections
|
||||
from dataclasses import replace
|
||||
if TYPE_CHECKING: from tinygrad.engine.realize import ExecContext
|
||||
from tinygrad.helpers import DEV, getenv, select_first_inited, select_by_name, suppress_finalizing, mv_address, round_up, DEBUG, dedup
|
||||
from tinygrad.device import Device, Buffer, BufferSpec, Compiled, LRUAllocator
|
||||
from tinygrad.uop.ops import Ops, sint, UOp, UPat, PatternMatcher, KernelInfo, graph_rewrite, track_rewrites
|
||||
from tinygrad.dtype import dtypes
|
||||
from dataclasses import dataclass, field
|
||||
from tinygrad.runtime.support.memory import BumpAllocator
|
||||
from tinygrad.runtime.support.hcq import MMIOInterface
|
||||
from tinygrad.renderer import Renderer, Estimates
|
||||
from tinygrad.engine.realize import to_program, track_stats, get_call_arg_uops, resolve_params
|
||||
|
||||
HCQDeviceType = TypeVar('HCQDeviceType', bound='HCQ2Compiled')
|
||||
|
||||
class HCQ2Compiled(Compiled):
|
||||
"""
|
||||
A base class for devices compatible with the HCQ (Hardware Command Queue) API.
|
||||
"""
|
||||
timestamp_divider: float = 1000.0 # GPU timestamp counter ticks per microsecond; override per device
|
||||
|
||||
def __init__(self, device:str, allocator:'HCQAllocator', compilers:list[type[Renderer]], runtime,
|
||||
kernargs_size=(16 << 20), can_recover:bool=False, arch=None):
|
||||
self.device_id:int = int(device.split(":")[1]) if ":" in device else 0
|
||||
|
||||
from extra.hcq2.graph.hcq import HCQ2Graph
|
||||
super().__init__(device, allocator, compilers, lambda *a, **kw: None, HCQ2Graph, arch=arch)
|
||||
|
||||
self.kernargs_size = kernargs_size
|
||||
self.kernargs_offset_allocator:BumpAllocator = BumpAllocator(kernargs_size, wrap=True)
|
||||
|
||||
@functools.cached_property
|
||||
def kernargs_buf(self) -> Buffer:
|
||||
return Buffer(self.device, self.kernargs_size, dtypes.uint8, options=BufferSpec(cpu_access=True), preallocate=True)
|
||||
|
||||
@functools.cached_property
|
||||
def timeline_signal(self) -> Buffer:
|
||||
return Buffer(self.device, 0x100, dtypes.uint8, options=BufferSpec(host=True, uncached=True, cpu_access=True), preallocate=True)
|
||||
|
||||
@functools.cached_property
|
||||
def timestamps_buf(self) -> Buffer:
|
||||
return Buffer(self.device, 0x100, dtypes.uint8, options=BufferSpec(cpu_access=True), preallocate=True)
|
||||
|
||||
@functools.cached_property
|
||||
def timeline_value(self) -> Buffer:
|
||||
buf = Buffer("CPU", 1, dtypes.uint64, preallocate=True)
|
||||
buf.as_memoryview(force_zero_copy=True).cast('Q')[0] = 1
|
||||
return buf
|
||||
|
||||
def synchronize(self, timeout:int|None=None):
|
||||
if not hasattr(self, 'iface'): return
|
||||
sig = self.timeline_signal._buf.cpu_view().mv.cast('Q')
|
||||
tl = self.timeline_value.as_memoryview(force_zero_copy=True).cast('Q')
|
||||
st = time.perf_counter()
|
||||
while sig[0] < tl[0] - 1:
|
||||
if time.perf_counter() - st > (timeout or 3000) / 1000: self.on_device_hang()
|
||||
|
||||
def device_props(self) -> dict[str,Any]: return {} # to be overridden if needed. dict keys are backend dependent.
|
||||
|
||||
def _realloc(self, oldbuf:HCQ2Buffer|None, new_size:int, options:BufferSpec|None=None, force=False) -> tuple[HCQ2Buffer, bool]:
|
||||
if oldbuf is not None: self.allocator.free(oldbuf, oldbuf.size, options=options)
|
||||
try: buf, realloced = self.allocator.alloc(new_size, options=options), True
|
||||
except MemoryError:
|
||||
if force: raise
|
||||
buf, realloced = self.allocator.alloc(oldbuf.size if oldbuf is not None else new_size, options=options), False
|
||||
return buf, realloced
|
||||
|
||||
def count(self) -> int: return self.iface.count if hasattr(self, 'iface') else 1
|
||||
|
||||
def _select_iface(self):
|
||||
assert (v:=getenv(k:=f'{type(self).__name__[:-6].upper()}_IFACE', "")) == "", \
|
||||
f"{k}={v} is deprecated, use DEV={replace(DEV.target(type(self).__name__[:-6]), interface=v)} instead"
|
||||
assert hasattr(self, "ifaces"), "must have ifaces to select an iface"
|
||||
t = DEV.target(dev:=type(self).__name__[:-6])
|
||||
filtered = select_by_name(self.ifaces, lambda i: i.__name__[:-5], t.interface, f"{dev} has no interface {t.interface!r}")
|
||||
filtered = [i for i in filtered if t.interface.startswith("MOCK") or not i.__name__[:-5].startswith("MOCK")] # never fall back to mock ifaces
|
||||
return select_first_inited([functools.partial(cast(Callable, iface), self, self.device_id) for iface in filtered],
|
||||
f"No interface for {dev}:{self.device_id} is available")
|
||||
|
||||
def _is_cpu(self) -> bool: return hasattr(self, 'device') and self.device.split(":")[0] == "CPU"
|
||||
|
||||
def finalize(self):
|
||||
try: self.synchronize() # try to finalize the device in any case
|
||||
except RuntimeError as e: print(f"{self.device} synchronization failed before finalizing: {e}")
|
||||
|
||||
# if the device has an interface, call device_fini to clean up resources
|
||||
if hasattr(self, 'iface') and hasattr(self.iface, 'device_fini'): self.iface.device_fini()
|
||||
|
||||
class HCQ2Buffer:
|
||||
def __init__(self, va_addr:sint, size:int, meta:Any=None, _base:HCQ2Buffer|None=None, view:MMIOInterface|None=None, owner:HCQ2Compiled|None=None):
|
||||
self.va_addr, self.size, self.meta, self._base, self.view, self.owner = va_addr, size, meta, _base, view, owner
|
||||
|
||||
def offset(self, offset:int=0, size:int|None=None) -> HCQ2Buffer:
|
||||
return HCQ2Buffer(self.va_addr+offset, size or (self.size - offset), owner=self.owner, meta=self.meta,
|
||||
_base=self._base or self, view=(self.view.view(offset=offset, size=size) if self.view is not None else None))
|
||||
|
||||
def cpu_view(self) -> MMIOInterface:
|
||||
assert self.view is not None, "buffer has no cpu_view"
|
||||
return self.view
|
||||
|
||||
@property
|
||||
def base(self) -> HCQ2Buffer: return self._base or self
|
||||
|
||||
class HCQAllocator(LRUAllocator[HCQDeviceType], Generic[HCQDeviceType]):
|
||||
def _map(self, buf:HCQ2Buffer) -> HCQ2Buffer:
|
||||
if not hasattr(self, '_do_map'): raise NotImplementedError("map failed: no method implemented")
|
||||
return self._do_map(buf)
|
||||
|
||||
@suppress_finalizing
|
||||
def _free(self, buf:HCQ2Buffer, options:BufferSpec|None=None):
|
||||
if options is not None and options.external_ptr is not None: return
|
||||
if hasattr(self, '_do_free'): self._do_free(buf, options)
|
||||
|
||||
def _unmap(self, mb):
|
||||
self.dev.synchronize()
|
||||
self.dev.iface.dev_impl.mm.unmap_range(int(mb.va_addr), round_up(mb.size, 0x1000))
|
||||
|
||||
def _offset(self, buf, size:int, offset:int) -> HCQ2Buffer: return buf.offset(offset=offset, size=size)
|
||||
|
||||
def _wrap(self, dev:str, sz:int, opaque:HCQ2Buffer) -> Buffer:
|
||||
return Buffer(dev, sz, dtypes.uint8, opaque=opaque, options=BufferSpec(external_ptr=1))
|
||||
|
||||
def _copy(self, dst:Buffer, src:Buffer):
|
||||
from tinygrad.engine.realize import run_linear
|
||||
su = UOp.from_buffer(src)
|
||||
run_linear(UOp(Ops.LINEAR, dtypes.void, (su.copy_to_device(dst.device).call(UOp.from_buffer(dst), su),)), jit=True, update_stats=False)
|
||||
|
||||
def _copyin(self, dest:HCQ2Buffer, src:memoryview):
|
||||
s = Buffer(self.dev.device, len(src), dtypes.uint8, options=BufferSpec(host=True), preallocate=True)
|
||||
s._buf.cpu_view()[:len(src)] = src
|
||||
self._copy(self._wrap(self.dev.device, len(src), dest), s)
|
||||
|
||||
def _copyout(self, dest:memoryview, src:HCQ2Buffer):
|
||||
d = Buffer(self.dev.device, len(dest), dtypes.uint8, options=BufferSpec(host=True), preallocate=True)
|
||||
self._copy(d, self._wrap(self.dev.device, len(dest), src))
|
||||
self.dev.synchronize()
|
||||
dest[:] = d._buf.cpu_view()[:len(dest)]
|
||||
|
||||
def _as_buffer(self, buf): return buf.cpu_view().mv
|
||||
|
||||
# **************** lower context ****************
|
||||
|
||||
@dataclass
|
||||
class HCQ2DeviceCtx:
|
||||
device:str # device name; resolve to instance via Device[device]
|
||||
kernargs_host:UOp # UOp whose .buffer is dev.kernargs_buf (BUFFER UOp in runtime, PARAM in graph)
|
||||
kernargs_gpu:UOp # va_addr const of dev.kernargs_buf
|
||||
kernargs_allocator:BumpAllocator = field(default_factory=lambda: BumpAllocator(2 << 20, wrap=False))
|
||||
|
||||
@dataclass
|
||||
class HCQ2LowerCtx:
|
||||
name:str
|
||||
inputs:list[Buffer] = field(default_factory=list)
|
||||
holds:list[UOp] = field(default_factory=list)
|
||||
devs:dict[str, HCQ2DeviceCtx] = field(default_factory=dict)
|
||||
|
||||
def host_param(self, buf:Buffer) -> UOp:
|
||||
if buf not in self.inputs: self.inputs.append(buf)
|
||||
return UOp.placeholder((buf.size,), buf.dtype, self.inputs.index(buf))
|
||||
|
||||
class HCQEncoder:
|
||||
def __init__(self, ctx:HCQ2LowerCtx, dev:HCQ2Compiled): self.ctx, self.dev, self.blob, self.patches, self.deps = ctx, dev, b'', [], []
|
||||
|
||||
@property
|
||||
def src(self) -> tuple[UOp, ...]: return tuple(self.patches + dedup(self.deps))
|
||||
|
||||
def get_dev_addr(self, uop:UOp) -> sint|UOp:
|
||||
while uop.op is Ops.AFTER:
|
||||
self.deps.extend(uop.src[1:])
|
||||
uop = uop.src[0]
|
||||
if isinstance(val:=uop.ssimplify(), UOp): self.deps.append(uop)
|
||||
return uop.buffer.get_buf(self.dev.device).va_addr if uop.op in (Ops.BUFFER, Ops.BUFFER_VIEW) else val
|
||||
|
||||
def append(self, *data, dtype=dtypes.uint32):
|
||||
for d in data:
|
||||
if isinstance(d, int): self.blob += struct.pack(f'<{dtype.fmt}', d)
|
||||
elif d.op is Ops.CONST: self.blob += struct.pack(f'<{dtype.fmt}', d.arg)
|
||||
else:
|
||||
self.patches.append(UOp(Ops.PATCH, dtype, src=(d,), arg=len(self.blob)))
|
||||
self.blob += struct.pack(f'<{dtype.fmt}', 0)
|
||||
|
||||
def q(self, *values): self.append(*values)
|
||||
|
||||
# **************** prep runtime ****************
|
||||
|
||||
pm_prep_runtime = PatternMatcher([
|
||||
# device-specific lowering of the program
|
||||
(UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, src=(UPat(), UPat(Ops.DEVICE), UPat(), UPat(), UPat(Ops.BINARY)), name="p"),), name="c", allow_any_len=True),
|
||||
lambda ctx, c, p: c.replace(src=(Device[p.src[1].arg].pm_lower.rewrite(p, ctx),) + c.src[1:])),
|
||||
])
|
||||
|
||||
def calc_kernargs_sizes(ctx:dict[str,int], u:UOp) -> None:
|
||||
d = u.src[0].buffer.device
|
||||
ctx[d] = ctx.get(d, 0) + round_up(u.arg[0].kernargs_alloc_size, 16)
|
||||
pm_calc_kernargs_sizes = PatternMatcher([(UPat(Ops.PROGRAM, name="u"), calc_kernargs_sizes)])
|
||||
|
||||
# **************** lower kernargs ****************
|
||||
|
||||
def lower_kernargs(ctx:HCQ2LowerCtx, call:UOp, prg:UOp) -> UOp:
|
||||
data, info = prg.arg
|
||||
# after amd_build_program, prg.src is (BUFFER_lib_gpu,); the buffer's device names the device
|
||||
dctx = ctx.devs[prg.src[0].buffer.device]
|
||||
|
||||
enc = HCQEncoder(ctx, Device[dctx.device])
|
||||
for gi in info.globals: enc.append(enc.get_dev_addr(call.src[1+gi]), dtype=dtypes.uint64)
|
||||
for v in info.vars: enc.append(v, dtype=dtypes.uint32)
|
||||
|
||||
args_off = dctx.kernargs_allocator.alloc(data.kernargs_alloc_size, 16)
|
||||
dctx.kernargs_host.buffer.view(len(enc.blob), dtypes.uint8, args_off).ensure_allocated().as_memoryview(force_zero_copy=True)[:] = enc.blob
|
||||
|
||||
args_uop = (dctx.kernargs_gpu + args_off).after(dctx.kernargs_host.after(*tuple(p.replace(arg=p.arg+args_off) for p in enc.patches)))
|
||||
return call.replace(src=(prg.replace(src=prg.src + (args_uop,), arg=(data, info)),) + call.src[1:])
|
||||
|
||||
pm_lower_kernargs = PatternMatcher([
|
||||
(UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, src=(UPat(Ops.BUFFER),), name="prg"),), name="call", allow_any_len=True), lower_kernargs),
|
||||
])
|
||||
|
||||
# **************** lower ops ****************
|
||||
|
||||
def lower_program(ctx:HCQ2LowerCtx, call:UOp, prg:UOp) -> UOp:
|
||||
q = UOp(Ops.LINEAR, dtypes.void, (prg,), arg=(prg.src[0].buffer.device, "COMPUTE"))
|
||||
return UOp(Ops.LINEAR, dtypes.void, (q,), tag=call.tag)
|
||||
|
||||
def lower_copy(ctx:HCQ2LowerCtx, call:UOp, copy:UOp) -> UOp:
|
||||
dst, src = call.src[1], call.src[2]
|
||||
q = UOp(Ops.LINEAR, dtypes.void, (UOp(Ops.COPY, dtypes.void, src=(dst, src), arg=src.buffer.nbytes),), arg=(dst.buffer.device, "COPY"))
|
||||
return UOp(Ops.LINEAR, dtypes.void, (q,), tag=call.tag)
|
||||
|
||||
pm_lower_ops = PatternMatcher([
|
||||
(UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, src=(UPat(Ops.BUFFER), UPat()), name="prg"),), name="call", allow_any_len=True), lower_program),
|
||||
(UPat(Ops.CALL, src=(UPat(Ops.COPY, name="copy"),), name="call", allow_any_len=True), lower_copy),
|
||||
])
|
||||
|
||||
# **************** split into queues ****************
|
||||
|
||||
def split_into_queues(ctx:HCQ2LowerCtx, outer:UOp) -> UOp:
|
||||
groups:dict[tuple, list[UOp]] = collections.defaultdict(list)
|
||||
for child in outer.src:
|
||||
wrapper = child.src[0] if child.op is Ops.AFTER else child
|
||||
for q in wrapper.src: groups[q.arg].extend(q.src)
|
||||
return outer.replace(src=tuple(UOp(Ops.LINEAR, dtypes.void, tuple(cmds), arg=k) for k, cmds in groups.items()))
|
||||
pm_split_into_queues = PatternMatcher([(UPat(Ops.LINEAR, src=UPat(Ops.LINEAR, src=UPat(Ops.LINEAR)).or_after(), name="outer"), split_into_queues)])
|
||||
|
||||
# **************** add signals (runtime) ****************
|
||||
|
||||
def add_signals(ctx:HCQ2LowerCtx, outer:UOp) -> UOp:
|
||||
def wrap(q:UOp) -> UOp:
|
||||
(dev_name, qname), devs = q.arg, {q.arg[0]} | {u.buffer.device for u in q.toposort() if u.op in (Ops.BUFFER, Ops.BUFFER_VIEW)}
|
||||
sigs_tls = [(UOp.from_buffer(Device[d].timeline_signal), ctx.host_param(Device[d].timeline_value)) for d in sorted(devs) if d.startswith("AMD")]
|
||||
return q.replace(src=(*(s.wait(t[0]-1) for s,t in sigs_tls), *q.src, *(s.store(t[0]) for s,t in sigs_tls)), arg=qname)
|
||||
return outer.replace(src=tuple(wrap(q) for q in outer.src))
|
||||
|
||||
pm_add_barriers = PatternMatcher([(UPat(Ops.LINEAR, src=UPat(Ops.LINEAR), name="outer"),
|
||||
lambda ctx, outer: outer.replace(src=tuple(q.replace(src=(UOp(Ops.BARRIER, dtypes.void), *q.src)) for q in outer.src)))])
|
||||
|
||||
pm_add_signals = PatternMatcher([(UPat(Ops.LINEAR, src=UPat(Ops.LINEAR), name="outer"), add_signals)])
|
||||
|
||||
# **************** build host program ****************
|
||||
|
||||
def resolve_cmdbuf(ctx:HCQ2LowerCtx, blob:UOp) -> UOp:
|
||||
inner = blob.src[0] if blob.op is Ops.AFTER else blob
|
||||
dev_name, qtype = inner.tag
|
||||
|
||||
# prepare the cmdbuf and make it a param
|
||||
bb = Buffer("CPU", len(inner.arg)//4, dtypes.uint32, preallocate=True)
|
||||
bb.copyin(memoryview(bytearray(inner.arg)))
|
||||
bb_param = ctx.host_param(bb)
|
||||
|
||||
submit_cf = UOp(Ops.CUSTOM_FUNCTION, dtypes.void, src=(bb_param.after(*(blob.src[1:] if blob.op is Ops.AFTER else ())),),
|
||||
arg=f"submit_{qtype.lower()}", tag=dev_name)
|
||||
|
||||
# increment the timeline value
|
||||
tl = ctx.host_param(Device[dev_name].timeline_value)
|
||||
return tl.after(UOp(Ops.BARRIER, dtypes.void, src=(submit_cf,))).index(UOp.const(dtypes.int, 0), ptr=True).store(tl[0] + 1)
|
||||
|
||||
def resolve_patches(ctx:HCQ2LowerCtx, buf:UOp) -> UOp|None:
|
||||
inner = buf.src[0]
|
||||
|
||||
# buffer is accessed from the launcher, so transform it to a host param
|
||||
if inner.op is Ops.BUFFER: inner = ctx.host_param(inner.buffer)
|
||||
|
||||
return inner.after(*(inner.index(UOp.const(dtypes.int, p.arg//inner.dtype.base.itemsize), ptr=True).cast(p.dtype.ptr()).store(p.src[0].cast(p.dtype))
|
||||
if p.op is Ops.PATCH else p for p in buf.src[1:]))
|
||||
|
||||
def resolve_ref_buffers(ctx:HCQ2LowerCtx, buf:UOp) -> UOp:
|
||||
if buf not in ctx.holds: ctx.holds.append(buf)
|
||||
return UOp(Ops.NOOP)
|
||||
|
||||
def hcq_callify(ctx:HCQ2LowerCtx, sink:UOp) -> UOp:
|
||||
call = to_program(sink, Device["CPU"].renderer).call(*[UOp.from_buffer(b, "CPU") if isinstance(b, Buffer) else b for b in ctx.inputs])
|
||||
return call.replace(src=call.src + (UOp(Ops.BIND, dtypes.void, src=tuple(ctx.holds)),)) if ctx.holds else call
|
||||
|
||||
pm_create_host_sink = PatternMatcher([
|
||||
(UPat(Ops.LINEAR, name="l", allow_any_len=True), lambda ctx, l: UOp.sink(*l.src, arg=KernelInfo(name=ctx.name, estimates=Estimates()), tag=1))
|
||||
])
|
||||
|
||||
# lower cmdbuf submits
|
||||
pm_lower_cmdbufs = PatternMatcher([
|
||||
(UPat(Ops.AFTER, src=(UPat(Ops.BINARY),), name="blob", allow_any_len=True), resolve_cmdbuf),
|
||||
(UPat(Ops.BINARY, name="blob"), resolve_cmdbuf),
|
||||
])
|
||||
|
||||
# transform patches attached to buffers and params
|
||||
pm_resolve_patches = PatternMatcher([
|
||||
(UPat(Ops.AFTER, src=(UPat((Ops.BUFFER, Ops.PARAM)),), name="buf", allow_any_len=True), resolve_patches)
|
||||
])
|
||||
|
||||
# replace referenced buffers with noops
|
||||
pm_resolve_ref_buffers = PatternMatcher([(UPat((Ops.BUFFER, Ops.BUFFER_VIEW), name="buf"), resolve_ref_buffers)])
|
||||
|
||||
pm_callify = PatternMatcher([(UPat(Ops.SINK, name="sink"), hcq_callify)])
|
||||
|
||||
# **************** schedule ****************
|
||||
|
||||
def prep_runtime(ctx:HCQ2LowerCtx, linear:UOp) -> tuple[UOp, dict[str,int]]:
|
||||
linear = graph_rewrite(linear, pm_prep_runtime, ctx=ctx, name="hcq: prepare runtime")
|
||||
graph_rewrite(linear, pm_calc_kernargs_sizes, ctx=(sizes:={}), enter_calls=True)
|
||||
return linear, sizes
|
||||
|
||||
def build_host_program(ctx:HCQ2LowerCtx, linear:UOp, ast:UOp, dev:HCQ2Compiled) -> UOp:
|
||||
sink = graph_rewrite(linear, pm_create_host_sink, ctx=ctx, name="hcq: create host sink", walk=True)
|
||||
sink = graph_rewrite(sink, pm_lower_cmdbufs, ctx=ctx, bottom_up=True, name="hcq: lower cmdbufs")
|
||||
sink = graph_rewrite(sink, pm_resolve_patches, ctx=ctx, bottom_up=True, name="hcq: resolve patches")
|
||||
sink = graph_rewrite(sink, pm_resolve_ref_buffers, ctx=ctx, bottom_up=True, name="hcq: resolve ref buffers")
|
||||
sink = graph_rewrite(sink, dev.pm_lower, ctx=ctx, name=f"hcq: device lower {dev.device}", walk=True)
|
||||
return graph_rewrite(sink, pm_callify, ctx=ctx, name="hcq: callify")
|
||||
|
||||
@track_rewrites(name=lambda ctx,linear,ast,dev,**kw: f"hcq schedule {getattr(ast.arg, 'name', ast.op.name.lower())}")
|
||||
def hcq_schedule(ctx:HCQ2LowerCtx, linear:UOp, ast:UOp, dev:HCQ2Compiled) -> UOp:
|
||||
linear, sizes = prep_runtime(ctx, linear)
|
||||
for dev_name, sz in sizes.items():
|
||||
off = dev.kernargs_offset_allocator.alloc(sz, 16)
|
||||
ctx.devs[dev_name] = HCQ2DeviceCtx(dev_name, UOp.from_buffer(dev.kernargs_buf.view(sz, dtypes.uint8, off), dev_name),
|
||||
UOp.const(dtypes.uint64, dev.kernargs_buf.get_buf(dev_name).va_addr + off))
|
||||
linear = graph_rewrite(linear, pm_lower_kernargs + pm_lower_ops, ctx=ctx, name="hcq: lower ops")
|
||||
linear = graph_rewrite(linear, pm_split_into_queues, ctx=ctx, name="hcq: split into queues")
|
||||
linear = graph_rewrite(linear, pm_add_barriers, ctx=ctx, name="hcq: add barriers", walk=True)
|
||||
linear = graph_rewrite(linear, pm_add_signals, ctx=ctx, name="hcq: add signals", walk=True)
|
||||
linear = graph_rewrite(linear, dev.pm_lower, ctx=ctx, name=f"hcq: encode cmdbuf {dev.device}", walk=True)
|
||||
return build_host_program(ctx, linear, ast, dev)
|
||||
|
||||
def ensure_accessible(ctx:HCQ2LowerCtx, call:UOp, copy:UOp) -> UOp|None:
|
||||
src_buf = call.src[2].buffer # TODO: cleanup
|
||||
dev = call.src[1].buffer.device
|
||||
try: src_buf.get_buf(dev)
|
||||
except Exception:
|
||||
(cpubuf := Buffer("CPU", src_buf.nbytes, dtypes.uint8, preallocate=True)).copyin(src_buf.ensure_allocated().as_memoryview())
|
||||
ctx.holds.append(buf_uop:=UOp.from_buffer(cpubuf, dev))
|
||||
return call.replace(src=call.src[:2] + (buf_uop,) + call.src[3:])
|
||||
pm_ensure_bufs_accessible = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.COPY, name="copy"),), name="call", allow_any_len=True), ensure_accessible)])
|
||||
|
||||
def hcq_exec(ctx:ExecContext, call:UOp, ast:UOp) -> float|None:
|
||||
from tinygrad.engine.realize import run_linear
|
||||
|
||||
if ast.src[1].arg.split(":")[0] != "AMD": return None
|
||||
|
||||
# TODO: this mess should gone
|
||||
resolved_call = call.replace(src=(ast,) + tuple(resolve_params(call, ctx.input_uops)) + tuple(s for s in call.src[1:] if s.op is Ops.BIND))
|
||||
bufs = [cast(Buffer, resolved_call.src[1+gi].buffer) for gi in ast.arg.globals] if ast.op is Ops.PROGRAM \
|
||||
else [cast(Buffer, resolved_call.src[i].buffer) for i in range(1, len(resolved_call.src))]
|
||||
dev = cast(HCQ2Compiled, Device[bufs[0].device])
|
||||
hcq_ctx = HCQ2LowerCtx(name="submit")
|
||||
linear = graph_rewrite(UOp(Ops.LINEAR, dtypes.void, (resolved_call,)), pm_ensure_bufs_accessible, ctx=hcq_ctx)
|
||||
host_call = hcq_schedule(hcq_ctx, linear, ast, dev)
|
||||
with track_stats(ctx, call, dev.device, bufs, ctx.var_vals) as tm:
|
||||
st = time.perf_counter() if ctx.wait else 0.0
|
||||
run_linear(UOp(Ops.LINEAR, dtypes.void, (host_call,)), var_vals=ctx.var_vals, jit=True, update_stats=DEBUG>=3)
|
||||
if ctx.wait:
|
||||
dev.synchronize()
|
||||
tm[0] = time.perf_counter() - st
|
||||
return tm[0] if tm[0] is not None else 0.0
|
||||
|
||||
pm_hcq_exec = PatternMatcher([
|
||||
(UPat(Ops.CALL, src=(UPat({Ops.PROGRAM, Ops.COPY}, name="ast"),), name="call", allow_any_len=True), hcq_exec),
|
||||
])
|
||||
@@ -0,0 +1,522 @@
|
||||
from __future__ import annotations
|
||||
from typing import cast
|
||||
import os, ctypes, struct, hashlib, functools, importlib, mmap, errno, array, contextlib, sys, weakref, itertools, collections, atexit
|
||||
assert sys.platform != 'win32'
|
||||
from dataclasses import dataclass
|
||||
from extra.hcq2.hcq2 import HCQ2Compiled, HCQAllocator, HCQ2Buffer, HCQEncoder
|
||||
from tinygrad.uop.ops import sint, UOp
|
||||
from tinygrad.device import Compiled, BufferSpec, Buffer, Device
|
||||
from tinygrad.dtype import dtypes
|
||||
from tinygrad.helpers import getenv, round_up, data64_le, DEBUG, PROFILE, ProfileEvent, lo32, hi32, colored, prod, ContextVar, TracingKey
|
||||
from tinygrad.helpers import VIZ, ceildiv, unwrap, pluralize
|
||||
from tinygrad.renderer.cstyle import HIPRenderer, HIPCCRenderer
|
||||
from tinygrad.renderer.llvmir import AMDLLVMRenderer
|
||||
from tinygrad.runtime.autogen import kfd, hsa, sqtt, amdgpu_kd, amdgpu_drm
|
||||
from tinygrad.runtime.autogen.am import am
|
||||
from tinygrad.runtime.support.elf import elf_loader
|
||||
from tinygrad.runtime.support.am.amdev import AMDev, AMMemoryManager
|
||||
from tinygrad.runtime.support.amd import AMDReg, AMDIP, import_module, import_soc, import_pmc
|
||||
from tinygrad.runtime.support.system import PCIIfaceBase, PCIAllocationMeta, USBPCIDevice, MAP_FIXED, MAP_NORESERVE
|
||||
from tinygrad.runtime.support.usb import USB3
|
||||
from tinygrad.runtime.support.memory import AddrSpace, BumpAllocator
|
||||
from tinygrad.runtime.ops_amd import SQTT, SQTT_ITRACE_SE_MASK, SQTT_LIMIT_SE, SQTT_SIMD_SEL, SQTT_TOKEN_EXCLUDE, PMC
|
||||
from tinygrad.runtime.ops_amd import EVENT_INDEX_PARTIAL_FLUSH, WAIT_REG_MEM_FUNCTION_EQ, WAIT_REG_MEM_FUNCTION_NEQ, WAIT_REG_MEM_FUNCTION_GEQ
|
||||
if getenv("IOCTL"): import extra.hip_gpu_driver.hip_ioctl # noqa: F401 # pylint: disable=unused-import
|
||||
|
||||
from extra.hcq2.hcq2 import HCQ2LowerCtx
|
||||
from tinygrad.engine.realize import get_runtime
|
||||
from tinygrad.uop.ops import Ops, UPat, PatternMatcher, graph_rewrite
|
||||
|
||||
class AMDComputeQueue(HCQEncoder):
|
||||
def __init__(self, ctx:HCQ2LowerCtx, dev:AMDDevice):
|
||||
super().__init__(ctx, dev)
|
||||
self.pm4, self.gc, self.nbio, self.soc = self.dev.pm4, self.dev.gc, self.dev.nbio, self.dev.soc
|
||||
|
||||
def pkt3(self, cmd, *vals): self.q(self.pm4.PACKET3(cmd, len(vals) - 1), *vals)
|
||||
|
||||
def wreg(self, reg:AMDReg, *args:sint, **kwargs:int):
|
||||
if bool(args) == bool(kwargs): raise RuntimeError('One (and only one) of *args or **kwargs must be specified')
|
||||
if self.pm4.PACKET3_SET_SH_REG_START <= reg.addr[0] < self.pm4.PACKET3_SET_SH_REG_END:
|
||||
set_packet, set_packet_start = self.pm4.PACKET3_SET_SH_REG, self.pm4.PACKET3_SET_SH_REG_START
|
||||
elif self.pm4.PACKET3_SET_UCONFIG_REG_START <= reg.addr[0] < self.pm4.PACKET3_SET_UCONFIG_REG_START + 2**16-1:
|
||||
set_packet, set_packet_start = self.pm4.PACKET3_SET_UCONFIG_REG, self.pm4.PACKET3_SET_UCONFIG_REG_START
|
||||
else: raise RuntimeError(f'Cannot set {reg.name} ({reg.addr[0]}) via pm4 packet')
|
||||
self.pkt3(set_packet, reg.addr[0] - set_packet_start, *(args or (reg.encode(**kwargs),)))
|
||||
|
||||
def wait_reg_mem(self, value, mask=0xffffffff, mem=None, reg=None, reg_done=0, op=WAIT_REG_MEM_FUNCTION_GEQ):
|
||||
wrm_info_dw = self.pm4.WAIT_REG_MEM_MEM_SPACE(int(mem is not None)) | self.pm4.WAIT_REG_MEM_OPERATION(int(mem is None and reg_done > 0)) \
|
||||
| self.pm4.WAIT_REG_MEM_FUNCTION(op) | self.pm4.WAIT_REG_MEM_ENGINE(0)
|
||||
self.pkt3(self.pm4.PACKET3_WAIT_REG_MEM, wrm_info_dw, *(data64_le(mem) if mem is not None else (reg, reg_done)), value, mask, 4)
|
||||
|
||||
def acquire_mem(self, addr=0x0, sz=(1 << 64)-1, gli=1, glm=1, glk=1, glv=1, gl1=1, gl2=1):
|
||||
if self.dev.target[0] != 9:
|
||||
cache_flags_dw = self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(gli) \
|
||||
| self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(glm) | self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(glm) \
|
||||
| self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(glk) | self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_WB(glk) \
|
||||
| self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(glv) | self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(gl1) \
|
||||
| self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(gl2) | self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(gl2)
|
||||
self.pkt3(self.pm4.PACKET3_ACQUIRE_MEM, 0, *data64_le(sz), *data64_le(addr), 0, cache_flags_dw)
|
||||
else:
|
||||
cp_coher_cntl = self.pm4.PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(gli) | \
|
||||
self.pm4.PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(glk) | \
|
||||
self.pm4.PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(gl2) | \
|
||||
self.pm4.PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(gl1) | \
|
||||
self.pm4.PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(gl2)
|
||||
self.pkt3(self.pm4.PACKET3_ACQUIRE_MEM, cp_coher_cntl, *data64_le(sz), *data64_le(addr), 0x0000000A)
|
||||
|
||||
def release_mem(self, address=0x0, value=0, data_sel=0, int_sel=2, ctxid=0, cache_flush=False):
|
||||
if self.dev.target[0] != 9:
|
||||
cache_flags_dw = 0 if not cache_flush else (self.pm4.PACKET3_RELEASE_MEM_GCR_GLV_INV | self.pm4.PACKET3_RELEASE_MEM_GCR_GL1_INV \
|
||||
| self.pm4.PACKET3_RELEASE_MEM_GCR_GL2_INV | self.pm4.PACKET3_RELEASE_MEM_GCR_GLM_WB \
|
||||
| self.pm4.PACKET3_RELEASE_MEM_GCR_GLM_INV | self.pm4.PACKET3_RELEASE_MEM_GCR_GL2_WB | self.pm4.PACKET3_RELEASE_MEM_GCR_SEQ)
|
||||
event_dw = self.pm4.PACKET3_RELEASE_MEM_EVENT_TYPE(self.pm4.CACHE_FLUSH_AND_INV_TS_EVENT) \
|
||||
| self.pm4.PACKET3_RELEASE_MEM_EVENT_INDEX(self.pm4.event_index__mec_release_mem__end_of_pipe)
|
||||
memsel_dw = self.pm4.PACKET3_RELEASE_MEM_DATA_SEL(data_sel) | self.pm4.PACKET3_RELEASE_MEM_INT_SEL(int_sel) \
|
||||
| self.pm4.PACKET3_RELEASE_MEM_DST_SEL(0)
|
||||
else:
|
||||
cache_flags_dw = 0 if not cache_flush else (self.pm4.EOP_TC_WB_ACTION_EN | self.pm4.EOP_TC_NC_ACTION_EN)
|
||||
event_dw = self.pm4.EVENT_TYPE(self.pm4.CACHE_FLUSH_AND_INV_TS_EVENT) | self.pm4.EVENT_INDEX(self.pm4.event_index__mec_release_mem__end_of_pipe)
|
||||
memsel_dw = self.pm4.DATA_SEL(data_sel) | self.pm4.INT_SEL(int_sel)
|
||||
ctxid = 0
|
||||
self.pkt3(self.pm4.PACKET3_RELEASE_MEM, event_dw | cache_flags_dw, memsel_dw, *data64_le(address), *data64_le(value), ctxid)
|
||||
|
||||
def memory_barrier(self):
|
||||
pf = '' if self.nbio.version[0] == 2 else '0' if self.nbio.version[:2] != (7, 11) else '1'
|
||||
self.wait_reg_mem(reg=getattr(self.nbio, f'regBIF_BX_PF{pf}_GPU_HDP_FLUSH_REQ').addr[0],
|
||||
reg_done=getattr(self.nbio, f'regBIF_BX_PF{pf}_GPU_HDP_FLUSH_DONE').addr[0], value=0xffffffff)
|
||||
self.acquire_mem()
|
||||
|
||||
def wait(self, x): self.wait_reg_mem(x.src[1], mem=self.get_dev_addr(x.src[0]))
|
||||
|
||||
def barrier(self, x): self.memory_barrier()
|
||||
|
||||
def store(self, x):
|
||||
self.release_mem(self.get_dev_addr(x.src[0]), x.src[1], self.pm4.data_sel__mec_release_mem__send_32_bit_low,
|
||||
self.pm4.int_sel__mec_release_mem__send_interrupt_after_write_confirm, cache_flush=True)
|
||||
|
||||
def timestamp(self, x):
|
||||
self.release_mem(self.get_dev_addr(x.src[0]), 0, self.pm4.data_sel__mec_release_mem__send_gpu_clock_counter,
|
||||
self.pm4.int_sel__mec_release_mem__none)
|
||||
|
||||
def program(self, x):
|
||||
data, info = x.arg
|
||||
lib_gpu, args = x.src
|
||||
prog_addr = self.get_dev_addr(lib_gpu) + data.entry_point_offset
|
||||
|
||||
self.acquire_mem(gli=0, gl2=0)
|
||||
|
||||
args_addr = self.get_dev_addr(args)
|
||||
user_regs = []
|
||||
if data.enable_private_segment_sgpr:
|
||||
scratch_hilo = data64_le(self.dev.scratch.va_addr)
|
||||
user_regs = [scratch_hilo[0], scratch_hilo[1] | 1 << 31, 0xffffffff, 0x20c14000]
|
||||
if data.enable_dispatch_ptr: user_regs += [*data64_le(args_addr + data.kernargs_segment_size)]
|
||||
user_regs += [*data64_le(args_addr)]
|
||||
|
||||
self.wreg(self.gc.regCOMPUTE_PGM_LO, *data64_le(prog_addr >> 8))
|
||||
self.wreg(self.gc.regCOMPUTE_PGM_RSRC1, data.rsrc1, data.rsrc2)
|
||||
self.wreg(self.gc.regCOMPUTE_PGM_RSRC3, data.rsrc3)
|
||||
self.wreg(self.gc.regCOMPUTE_TMPRING_SIZE, self.dev.tmpring_size)
|
||||
|
||||
for xcc_id in range(self.dev.xccs):
|
||||
scratch_base = self.dev.scratch.va_addr + (self.dev.scratch.size // self.dev.xccs * xcc_id)
|
||||
self.wreg(self.gc.regCOMPUTE_DISPATCH_SCRATCH_BASE_LO, *data64_le(scratch_base >> 8))
|
||||
|
||||
self.wreg(self.gc.regCOMPUTE_RESTART_X, 0, 0, 0)
|
||||
self.wreg(self.gc.regCOMPUTE_USER_DATA_0, *user_regs)
|
||||
self.wreg(self.gc.regCOMPUTE_RESOURCE_LIMITS, self.gc.regCOMPUTE_RESOURCE_LIMITS.encode(waves_per_sh=getenv("WAVES_PER_SH")))
|
||||
self.wreg(self.gc.regCOMPUTE_START_X, 0, 0, 0, *(info.local_size or (1, 1, 1)), 0, 0)
|
||||
|
||||
dispatch_init = self.gc.regCOMPUTE_DISPATCH_INITIATOR.encode(
|
||||
**({'cs_w32_en': int(data.wave32)} if self.dev.target[0] != 9 else {}), force_start_at_000=1, compute_shader_en=1)
|
||||
self.pkt3(self.pm4.PACKET3_DISPATCH_DIRECT, *info.global_size, dispatch_init)
|
||||
self.pkt3(self.pm4.PACKET3_EVENT_WRITE, self.pm4.EVENT_TYPE(self.soc.CS_PARTIAL_FLUSH) | self.pm4.EVENT_INDEX(EVENT_INDEX_PARTIAL_FLUSH))
|
||||
|
||||
amd_inner_pm = PatternMatcher([
|
||||
(UPat(Ops.WAIT, name="x"), lambda ctx, x: ctx.wait(x)),
|
||||
(UPat(Ops.BARRIER, name="x"), lambda ctx, x: ctx.barrier(x)),
|
||||
(UPat(Ops.PROGRAM, name="x"), lambda ctx, x: ctx.program(x)),
|
||||
(UPat(Ops.CUSTOM_FUNCTION, arg="timestamp", name="x"), lambda ctx, x: ctx.timestamp(x)),
|
||||
(UPat(Ops.STORE, src=(UPat((Ops.BUFFER, Ops.PARAM)), UPat()), name="x"), lambda ctx, x: ctx.store(x)),
|
||||
])
|
||||
|
||||
def amd_lower_pm4(ctx, linear):
|
||||
prg = next(s for s in linear.src if s.op is Ops.PROGRAM)
|
||||
dev = Device[prg.src[1].arg]
|
||||
enc = AMDComputeQueue(ctx, dev)
|
||||
graph_rewrite(linear, amd_inner_pm, ctx=enc, name="amd: encode")
|
||||
return UOp(Ops.BINARY, dtypes.void, arg=enc.blob).rtag((dev.device, "COMPUTE")).after(*enc.src)
|
||||
|
||||
def amd_submit_pm4(ctx, cf):
|
||||
dev = Device[cf.tag]
|
||||
bb_param = cf.src[0]
|
||||
q = dev.compute_queue
|
||||
ring, wptr, doorbell, put_ptr = (ctx.host_param(b) for b in (q.ring, q.write_ptr, q.doorbell, q.put_value))
|
||||
size, ring_dwords = UOp.const(dtypes.uint32, bb_param.dtype.size), q.ring.size
|
||||
|
||||
put = put_ptr[0]
|
||||
i = UOp.range(size, 0, dtype=dtypes.int)
|
||||
next_put = put + size.cast(put.dtype)
|
||||
ring_idx = ((put + i.cast(put.dtype)) % ring_dwords).cast(dtypes.int)
|
||||
|
||||
copy_to_ring = ring[ring_idx].store(bb_param[i]).end(i)
|
||||
bump_put_ptr = put_ptr[0].store(next_put)
|
||||
bump_wptr = wptr[0].store(next_put)
|
||||
flush = UOp.barrier(copy_to_ring, bump_put_ptr, bump_wptr)
|
||||
return doorbell.after(flush)[0].store(next_put)
|
||||
|
||||
class AMDCopyQueue(HCQEncoder):
|
||||
def __init__(self, ctx:HCQ2LowerCtx, dev:AMDDevice, queue_idx=0):
|
||||
super().__init__(ctx, dev)
|
||||
self.sdma, self.queue_idx, self.max_copy_size = self.dev.sdma, queue_idx, self.dev.max_copy_size
|
||||
|
||||
def copy(self, x):
|
||||
dest, src, copy_size = self.get_dev_addr(x.src[0]), self.get_dev_addr(x.src[1]), x.arg
|
||||
copied = 0
|
||||
while copied < copy_size:
|
||||
step = min(copy_size - copied, self.max_copy_size)
|
||||
self.q(self.sdma.SDMA_OP_COPY | self.sdma.SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(self.sdma.SDMA_SUBOP_COPY_LINEAR),
|
||||
self.sdma.SDMA_PKT_COPY_LINEAR_COUNT_COUNT(step - 1), 0, *data64_le(src + copied), *data64_le(dest + copied))
|
||||
copied += step
|
||||
|
||||
def wait(self, x):
|
||||
self.q(self.sdma.SDMA_OP_POLL_REGMEM | self.sdma.SDMA_PKT_POLL_REGMEM_HEADER_FUNC(WAIT_REG_MEM_FUNCTION_GEQ) | \
|
||||
self.sdma.SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1), *data64_le(self.get_dev_addr(x.src[0])), x.src[1], 0xffffffff,
|
||||
self.sdma.SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(0x04) | self.sdma.SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff))
|
||||
|
||||
def store(self, x):
|
||||
fence_flags = self.sdma.SDMA_PKT_FENCE_HEADER_MTYPE(3) if self.dev.target[0] != 9 else 0
|
||||
self.q(self.sdma.SDMA_OP_FENCE | fence_flags, *data64_le(self.get_dev_addr(x.src[0])), x.src[1])
|
||||
self.q(self.sdma.SDMA_OP_TRAP, 0)
|
||||
|
||||
def timestamp(self, x):
|
||||
self.q(self.sdma.SDMA_OP_TIMESTAMP | self.sdma.SDMA_PKT_TIMESTAMP_GET_HEADER_SUB_OP(self.sdma.SDMA_SUBOP_TIMESTAMP_GET_GLOBAL),
|
||||
*data64_le(self.get_dev_addr(x.src[0])))
|
||||
|
||||
def amd_lower_sdma(ctx, linear):
|
||||
copy = next(s for s in linear.src if s.op is Ops.COPY)
|
||||
dev = Device[copy.src[0].buffer.device]
|
||||
enc = AMDCopyQueue(ctx, dev)
|
||||
graph_rewrite(linear, amd_inner_sdma_pm, ctx=enc, name="amd: encode sdma")
|
||||
return UOp(Ops.BINARY, dtypes.void, arg=enc.blob).rtag((dev.device, "COPY")).after(*enc.src)
|
||||
|
||||
amd_inner_sdma_pm = PatternMatcher([
|
||||
(UPat(Ops.WAIT, name="x"), lambda ctx, x: ctx.wait(x)),
|
||||
(UPat(Ops.BARRIER, name="x"), lambda ctx, x: None),
|
||||
(UPat(Ops.COPY, name="x"), lambda ctx, x: ctx.copy(x)),
|
||||
(UPat(Ops.CUSTOM_FUNCTION, arg="timestamp", name="x"), lambda ctx, x: ctx.timestamp(x)),
|
||||
(UPat(Ops.STORE, src=(UPat((Ops.BUFFER, Ops.PARAM)), UPat()), name="x"), lambda ctx, x: ctx.store(x)),
|
||||
])
|
||||
|
||||
def amd_submit_sdma(ctx, cf):
|
||||
dev = Device[cf.tag]
|
||||
bb_param = cf.src[0]
|
||||
q = dev.sdma_queue(0)
|
||||
ring, wptr, doorbell, put_ptr = (ctx.host_param(b) for b in (q.ring, q.write_ptr, q.doorbell, q.put_value))
|
||||
size_dw, ring_bytes = bb_param.dtype.size, q.ring.size * 4
|
||||
|
||||
put_b = put_ptr[0]
|
||||
tail_off_dw = ((put_b % ring_bytes) // 4).cast(dtypes.int)
|
||||
fits = (size_dw <= q.ring.size - tail_off_dw).cast(dtypes.int)
|
||||
start_dw = fits * tail_off_dw
|
||||
zero_amt_dw = (1 - fits) * (q.ring.size - tail_off_dw)
|
||||
|
||||
zi = UOp.range(zero_amt_dw, 0, dtype=dtypes.int)
|
||||
zero_tail = ring[tail_off_dw + zi].store(UOp.const(dtypes.uint32, 0)).end(zi)
|
||||
|
||||
i = UOp.range(UOp.const(dtypes.int, size_dw), 0, dtype=dtypes.int)
|
||||
copy_to_ring = ring[start_dw + i].store(bb_param[i]).end(i)
|
||||
|
||||
next_put_b = put_b + ((zero_amt_dw + size_dw) * 4).cast(put_b.dtype)
|
||||
bump_put_ptr = put_ptr[0].store(next_put_b)
|
||||
bump_wptr = wptr[0].store(next_put_b)
|
||||
flush = UOp.barrier(zero_tail, copy_to_ring, bump_put_ptr, bump_wptr)
|
||||
return doorbell.after(flush)[0].store(next_put_b)
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AMDProgramData:
|
||||
entry_point_offset:int; rsrc1:int; rsrc2:int; rsrc3:int; wave32:bool
|
||||
kernargs_segment_size:int; kernargs_alloc_size:int
|
||||
enable_dispatch_ptr:int; enable_private_segment_sgpr:int
|
||||
|
||||
_amd_program_cache:dict[tuple[bytes,str], tuple[AMDProgramData,Buffer]] = {}
|
||||
|
||||
def amd_build_program(ctx:HCQ2LowerCtx, prg:UOp) -> UOp:
|
||||
dev = Device[prg.src[1].arg]
|
||||
if (cached:=_amd_program_cache.get(key:=(lib:=prg.src[4].arg, dev.device))) is None:
|
||||
image, sections, relocs = elf_loader(lib)
|
||||
rodata = next(sh.header.sh_addr for sh in sections if sh.name == ".rodata")
|
||||
for off, sym, typ, addent in relocs:
|
||||
assert typ == 5, f"unknown AMD reloc {typ}" # R_AMDGPU_REL64
|
||||
image[off:off+8] = struct.pack('<q', sym - off + addent)
|
||||
lib_gpu = Buffer(dev.device, round_up(image.nbytes, 0x1000), dtypes.uint8, options=BufferSpec(nolru=True), preallocate=True)
|
||||
dev.allocator._copyin(lib_gpu._buf, image)
|
||||
dev.synchronize()
|
||||
desc = amdgpu_kd.llvm_amdhsa_kernel_descriptor_t.from_buffer_copy(bytes(image[rodata:rodata+ctypes.sizeof(amdgpu_kd.llvm_amdhsa_kernel_descriptor_t)]))
|
||||
if (lds:=((desc.group_segment_fixed_size+511)//512)&0x1FF) > (dev.iface.props['lds_size_in_kb']*1024)//512:
|
||||
raise RuntimeError("Too many resources requested: group_segment_size")
|
||||
dev._ensure_has_local_memory(desc.private_segment_fixed_size)
|
||||
edp = desc.kernel_code_properties & hsa.AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_DISPATCH_PTR
|
||||
cached = _amd_program_cache[key] = (AMDProgramData(
|
||||
entry_point_offset=rodata + desc.kernel_code_entry_byte_offset,
|
||||
rsrc1=desc.compute_pgm_rsrc1 | ((1<<20) if dev.target[0]==11 else 0), # priv=1 on gfx11 for cwsr
|
||||
rsrc2=desc.compute_pgm_rsrc2 | (lds<<15), rsrc3=desc.compute_pgm_rsrc3,
|
||||
wave32=bool(desc.kernel_code_properties & 0x400),
|
||||
kernargs_segment_size=desc.kernarg_size,
|
||||
kernargs_alloc_size=desc.kernarg_size + (ctypes.sizeof(hsa.hsa_kernel_dispatch_packet_t) if edp else 0),
|
||||
enable_dispatch_ptr=edp,
|
||||
enable_private_segment_sgpr=desc.kernel_code_properties & hsa.AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
|
||||
), lib_gpu)
|
||||
data, lib_gpu = cached
|
||||
return prg.replace(src=(UOp.from_buffer(lib_gpu, dev.device),), arg=(data, prg.arg))
|
||||
|
||||
class AMDAllocator(HCQAllocator['AMDDevice']):
|
||||
def __init__(self, dev:AMDDevice):
|
||||
super().__init__(dev, supports_copy_from_disk=dev.has_sdma_queue, supports_transfer=dev.has_sdma_queue and not dev.is_usb())
|
||||
|
||||
def _alloc(self, size:int, options:BufferSpec) -> HCQ2Buffer:
|
||||
return self.dev.iface.alloc(size, host=True, uncached=options.uncached, cpu_access=True)
|
||||
|
||||
def _do_free(self, opaque, options:BufferSpec): self.dev.iface.free(opaque)
|
||||
|
||||
def _do_map(self, buf:HCQ2Buffer): return self.dev.iface.map(buf._base if buf._base is not None else buf)
|
||||
|
||||
@dataclass
|
||||
class AMDQueueDesc:
|
||||
ring: Buffer # uint32[ring_size//4]
|
||||
read_ptr: Buffer # uint64[1]
|
||||
write_ptr: Buffer # uint64[1]
|
||||
doorbell: Buffer # uint64[1]
|
||||
put_value: Buffer # uint64[1]
|
||||
params: tuple|None = None # setup_ring params for recovery
|
||||
|
||||
class PCIIface(PCIIfaceBase):
|
||||
def __init__(self, dev, dev_id):
|
||||
super().__init__(dev, dev_id, vendor=0x1002, devices=((0xffff, (0x74a1,0x744c,0x7480,0x7550,0x7551,0x7590,0x75a0)),), vram_bar=0,
|
||||
va_start=AMMemoryManager.va_allocator.base, va_size=AMMemoryManager.va_allocator.size, dev_impl_t=AMDev)
|
||||
self._compute_props()
|
||||
|
||||
def p2p_paddrs(self, paddrs:list[tuple[int,int]]) -> tuple[list[tuple[int,int]], AddrSpace]:
|
||||
return ([(self.dev_impl.paddr2xgmi(p), sz) for p, sz in paddrs], AddrSpace.PEER) if self.dev_impl.is_hive() else super().p2p_paddrs(paddrs)
|
||||
|
||||
def require_profile_mode(self): return True
|
||||
def is_wgp_active(self, xcc, se, sa, wgp) -> bool: return True # TODO: account for WGP disablement on some asics.
|
||||
|
||||
def _compute_props(self):
|
||||
self.ip_versions = self.dev_impl.ip_ver
|
||||
|
||||
gfxver = int(f"{self.dev_impl.ip_ver[am.GC_HWIP][0]:02d}{self.dev_impl.ip_ver[am.GC_HWIP][1]:02d}{self.dev_impl.ip_ver[am.GC_HWIP][2]:02d}")
|
||||
if self.dev_impl.gc_info.header.version_major == 2:
|
||||
cu_per_sa = self.dev_impl.gc_info.gc_num_cu_per_sh
|
||||
max_sh_per_se = self.dev_impl.gc_info.gc_num_sh_per_se
|
||||
else:
|
||||
cu_per_sa = 2 * (self.dev_impl.gc_info.gc_num_wgp0_per_sa + self.dev_impl.gc_info.gc_num_wgp1_per_sa)
|
||||
max_sh_per_se = self.dev_impl.gc_info.gc_num_sa_per_se
|
||||
|
||||
array_count = max_sh_per_se * self.dev_impl.gc_info.gc_num_se * self.dev_impl.gfx.xccs
|
||||
self.props = {'cu_per_simd_array': cu_per_sa, 'simd_count': 2 * cu_per_sa * array_count, 'simd_per_cu': 2, 'array_count': array_count,
|
||||
'max_slots_scratch_cu': self.dev_impl.gc_info.gc_max_scratch_slots_per_cu, 'max_waves_per_simd': self.dev_impl.gc_info.gc_max_waves_per_simd,
|
||||
'simd_arrays_per_engine': max_sh_per_se, 'lds_size_in_kb': self.dev_impl.gc_info.gc_lds_size, 'num_xcc': self.dev_impl.gfx.xccs,
|
||||
'gfx_target_version': {90403: 90402}.get(gfxver, gfxver)}
|
||||
|
||||
def create_queue(self, queue_type, ring, gart, rptr, wptr, eop_buffer=None, cwsr_buffer=None, ctl_stack_size=0, ctx_save_restore_size=0,
|
||||
xcc_id=0, idx=0):
|
||||
assert cwsr_buffer is None, "no cwsr buffer for am"
|
||||
|
||||
rcvr_params: tuple
|
||||
if queue_type == kfd.KFD_IOC_QUEUE_TYPE_SDMA:
|
||||
doorbell_index = self.dev_impl.sdma.setup_ring(*(rcvr_params:=(ring.va_addr, ring.size, gart.va_addr+rptr, gart.va_addr+wptr, idx)))
|
||||
else:
|
||||
doorbell_index = self.dev_impl.gfx.setup_ring(*(rcvr_params:=(ring.va_addr, ring.size, gart.va_addr+rptr, gart.va_addr+wptr,
|
||||
eop_buffer.va_addr, eop_buffer.size, is_aql:=(queue_type==kfd.KFD_IOC_QUEUE_TYPE_COMPUTE_AQL), is_aql)))
|
||||
|
||||
ext = lambda addr,n,dt: Buffer("CPU", n, dt, options=BufferSpec(external_ptr=addr), preallocate=True)
|
||||
(put_value := Buffer("CPU", 1, dtypes.uint64, preallocate=True))._buf.view.view(fmt='Q')[0] = 0
|
||||
return AMDQueueDesc(ring=ext(ring.va_addr, ring.size//4, dtypes.uint32),
|
||||
doorbell=ext(self.dev_impl.doorbell64.addr + doorbell_index*8, 1, dtypes.uint64),
|
||||
read_ptr=ext(gart.va_addr+rptr, 1, dtypes.uint64), write_ptr=ext(gart.va_addr+wptr, 1, dtypes.uint64),
|
||||
put_value=put_value, params=rcvr_params)
|
||||
|
||||
def _collect_interrupts(self, reset=False, drain_only=False):
|
||||
d = self.dev
|
||||
if drain_only: d.iface.dev_impl.ih.drain()
|
||||
else: d.iface.dev_impl.ih.interrupt_handler()
|
||||
|
||||
if reset and d.iface.dev_impl.recover():
|
||||
cq = d.compute_queue
|
||||
for b in (cq.put_value, cq.read_ptr, cq.write_ptr): b._buf.view.view(fmt='Q')[0] = 0
|
||||
d.iface.dev_impl.gfx.setup_ring(*cq.params)
|
||||
d.timeline_signal._buf.cpu_view().mv.cast('Q')[0] = d.timeline_value.as_memoryview(force_zero_copy=True).cast('Q')[0] - 1
|
||||
|
||||
def sleep(self, timeout):
|
||||
if hasattr(self.pci_dev, 'irq_poller') and self.pci_dev.irq_poller is not None and (events_cnt:=len(self.pci_dev.irq_poller.poll(timeout))):
|
||||
self.pci_dev.irq_fd.read(8 * events_cnt)
|
||||
self._collect_interrupts()
|
||||
if self.dev_impl.is_err_state: raise RuntimeError("Device is in error state")
|
||||
|
||||
def on_device_hang(self):
|
||||
self._collect_interrupts(reset=True)
|
||||
raise RuntimeError("Device hang detected")
|
||||
|
||||
def device_fini(self): self.dev_impl.fini()
|
||||
|
||||
def _mock(iface, name=None): return type(name or f"MOCK{iface.__name__}", (iface,), {})
|
||||
|
||||
class AMDDevice(HCQ2Compiled):
|
||||
timestamp_divider = 100.0 # AMD GPU clock: ticks/us
|
||||
|
||||
pm_lower = PatternMatcher([
|
||||
(UPat(Ops.PROGRAM, src=(UPat(), UPat(), UPat(), UPat(), UPat(Ops.BINARY)), name="prg"), amd_build_program),
|
||||
(UPat(Ops.LINEAR, arg="COMPUTE", name="linear"), amd_lower_pm4),
|
||||
(UPat(Ops.LINEAR, arg="COPY", name="linear"), amd_lower_sdma),
|
||||
(UPat(Ops.CUSTOM_FUNCTION, arg="submit_compute", name="cf"), amd_submit_pm4),
|
||||
(UPat(Ops.CUSTOM_FUNCTION, arg="submit_copy", name="cf"), amd_submit_sdma),
|
||||
])
|
||||
|
||||
ifaces = [PCIIface]
|
||||
|
||||
def is_am(self) -> bool: return isinstance(self.iface, (PCIIface,))
|
||||
def is_usb(self) -> bool: return False
|
||||
|
||||
def __init__(self, device:str=""):
|
||||
self.device_id = int(device.split(":")[1]) if ":" in device else 0
|
||||
|
||||
self.iface = self._select_iface()
|
||||
|
||||
self.target:tuple[int, ...] = ((trgt:=self.iface.props['gfx_target_version']) // 10000, (trgt // 100) % 100, trgt % 100)
|
||||
self.arch = "gfx%d%x%x" % self.target
|
||||
assert (self.target in ((9,4,2),(9,5,0))) or self.target[0] in (11, 12), f"Unsupported arch: {self.arch}"
|
||||
if DEBUG >= 1: print(f"AMDDevice: opening {self.device_id} with target {self.target} arch {self.arch}")
|
||||
|
||||
self.xccs = self.iface.props.get('num_xcc', 1)
|
||||
self.se_cnt = self.iface.props['array_count'] // self.iface.props['simd_arrays_per_engine'] // self.xccs
|
||||
self.cu_cnt = self.iface.props['simd_count'] // self.iface.props['simd_per_cu'] // self.xccs
|
||||
self.waves_per_cu = self.iface.props['max_waves_per_simd'] * self.iface.props['simd_per_cu']
|
||||
self.wave_cnt = (self.cu_cnt * self.waves_per_cu) if self.target[0] != 9 else min(self.cu_cnt * 40, self.se_cnt * self.xccs * 512)
|
||||
|
||||
self.ip_off = importlib.import_module(f"tinygrad.runtime.autogen.am.{'vega' if self.target[0] == 9 else 'navi'}_offsets")
|
||||
self.soc = import_soc(self.target)
|
||||
self.pm4 = importlib.import_module(f"tinygrad.runtime.autogen.am.pm4_{'soc15' if self.target[0] == 9 else 'nv'}")
|
||||
self.sdma = import_module('sdma', min(self.iface.ip_versions[am.SDMA0_HWIP], (6, 0, 0)))
|
||||
self.gc = AMDIP('gc', self.iface.ip_versions[am.GC_HWIP],
|
||||
bases={i: tuple(getattr(self.ip_off, f'GC_BASE__INST{i}_SEG{s}', 0) for s in range(6)) for i in range(6)})
|
||||
|
||||
self.nbio = AMDIP('nbio' if self.target[0] < 12 else 'nbif', self.iface.ip_versions[am.NBIF_HWIP],
|
||||
bases={i: tuple(getattr(self.ip_off, f'NBIO_BASE__INST{i}_SEG{s}', 0) for s in range(9)) for i in range(6)})
|
||||
|
||||
self.is_aql = getenv("AMD_AQL", int(self.xccs > 1))
|
||||
if self.is_aql:
|
||||
self.pm4_ibs = self.iface.alloc(0x2000 if self.is_usb() else (16 << 20), uncached=True, cpu_access=True)
|
||||
self.pm4_ib_alloc = BumpAllocator(self.pm4_ibs.size, wrap=True)
|
||||
|
||||
self.max_copy_size = 0x40000000 if self.iface.ip_versions[am.SDMA0_HWIP][0] >= 5 else 0x400000
|
||||
self.sdma_queues:dict = {}
|
||||
self.has_sdma_queue = self.sdma_queue(0) is not None
|
||||
|
||||
super().__init__(device, AMDAllocator(self), [HIPRenderer, AMDLLVMRenderer, HIPCCRenderer], None,
|
||||
kernargs_size=16 << 20, can_recover=self.is_am(), arch=self.arch)
|
||||
|
||||
# Scratch setup
|
||||
self.max_private_segment_size = 0
|
||||
self._ensure_has_local_memory(128) # set default scratch size to 128 bytes per thread
|
||||
|
||||
self.pmc_enabled:bool = PROFILE > 0 and PMC > 0
|
||||
if self.pmc_enabled:
|
||||
self.iface.require_profile_mode()
|
||||
|
||||
self.pmc_sched:list[PMCSample] = []
|
||||
self.pmc_counters = import_pmc(self.target)
|
||||
|
||||
# validate counters: SQ for SIMD busy/instruction counts, LDS stats, GRBM for GPU cycles, L2 cache hits/misses
|
||||
l2, lds = ("TCC", "SQ") if self.target[0] == 9 else ("GL2C", "SQC")
|
||||
pmc_default = f"SQ_BUSY_CYCLES,SQ_INSTS_VALU,SQ_INSTS_SALU,{lds}_LDS_IDX_ACTIVE,{lds}_LDS_BANK_CONFLICT,GRBM_GUI_ACTIVE,{l2}_HIT,{l2}_MISS"
|
||||
for k in (PMC_COUNTERS:=getenv("PMC_COUNTERS", pmc_default).split(",")):
|
||||
if k not in self.pmc_counters: raise RuntimeError(f"PMC counter {k} is not supported. Available: {','.join(self.pmc_counters.keys())}")
|
||||
|
||||
raise NotImplementedError("PMC start not migrated to hcq2 yet")
|
||||
|
||||
# SQTT is disabled by default because of runtime overhead and big file sizes (~200mb to Tensor.full() two 4096x4096 tensors and matmul them)
|
||||
self.sqtt_enabled:bool = PROFILE > 0 and SQTT > 0
|
||||
if self.sqtt_enabled:
|
||||
self.iface.require_profile_mode()
|
||||
|
||||
SQTT_BUFFER_SIZE = getenv("SQTT_BUFFER_SIZE", 256) # in mb, per shader engine
|
||||
self.sqtt_buffers = [self.allocator.alloc(SQTT_BUFFER_SIZE<<20, BufferSpec(nolru=True, uncached=True)) for _ in range(self.se_cnt * self.xccs)]
|
||||
self.sqtt_wptrs = self.allocator.alloc(round_up(self.se_cnt * self.xccs * 4, 0x1000), BufferSpec(cpu_access=True, nolru=True))
|
||||
self.sqtt_next_cmd_id = itertools.count(0)
|
||||
|
||||
@functools.cached_property
|
||||
def compute_queue(self) -> AMDQueueDesc:
|
||||
# https://gitlab.freedesktop.org/agd5f/linux/-/blob/a1fc9f584c4aaf8bc1ebfa459fc57a3f26a290d8/drivers/gpu/drm/amd/amdkfd/kfd_queue.c#L391
|
||||
sgrp_size_per_cu, hwreg_size_per_cu = 0x4000, 0x1000
|
||||
lds_size_per_cu = self.iface.props["lds_size_in_kb"] << 10 if self.target[:2] == (9,5) else 0x10000
|
||||
vgpr_size_per_cu = 0x60000 if self.target in {(11,0,0), (11,0,1), (11,5,1), (12,0,0), (12,0,1)} else 0x80000 if self.target[0] == 9 else 0x40000
|
||||
wg_data_size = round_up((vgpr_size_per_cu + sgrp_size_per_cu + lds_size_per_cu + hwreg_size_per_cu) * self.cu_cnt, mmap.PAGESIZE)
|
||||
ctl_stack_size = round_up((12 if self.target[0] != 9 else 8) * self.wave_cnt + 8 + 40, mmap.PAGESIZE)
|
||||
return self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_COMPUTE_AQL if self.is_aql else kfd.KFD_IOC_QUEUE_TYPE_COMPUTE,
|
||||
0x2000 if self.is_usb() else (16 << 20), eop_buffer_size=0x1000,
|
||||
ctx_save_restore_size=0 if self.is_am() else wg_data_size + ctl_stack_size, ctl_stack_size=ctl_stack_size,
|
||||
debug_memory_size=round_up(self.wave_cnt * 32, 64))
|
||||
|
||||
def create_queue(self, queue_type, ring_size, ctx_save_restore_size=0, eop_buffer_size=0, ctl_stack_size=0, debug_memory_size=0, idx=0):
|
||||
ring = self.iface.alloc(ring_size, uncached=True, cpu_access=True)
|
||||
gart = self.iface.alloc(0x100, uncached=True, cpu_access=True)
|
||||
|
||||
if queue_type == kfd.KFD_IOC_QUEUE_TYPE_COMPUTE_AQL:
|
||||
self.aql_gart = gart
|
||||
self.aql_desc = hsa.amd_queue_t(queue_properties=hsa.AMD_QUEUE_PROPERTIES_IS_PTR64 | hsa.AMD_QUEUE_PROPERTIES_ENABLE_PROFILING,
|
||||
read_dispatch_id_field_base_byte_offset=getattr(hsa.amd_queue_t, 'read_dispatch_id').offset,
|
||||
max_cu_id=(self.cu_cnt * self.xccs) - 1, max_wave_id=self.waves_per_cu - 1)
|
||||
self.aql_gart.cpu_view().view(fmt='B')[:ctypes.sizeof(self.aql_desc)] = bytes(self.aql_desc)
|
||||
|
||||
cwsr_buffer_size = round_up((ctx_save_restore_size + debug_memory_size) * self.xccs, mmap.PAGESIZE)
|
||||
cwsr_buffer = self.iface.alloc(cwsr_buffer_size) if ctx_save_restore_size else None
|
||||
eop_buffer = self.iface.alloc(eop_buffer_size) if eop_buffer_size else None
|
||||
|
||||
return (self.iface.create_queue(queue_type, ring, gart, rptr=getattr(hsa.amd_queue_t, 'read_dispatch_id').offset,
|
||||
wptr=getattr(hsa.amd_queue_t, 'write_dispatch_id').offset, eop_buffer=eop_buffer, cwsr_buffer=cwsr_buffer,
|
||||
ctx_save_restore_size=ctx_save_restore_size, ctl_stack_size=ctl_stack_size, idx=idx))
|
||||
|
||||
def sdma_queue(self, idx:int):
|
||||
if getenv("AMD_DISABLE_SDMA"): return None
|
||||
if idx in self.sdma_queues: return self.sdma_queues[idx]
|
||||
with contextlib.suppress(OSError):
|
||||
self.sdma_queues[idx] = self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_SDMA, 0x200 if self.is_usb() else (16 << 20), idx=idx)
|
||||
return self.sdma_queues.get(idx, None)
|
||||
|
||||
def _ensure_has_local_memory(self, private_segment_size):
|
||||
if self.max_private_segment_size >= private_segment_size: return
|
||||
|
||||
lanes_per_wave = 64 # wave64
|
||||
mem_alignment_size = 256 if self.target[0] != 9 else 1024
|
||||
size_per_thread = round_up(private_segment_size, mem_alignment_size // lanes_per_wave)
|
||||
size_per_xcc = size_per_thread * lanes_per_wave * self.iface.props['max_slots_scratch_cu'] * self.cu_cnt
|
||||
self.scratch, ok = self._realloc(getattr(self, 'scratch', None), size_per_xcc * self.xccs)
|
||||
if ok:
|
||||
# NOTE: xcc logic is correct only for GFX9.
|
||||
max_scratch_waves = self.cu_cnt * self.iface.props['max_slots_scratch_cu'] * self.xccs
|
||||
wave_scratch = ceildiv(lanes_per_wave * size_per_thread, mem_alignment_size)
|
||||
num_waves = (size_per_xcc // (wave_scratch * mem_alignment_size)) // (self.se_cnt if self.target[0] != 9 else 1)
|
||||
|
||||
tmpring_t = getattr(hsa, f'union_COMPUTE_TMPRING_SIZE{"_GFX"+str(self.target[0]) if self.target[0] != 9 else ""}_bitfields')
|
||||
self.tmpring_size = int.from_bytes(tmpring_t(WAVES=min(num_waves, max_scratch_waves), WAVESIZE=wave_scratch), 'little')
|
||||
self.max_private_segment_size = private_segment_size
|
||||
|
||||
if hasattr(self, 'aql_desc'):
|
||||
gfx9_rsrc = {'NUM_FORMAT':hsa.BUF_NUM_FORMAT_UINT, 'DATA_FORMAT':hsa.BUF_DATA_FORMAT_32, 'ELEMENT_SIZE':1, 'INDEX_STRIDE':3}
|
||||
rsrc = {'DST_SEL_X':hsa.SQ_SEL_X, 'DST_SEL_Y':hsa.SQ_SEL_Y, 'DST_SEL_Z':hsa.SQ_SEL_Z, 'DST_SEL_W':hsa.SQ_SEL_W, 'ADD_TID_ENABLE':1,
|
||||
'TYPE':hsa.SQ_RSRC_BUF, **(gfx9_rsrc if self.target[0] == 9 else {'FORMAT':hsa.BUF_FORMAT_32_UINT, 'OOB_SELECT':2})}
|
||||
rsrc1_t = getattr(hsa, f'union_SQ_BUF_RSRC_WORD1{"_GFX11" if self.target[0] != 9 else ""}_bitfields')
|
||||
rsrc3_t = getattr(hsa, f'union_SQ_BUF_RSRC_WORD3{"_GFX"+str(self.target[0]) if self.target[0] != 9 else ""}_bitfields')
|
||||
|
||||
self.aql_desc.scratch_backing_memory_location = int(self.scratch.va_addr)
|
||||
self.aql_desc.scratch_wave64_lane_byte_size = self.max_private_segment_size * lanes_per_wave // 64
|
||||
self.aql_desc.scratch_resource_descriptor[:] = [lo32(self.scratch.va_addr),
|
||||
int.from_bytes(rsrc1_t(BASE_ADDRESS_HI=hi32(self.scratch.va_addr), SWIZZLE_ENABLE=1), 'little'),
|
||||
lo32(size_per_xcc), int.from_bytes(bytes(rsrc3_t(**rsrc)), 'little')]
|
||||
self.aql_desc.compute_tmpring_size = self.tmpring_size
|
||||
self.aql_gart.cpu_view()[:ctypes.sizeof(self.aql_desc)] = bytes(self.aql_desc)
|
||||
|
||||
def on_device_hang(self): self.iface.on_device_hang()
|
||||
|
||||
def device_props(self): return self.iface.props
|
||||
Reference in new issue
Block a user