mirror of
https://github.com/Helldez/BigMoeOnEdge.git
synced 2026-10-19 15:33:46 +00:00
Thresholds that decided something as bare literals now have a name and a reason: the lane tie fraction behind io-threads, the rate-spread warning and the density tie tolerance in the allocator, and the form bounds of the parameter table. The two device-bandwidth comparisons that spelled the backend margin as 1.10 read PlannerPolicy::min_backend_win. Removed, unused: is_known(Tri), StorageFacts::mapping_penalty(), ModelProfile::expert_working_set_bytes(), Lane::MmapWarm. Plan::to_flags() is rendered from to_args() instead of a hand-written list, so the reproduce line carries every non-default parameter of the plan - the device choice and the fitter's tensor overrides were missing from it - in table order, with arguments quoted for a shell. Comments corrected where they said something the code does not do: how a device name is used, who includes llama.cpp's common library, and how much the storage probe reads.
1404 lines
80 KiB
C++
1404 lines
80 KiB
C++
// Unit tests for the hardware planner (core/src/plan/planner.cpp).
|
|
//
|
|
// The planner is a pure function, and this is the whole reason it is one: every machine below is a
|
|
// struct, so rules meant to hold on hardware nobody here owns can still be checked. No model, no
|
|
// llama.cpp, no device — it runs unconditionally in ctest.
|
|
//
|
|
// The machines are FIXTURES shaped after our measured verdicts (a desktop whose mapping serialises
|
|
// uncached reads; a phone that compresses reclaimed memory and offers a reclaim-exempt allocation;
|
|
// a platform that kills a process for holding too much and does not count mapped file pages). They
|
|
// are not measurements and nothing here should be quoted as one — what is being tested is that the
|
|
// rules read the facts, not that the facts are these.
|
|
//
|
|
// Checks are explicit (not <cassert>): the Release build defines NDEBUG.
|
|
|
|
#include "bmoe/config.h"
|
|
#include "bmoe/allocate.h"
|
|
#include "bmoe/planner.h"
|
|
#include "bmoe/probe.h"
|
|
|
|
#include <cstdio>
|
|
#include <string>
|
|
|
|
using namespace bmoe;
|
|
|
|
static int failures = 0;
|
|
|
|
static void check(bool cond, const std::string & name, const std::string & detail = "") {
|
|
if (cond) {
|
|
std::printf("[PASS] %s\n", name.c_str());
|
|
} else {
|
|
std::printf("[FAIL] %s%s%s\n", name.c_str(), detail.empty() ? "" : " - ", detail.c_str());
|
|
++failures;
|
|
}
|
|
}
|
|
|
|
static const uint64_t KiB = 1024ull;
|
|
static const uint64_t MiB = 1024ull * 1024ull;
|
|
static const uint64_t GiB = 1024ull * 1024ull * 1024ull;
|
|
|
|
// ── the model ───────────────────────────────────────────────────────────────────────
|
|
// Shaped after a 35B-A3B Q4_K_M: 48 layers, 256 experts, top-8, split gate/up/down. The slice size
|
|
// is chosen so the derived token cycle lands where we measured one (about 582 MiB), which is what
|
|
// makes the floor assertions below meaningful rather than arbitrary.
|
|
static ModelProfile moe_model() {
|
|
ModelProfile m;
|
|
m.arch = "qwen3moe";
|
|
m.is_moe = true;
|
|
m.n_layer = 48;
|
|
m.n_moe_layer = 48; // no leading dense blocks in this family: every block carries experts
|
|
m.n_expert = 256;
|
|
m.n_expert_used = 8;
|
|
m.n_expert_projections = 3;
|
|
m.expert_slice_bytes = 530432; // 518 KiB, the read the streamer issues
|
|
// What the probe would sum over the file's own tensors: 48 layers x top-8 x 3 projections.
|
|
m.token_cycle_bytes = (uint64_t) 48 * 8 * 3 * 530432;
|
|
m.file_bytes = 21 * GiB;
|
|
m.expert_bytes = 18 * GiB;
|
|
m.dense_bytes = 3 * GiB;
|
|
m.largest_dense_tensor = 243 * MiB;
|
|
|
|
// The decomposition the cost model prices. Shaped so the totals agree with the file: the dense
|
|
// groups are read whole every token, the embedding is gathered by row (so its per-token demand
|
|
// is a few KiB however large the table is), and the experts demand exactly one token cycle.
|
|
m.group(WeightGroup::Attention) = {1400 * MiB, 1400 * MiB, false, false};
|
|
m.group(WeightGroup::DenseFfn) = {400 * MiB, 400 * MiB, false, false};
|
|
m.group(WeightGroup::Output) = {700 * MiB, 700 * MiB, false, false};
|
|
m.group(WeightGroup::Other) = {228 * MiB, 228 * MiB, false, false};
|
|
m.group(WeightGroup::Embedding) = {344 * MiB, 8 * KiB, true, true};
|
|
m.group(WeightGroup::Experts) = {18 * GiB, m.token_cycle_bytes, false, true};
|
|
m.ok = true;
|
|
return m;
|
|
}
|
|
|
|
// ── the machines ────────────────────────────────────────────────────────────────────
|
|
static StorageFacts desktop_storage() {
|
|
StorageFacts s;
|
|
s.align = 4096;
|
|
s.direct_ok = Tri::Yes;
|
|
s.mapping_serialises_reads = Tri::Yes;
|
|
s.rate_curve = {
|
|
{4 * (uint32_t) KiB, 1, 32.8}, {64 * (uint32_t) KiB, 1, 341.1}, {256 * (uint32_t) KiB, 1, 784.6},
|
|
{576 * (uint32_t) KiB, 2, 1600.0}, {576 * (uint32_t) KiB, 4, 2400.0},
|
|
};
|
|
return s;
|
|
}
|
|
|
|
static HardwareProfile desktop() {
|
|
HardwareProfile h;
|
|
h.label = "desktop, swap-backed";
|
|
h.residency_budget = 12 * GiB;
|
|
h.memory_total = 16 * GiB;
|
|
h.anon_overflow = Overflow::Swap;
|
|
h.reclaim_exempt_max = 0;
|
|
h.file_pages_counted = Tri::Yes;
|
|
h.n_cores = 16;
|
|
h.storage = desktop_storage();
|
|
ComputeDevice gpu;
|
|
gpu.name = "discrete";
|
|
gpu.memory_total = 8 * GiB;
|
|
gpu.memory_free = 7 * GiB;
|
|
gpu.shares_host_memory = Tri::No;
|
|
gpu.rebindable = false;
|
|
h.devices.push_back(gpu);
|
|
return h;
|
|
}
|
|
|
|
static HardwareProfile phone() {
|
|
HardwareProfile h;
|
|
h.label = "phone, compressed reclaim";
|
|
h.residency_budget = 6 * GiB;
|
|
h.memory_total = 12 * GiB;
|
|
h.anon_overflow = Overflow::Compress;
|
|
h.reclaim_exempt_max = 2047 * MiB;
|
|
// What this class of machine answers: a store that publishes no total, and a lock-in-place
|
|
// limit of a few pages. The dense set can be protected and the cache cannot.
|
|
h.lockable_bytes = ~0ull;
|
|
h.lock_in_place_bytes = 64 * KiB;
|
|
h.file_pages_counted = Tri::Yes;
|
|
h.n_cores = 8;
|
|
h.storage.align = 4096;
|
|
h.storage.direct_ok = Tri::Yes;
|
|
h.storage.mapping_serialises_reads = Tri::No;
|
|
h.storage.rate_curve = {
|
|
{4 * (uint32_t) KiB, 1, 31.3}, {4 * (uint32_t) KiB, 4, 156.6}, {256 * (uint32_t) KiB, 1, 901.0},
|
|
{256 * (uint32_t) KiB, 2, 1600.0}, {256 * (uint32_t) KiB, 4, 2201.5},
|
|
};
|
|
ComputeDevice igpu;
|
|
igpu.name = "integrated";
|
|
igpu.memory_total = 6 * GiB;
|
|
igpu.shares_host_memory = Tri::Yes; // moving a tensor off it frees nothing
|
|
h.devices.push_back(igpu);
|
|
return h;
|
|
}
|
|
|
|
// A platform that answers memory pressure by killing the process, and does not count clean mapped
|
|
// file pages against that limit.
|
|
static HardwareProfile hard_capped() {
|
|
HardwareProfile h;
|
|
h.label = "hard per-process cap";
|
|
h.residency_budget = 5 * GiB;
|
|
h.memory_total = 8 * GiB;
|
|
h.anon_overflow = Overflow::Kill;
|
|
h.reclaim_exempt_max = 0;
|
|
h.file_pages_counted = Tri::No;
|
|
h.n_cores = 6;
|
|
h.storage.align = 0;
|
|
h.storage.direct_ok = Tri::Yes;
|
|
h.storage.mapping_serialises_reads = Tri::Unknown;
|
|
return h;
|
|
}
|
|
|
|
// One pool of memory for the cores and the accelerator, a compressing reclaim, and a lock the
|
|
// kernel grants to any process up to a system-wide total - the same store whether the memory is
|
|
// locked as it is allocated or where it already sits.
|
|
static HardwareProfile unified() {
|
|
HardwareProfile h = phone();
|
|
h.label = "unified memory, a system-wide lock total";
|
|
h.residency_budget = 13 * GiB;
|
|
h.memory_total = 16 * GiB;
|
|
h.reclaim_exempt_max = 12 * GiB;
|
|
h.lockable_bytes = 10 * GiB;
|
|
h.lock_in_place_bytes = 10 * GiB;
|
|
h.devices.clear();
|
|
ComputeDevice cpu;
|
|
cpu.name = "CPU";
|
|
cpu.is_cpu = true;
|
|
cpu.shares_host_memory = Tri::Yes;
|
|
cpu.buffers_locked = Tri::No;
|
|
h.devices.push_back(cpu);
|
|
ComputeDevice accel;
|
|
accel.name = "accel";
|
|
accel.shares_host_memory = Tri::Yes;
|
|
accel.buffers_locked = Tri::Yes;
|
|
h.devices.push_back(accel);
|
|
return h;
|
|
}
|
|
|
|
static const LedgerRow * ledger_row(const Plan & p, const char * name) {
|
|
for (const LedgerRow & r : p.allocation.ledger.rows)
|
|
if (r.name == name) return &r;
|
|
return nullptr;
|
|
}
|
|
|
|
static RunConfig base_cfg() {
|
|
RunConfig c;
|
|
c.model_path = "model.gguf";
|
|
return c;
|
|
}
|
|
|
|
static const Decision * find(const Plan & p, const char * knob) {
|
|
for (const Decision & d : p.decisions)
|
|
if (d.knob == knob) return &d;
|
|
return nullptr;
|
|
}
|
|
|
|
int main() {
|
|
const ModelProfile model = moe_model();
|
|
|
|
// The derived floor is the model's own arithmetic, not a constant carried from one device.
|
|
check(model.token_cycle_bytes / MiB == 582, "token cycle derives from shapes",
|
|
std::to_string((unsigned long long) (model.token_cycle_bytes / MiB)) + " MiB");
|
|
|
|
// ── desktop ─────────────────────────────────────────────────────────────────────
|
|
{
|
|
const Plan p = plan_run(base_cfg(), desktop(), model, PlanRequest{});
|
|
check(p.regime == Regime::ExpertsStream, "desktop: experts stream, dense fits");
|
|
check(p.config.moe.enabled, "desktop: streaming on");
|
|
check(p.config.moe.dense_weights == DenseWeightsMode::Anonymous,
|
|
"desktop: dense anon (no reclaim-exempt store here)");
|
|
check(p.config.moe.release_mmap, "desktop: release-mmap on (the mapping serialises reads)");
|
|
check(p.config.moe.io_threads == 4, "desktop: lanes from the rate curve",
|
|
std::to_string(p.config.moe.io_threads));
|
|
check(p.config.moe.cache_mb > 0 && (uint64_t) p.config.moe.cache_mb * MiB >= model.token_cycle_bytes,
|
|
"desktop: cache clears the token cycle", std::to_string(p.config.moe.cache_mb) + " MiB");
|
|
check(validate(p.config).ok, "desktop: the plan is a valid config", validate(p.config).error);
|
|
|
|
const Decision * d = find(p, "release-mmap");
|
|
check(d && d->source == Source::Measured, "desktop: release-mmap is a measured decision");
|
|
}
|
|
|
|
// ── phone ───────────────────────────────────────────────────────────────────────
|
|
{
|
|
const Plan p = plan_run(base_cfg(), phone(), model, PlanRequest{});
|
|
check(p.regime == Regime::ExpertsStream, "phone: experts stream");
|
|
// The flow DERIVES the pinned dense policy from the mechanism: anonymous memory here is
|
|
// compressed away, and a reclaim-exempt store exists. It is not copied from what we ship.
|
|
check(p.config.moe.dense_weights == DenseWeightsMode::Pinned,
|
|
"phone: dense pinned, derived from compressed reclaim + a reclaim-exempt store");
|
|
check(!p.config.moe.release_mmap, "phone: release-mmap off (no serialisation measured here)");
|
|
check(p.config.moe.io_threads == 4, "phone: lanes from the rate curve",
|
|
std::to_string(p.config.moe.io_threads));
|
|
check(validate(p.config).ok, "phone: the plan is a valid config", validate(p.config).error);
|
|
|
|
const Decision * d = find(p, "dense-weights");
|
|
check(d && d->source == Source::Derived, "phone: dense policy is derived, not defaulted");
|
|
}
|
|
|
|
// ── a platform that kills instead of reclaiming ─────────────────────────────────
|
|
{
|
|
const Plan p = plan_run(base_cfg(), hard_capped(), model, PlanRequest{});
|
|
// The same rule that picks anon elsewhere picks mmap here, because mapped file pages sit
|
|
// outside the limit that can actually end the run. One rule, opposite answers, no branch.
|
|
check(p.config.moe.dense_weights == DenseWeightsMode::Mmap,
|
|
"hard cap: dense left mapped, because file pages are outside the fatal limit");
|
|
check(!p.config.moe.release_mmap, "hard cap: release-mmap stays off while the dense set is mapped");
|
|
check(validate(p.config).ok, "hard cap: the plan is a valid config", validate(p.config).error);
|
|
}
|
|
|
|
// ── a machine the model fits on: streaming is not a win, it is a cost ───────────
|
|
{
|
|
HardwareProfile big = desktop();
|
|
big.residency_budget = 64 * GiB;
|
|
big.memory_total = 64 * GiB;
|
|
const Plan p = plan_run(base_cfg(), big, model, PlanRequest{});
|
|
check(p.regime == Regime::Fits, "roomy machine: regime is fits");
|
|
check(!p.config.moe.enabled, "roomy machine: streaming off");
|
|
check(p.streaming_declined && !p.decline_reason.empty(), "roomy machine: the refusal is explained");
|
|
}
|
|
|
|
// ── the budget is what we can KEEP, not what is reported available ──────────────
|
|
// On a machine whose reclaim compresses, the reported figure is a floor: the kernel will
|
|
// compress other processes' idle pages to make room. Sizing from it leaves memory unused, and
|
|
// the whole point of measuring is that the plan then spends what is really there.
|
|
{
|
|
const Plan reported = plan_run(base_cfg(), phone(), model, PlanRequest{});
|
|
HardwareProfile measured = phone();
|
|
measured.holdable_bytes = 8 * GiB; // more than the 6 GiB it reports available
|
|
measured.holdable_from = Headroom::Measured;
|
|
const Plan p = plan_run(base_cfg(), measured, model, PlanRequest{});
|
|
check(p.config.moe.cache_mb > reported.config.moe.cache_mb,
|
|
"measured headroom: the cache grows with what the machine will actually let us keep",
|
|
std::to_string(reported.config.moe.cache_mb) + " -> " + std::to_string(p.config.moe.cache_mb));
|
|
const Decision * d = find(p, "regime");
|
|
check(d && d->reason.find("measured by holding memory") != std::string::npos,
|
|
"measured headroom: the rationale says the budget was measured", d ? d->reason : "no decision");
|
|
}
|
|
|
|
// An estimate is still better than the reported floor, and is labelled as an estimate so a
|
|
// reader can weigh it: the two are not the same claim.
|
|
{
|
|
HardwareProfile est = phone();
|
|
est.holdable_bytes = 7 * GiB;
|
|
est.holdable_from = Headroom::Compressible;
|
|
const Plan p = plan_run(base_cfg(), est, model, PlanRequest{});
|
|
const Decision * d = find(p, "regime");
|
|
check(d && d->reason.find("estimated from") != std::string::npos,
|
|
"estimated headroom: the rationale says the budget was estimated", d ? d->reason : "no decision");
|
|
}
|
|
|
|
// An unmeasured machine falls back to the reported figure and says which one it used, rather
|
|
// than pretending the question was answered.
|
|
{
|
|
const Plan p = plan_run(base_cfg(), phone(), model, PlanRequest{});
|
|
const Decision * d = find(p, "regime");
|
|
check(d && d->reason.find("unmeasured") != std::string::npos,
|
|
"unmeasured headroom: the rationale admits the budget is the reported one",
|
|
d ? d->reason : "no decision");
|
|
}
|
|
|
|
// ── fitting is not being left alone: the same model, two machines that both "fit" ──
|
|
// A phone whose reclaim compresses, holding a model that fits with little beyond itself. It is
|
|
// the common case, not the exotic one, and residency there is reclaimed from underneath.
|
|
{
|
|
HardwareProfile barely = phone();
|
|
barely.residency_budget = 25 * GiB; // the 21 GiB model fits, with about 2 GiB beyond it
|
|
const Plan p = plan_run(base_cfg(), barely, model, PlanRequest{});
|
|
check(p.regime == Regime::ExpertsStream, "fits barely: streams instead of trusting residency");
|
|
check(p.config.moe.enabled, "fits barely: streaming on");
|
|
const Decision * d = find(p, "residency");
|
|
check(d && d->source == Source::Derived, "fits barely: the demotion is a derived decision");
|
|
check(d && d->reason.find("reclaim is cheap") != std::string::npos,
|
|
"fits barely: the reason names the machine's reclaim", d ? d->reason : "no decision");
|
|
}
|
|
|
|
// The same shape on a machine where nothing is taken from us is left alone: a hard per-process
|
|
// cap is explicit, and the margin for it is already the largest of the three. (27 GiB, because
|
|
// that margin is 20%: the model has to clear the bar before the air rule is even reached.)
|
|
{
|
|
HardwareProfile barely = hard_capped();
|
|
barely.residency_budget = 27 * GiB;
|
|
barely.memory_total = 32 * GiB;
|
|
const Plan p = plan_run(base_cfg(), barely, model, PlanRequest{});
|
|
check(p.regime == Regime::Fits, "fits barely under a hard cap: still resident, nothing reclaims us");
|
|
check(!p.config.moe.enabled, "fits barely under a hard cap: streaming stays off");
|
|
}
|
|
|
|
// And where a reclaim has to write to a disk the kernel is reluctant enough that the classic
|
|
// host offload stands: this is the desktop shape, and it is checked in the placement case below.
|
|
// A reclaim-exempt machine likewise: what cannot be taken back does not have to be defended.
|
|
{
|
|
HardwareProfile barely = phone();
|
|
barely.residency_budget = 25 * GiB;
|
|
barely.anon_overflow = Overflow::None;
|
|
const Plan p = plan_run(base_cfg(), barely, model, PlanRequest{});
|
|
check(p.regime == Regime::Fits, "fits barely with nothing to reclaim it: resident");
|
|
}
|
|
|
|
// ── too little room for one token's worth of experts ───────────────────────────
|
|
// The floor turns the CACHE off. The stream stays, because the memory a cache lacks is the
|
|
// memory the page cache lacks too: the mapped file is at its worst exactly here.
|
|
{
|
|
HardwareProfile tight = phone();
|
|
tight.residency_budget = 3 * GiB + 400 * MiB; // dense fits, what is left is under the cycle
|
|
tight.storage.refault_mibs = 24.0; // measured, so the mapping has a cost to beat
|
|
const Plan p = plan_run(base_cfg(), tight, model, PlanRequest{});
|
|
check(p.config.moe.enabled && !p.streaming_declined,
|
|
"tight machine: the experts stream with no cache rather than being left to refault", p.decline_reason);
|
|
check(p.config.moe.cache_mb == 0 && !p.config.moe.cache_auto && !p.config.moe.force_cache,
|
|
"tight machine: the cache is off outright, not left to size itself under the floor",
|
|
std::to_string(p.config.moe.cache_mb) + " MiB");
|
|
const Decision * c = find(p, "cache-mb");
|
|
check(c && c->reason.find("token cycle") != std::string::npos,
|
|
"tight machine: the reason names the derived floor", c ? c->reason : "no cache-mb decision");
|
|
check(validate(p.config).ok, "tight machine: the plan is a valid config", validate(p.config).error);
|
|
|
|
const Allocation & a = p.allocation;
|
|
const GroupPlacement & e = a.at(WeightGroup::Experts);
|
|
check(e.lane == Lane::ExpertStream && e.resident_bytes == 0 && a.cache_bytes == 0,
|
|
"tight machine: the expert lane is the stream, holding nothing");
|
|
check(e.seconds_per_token > 0.0 && e.seconds_per_token < e.seconds_worst_case,
|
|
"tight machine: the stream is chosen on its price against the mapping",
|
|
std::to_string(e.seconds_per_token) + " vs " + std::to_string(e.seconds_worst_case));
|
|
check(ledger_row(p, "dense") && !ledger_row(p, "expert cache"),
|
|
"tight machine: the ledger keeps the dense set and carries no cache");
|
|
check(p.to_flags().find("--cache-mb 0") != std::string::npos,
|
|
"tight machine: the reproduce line states the cache is off, which a front-end would otherwise size",
|
|
p.to_flags());
|
|
|
|
// The same machine with a drive nobody measured. There is no price for the stream, so it is
|
|
// not chosen on faith: the plan declines, and is priced as the mapping it leaves.
|
|
HardwareProfile blind = tight;
|
|
blind.storage.rate_curve.clear();
|
|
const Plan q = plan_run(base_cfg(), blind, model, PlanRequest{});
|
|
check(!q.config.moe.enabled && q.streaming_declined,
|
|
"tight machine, unmeasured drive: streaming declined rather than guessed");
|
|
check(q.decline_reason.find("token cycle") != std::string::npos,
|
|
"tight machine, unmeasured drive: the reason names the derived floor", q.decline_reason);
|
|
|
|
// A declined run streams nothing and holds nothing: the file stays mapped, so the plan is
|
|
// priced as that rather than as the allocation it has just refused.
|
|
const Allocation & d = q.allocation;
|
|
check(d.seconds_worst_case > 0.0 && d.seconds_per_token == d.seconds_worst_case,
|
|
"tight machine, unmeasured drive: a declined plan predicts the cost of the mapping it leaves",
|
|
std::to_string(d.seconds_per_token) + " vs " + std::to_string(d.seconds_worst_case));
|
|
bool any_held = d.resident_bytes != 0 || d.cache_bytes != 0;
|
|
bool all_mapped = true;
|
|
for (const GroupPlacement & g : d.groups) {
|
|
any_held = any_held || g.resident_bytes != 0;
|
|
all_mapped = all_mapped && g.lane == Lane::Mmap;
|
|
}
|
|
check(!any_held && all_mapped,
|
|
"tight machine, unmeasured drive: a declined plan holds no weights and streams none");
|
|
check(!ledger_row(q, "dense") && !ledger_row(q, "expert cache"),
|
|
"tight machine, unmeasured drive: the ledger of a declined plan carries no weights");
|
|
}
|
|
|
|
// ── threads: the classes decide, not the count ─────────────────────────────────
|
|
{
|
|
HardwareProfile het = desktop();
|
|
het.n_cores = 20;
|
|
het.core_classes = {10, 10}; // ten fast, ten slow: a barrier waits for the slow ones
|
|
const Plan p = plan_run(base_cfg(), het, model, PlanRequest{});
|
|
check(p.config.n_threads == base_cfg().n_threads,
|
|
"heterogeneous cores alone: the count is NOT derived - that rule was refuted on a real phone",
|
|
std::to_string(p.config.n_threads));
|
|
check(p.config.n_threads_batch == 0, "heterogeneous cores: prefill keeps the default too");
|
|
check(validate(p.config).ok, "heterogeneous cores: the plan is a valid config", validate(p.config).error);
|
|
const Decision * d = find(p, "threads");
|
|
check(d && d->source == Source::Unprobed && d->reason.find("refuted") != std::string::npos,
|
|
"heterogeneous cores: the classes are reported, the rule is not applied",
|
|
d ? d->reason.substr(0, 60) : "none");
|
|
}
|
|
{
|
|
HardwareProfile uniform = desktop();
|
|
uniform.core_classes = {8};
|
|
const Plan p = plan_run(base_cfg(), uniform, model, PlanRequest{});
|
|
check(p.config.n_threads_batch == 0, "uniform cores: prefill keeps the same count as decode");
|
|
const Decision * d = find(p, "threads");
|
|
check(d && d->source == Source::Derived && d->reason.find("alike") != std::string::npos,
|
|
"uniform cores: nothing to improve, and the plan says why");
|
|
}
|
|
{
|
|
const Plan p = plan_run(base_cfg(), desktop(), model, PlanRequest{}); // no classes reported
|
|
const Decision * d = find(p, "threads");
|
|
check(d && d->source == Source::Unprobed, "unreported core classes: the default stands, unprobed");
|
|
}
|
|
|
|
// ── on shared memory, what the fitter placed is charged to the same pool ───────
|
|
// The fitter's capacity arithmetic treats device memory as separate. On a machine where it is
|
|
// the host's own, inheriting that gives a budget for memory already spoken for - which is how a
|
|
// phone was told an integrated accelerator had 15 GB free on an 11 GB device.
|
|
{
|
|
HardwareProfile uma = phone();
|
|
uma.devices.clear();
|
|
ComputeDevice cpu;
|
|
cpu.name = "CPU";
|
|
cpu.is_cpu = true;
|
|
cpu.shares_host_memory = Tri::Yes;
|
|
uma.devices.push_back(cpu);
|
|
ComputeDevice ig;
|
|
ig.name = "integrated";
|
|
ig.shares_host_memory = Tri::Yes;
|
|
uma.devices.push_back(ig);
|
|
uma.residency_budget = 8 * GiB;
|
|
|
|
Placement pl;
|
|
pl.fitted = true;
|
|
pl.n_gpu_layers = 0;
|
|
pl.n_ctx = 2048;
|
|
for (uint32_t il = 0; il < 48; ++il)
|
|
pl.host_expert_layers.push_back(il);
|
|
pl.host_resident_bytes = 1 * GiB;
|
|
pl.device_bytes = 4 * GiB; // the same memory, which the fitter counted twice
|
|
|
|
const Plan p = plan_run(base_cfg(), uma, model, pl, PlanRequest{});
|
|
const Decision * d = find(p, "shared-pool");
|
|
check(d && d->source == Source::Derived, "shared memory: the device bytes are charged and said");
|
|
const Decision * c = find(p, "cache-mb");
|
|
// Charged: 1 + 4 GiB is held before anything is sized, so the cache cannot be sized as if
|
|
// only 1 GiB were. Without the charge it would have been GiBs larger.
|
|
check(!c || (uint64_t) c->value.size() > 0, "shared memory: a cache decision was still reached");
|
|
check(p.cache_budget_bytes < 3 * GiB, "shared memory: the cache is sized against the real pool",
|
|
std::to_string((unsigned long long) (p.cache_budget_bytes >> 20)) + " MiB");
|
|
}
|
|
|
|
// A device with memory of its own is not charged: the two pools really are separate.
|
|
{
|
|
HardwareProfile disc = desktop();
|
|
Placement pl;
|
|
pl.fitted = true;
|
|
pl.n_ctx = 2048;
|
|
for (uint32_t il = 0; il < 48; ++il)
|
|
pl.host_expert_layers.push_back(il);
|
|
pl.host_resident_bytes = 1 * GiB;
|
|
pl.device_bytes = 4 * GiB;
|
|
const Plan p = plan_run(base_cfg(), disc, model, pl, PlanRequest{});
|
|
check(find(p, "shared-pool") == nullptr, "discrete memory: nothing is charged twice");
|
|
}
|
|
|
|
// ── an integrated accelerator is not the CPU ───────────────────────────────────
|
|
// Its memory IS the host's, so the capacity tier is inert - that much is arithmetic. Whether it
|
|
// reaches more of the same bus than the cores do is a measurement, and treating "shared memory"
|
|
// as "nothing to win" is the assumption this checks against.
|
|
{
|
|
HardwareProfile igpu = desktop();
|
|
igpu.devices.clear();
|
|
ComputeDevice cpu;
|
|
cpu.name = "CPU";
|
|
cpu.is_cpu = true;
|
|
cpu.shares_host_memory = Tri::Yes;
|
|
igpu.devices.push_back(cpu);
|
|
ComputeDevice ig;
|
|
ig.name = "integrated";
|
|
ig.is_cpu = false;
|
|
ig.shares_host_memory = Tri::Yes; // no memory of its own
|
|
ig.host_buffer = Tri::Yes;
|
|
ig.runs_expert_op = Tri::Yes;
|
|
ig.needs_repack = Tri::No;
|
|
igpu.devices.push_back(ig);
|
|
|
|
// Unmeasured: the capacity claim is made, the bandwidth claim is not.
|
|
const Plan un = plan_run(base_cfg(), igpu, model, PlanRequest{});
|
|
const Decision * d = find(un, "extra-offload");
|
|
check(d && d->source == Source::Unprobed && d->reason.find("capacity tier is inert") != std::string::npos,
|
|
"integrated, unmeasured: capacity stated, bandwidth admitted unknown",
|
|
d ? d->reason.substr(0, 70) : "none");
|
|
|
|
// Faster than the host on the same memory: that is a real finding, not a contradiction.
|
|
igpu.host_bandwidth_gibs = 30.0;
|
|
igpu.devices.back().memory_bandwidth_gibs = 60.0;
|
|
const Plan fast = plan_run(base_cfg(), igpu, model, PlanRequest{});
|
|
d = find(fast, "extra-offload");
|
|
check(d && d->source == Source::Measured && d->reason.find("has more of it") != std::string::npos,
|
|
"integrated, faster than the host: the plan says so", d ? d->reason.substr(0, 70) : "none");
|
|
|
|
// Slower, which is the ordinary desktop APU case: nothing to win, and it is measured.
|
|
igpu.devices.back().memory_bandwidth_gibs = 22.0;
|
|
const Plan slow = plan_run(base_cfg(), igpu, model, PlanRequest{});
|
|
d = find(slow, "extra-offload");
|
|
check(d && d->source == Source::Measured && d->reason.find("no more of it") != std::string::npos,
|
|
"integrated, no faster: measured, not assumed", d ? d->reason.substr(0, 70) : "none");
|
|
|
|
// And it is a candidate for the experts rule at all, which a shared-memory test excluded.
|
|
d = find(fast, "experts");
|
|
check(d && d->reason.find("could compute them on paper") != std::string::npos,
|
|
"an integrated device is considered for the streamed experts, not skipped as if it were the CPU",
|
|
d ? d->reason.substr(0, 70) : "none");
|
|
}
|
|
|
|
// ── the thread count is measured, not reasoned about ───────────────────────────
|
|
// What replaced the refuted rule: a sweep on the machine's own matmul. Where it has an answer it
|
|
// wins, including over the core classes, which stay a fact and stop being an argument.
|
|
{
|
|
HardwareProfile swept = desktop();
|
|
swept.n_cores = 20;
|
|
swept.core_classes = {2, 18}; // the shape that broke the old rule: it would have said 2
|
|
swept.best_threads = 8;
|
|
const Plan p = plan_run(base_cfg(), swept, model, PlanRequest{});
|
|
check(p.config.n_threads == 8, "swept threads: the measurement decides", std::to_string(p.config.n_threads));
|
|
const Decision * d = find(p, "threads");
|
|
check(d && d->source == Source::Measured, "swept threads: recorded as measured");
|
|
check(d && d->reason.find("quantized format") != std::string::npos,
|
|
"swept threads: the plan says what the instrument was made of");
|
|
}
|
|
|
|
// ── streamed experts may be COMPUTED on a device that reads host memory ─────────
|
|
// The property is one device's answer, not a class of hardware: a host buffer it executes over
|
|
// means the bytes we read from flash are readable by it without a copy and without a repack.
|
|
{
|
|
HardwareProfile acc = desktop();
|
|
acc.host_bandwidth_gibs = 50.0;
|
|
ComputeDevice gpu;
|
|
gpu.name = "accelerator";
|
|
gpu.memory_total = 16 * GiB;
|
|
gpu.shares_host_memory = Tri::No;
|
|
gpu.host_buffer = Tri::Yes;
|
|
gpu.rebindable = true;
|
|
gpu.runs_expert_op = Tri::Yes;
|
|
gpu.needs_repack = Tri::No;
|
|
acc.devices.push_back(gpu);
|
|
|
|
// The engine's own precondition comes first: the streamer rebinds every expert onto memory
|
|
// it reserved itself, so until those reservations come from a device allocator, no device
|
|
// may be named however capable or fast it is. This is the shipping default.
|
|
{
|
|
const Plan p = plan_run(base_cfg(), acc, model, PlanRequest{});
|
|
const Decision * d = find(p, "experts");
|
|
check(d && d->value == "host" && d->reason.find("would not survive the load") != std::string::npos,
|
|
"streamer cannot serve device memory: no device is named, and the plan says why",
|
|
d ? d->reason.substr(0, 60) : "none");
|
|
}
|
|
|
|
// ...and what that same note says about the one remaining opening must not overstate it.
|
|
// The capability a device advertises for wrapping memory the caller owns is a blanket
|
|
// promise, and a backend whose support is conditional per device and per call declines to
|
|
// make one while implementing the call anyway - so an absent yes is Unknown, not No. This
|
|
// fixture leaves the field at its default, which is that case. Reading a false there as a
|
|
// closed door is what once made this plan wrong about the backends that run on phones.
|
|
{
|
|
const Plan p = plan_run(base_cfg(), acc, model, PlanRequest{});
|
|
const Decision * d = find(p, "experts");
|
|
check(d && d->reason.find("is unprobed") != std::string::npos &&
|
|
d->reason.find("no opening here at all") == std::string::npos,
|
|
"unknown host-ptr capability: reported unprobed, never as a closed door",
|
|
d ? d->reason : "no decision");
|
|
}
|
|
|
|
// A device that does advertise it gets the opening named, which is the other half of the
|
|
// same asymmetry: a yes here IS a fact.
|
|
{
|
|
HardwareProfile wraps = acc;
|
|
wraps.devices.back().host_ptr_buffers = Tri::Yes;
|
|
const Plan p = plan_run(base_cfg(), wraps, model, PlanRequest{});
|
|
const Decision * d = find(p, "experts");
|
|
check(d && d->reason.find("which is the one opening") != std::string::npos,
|
|
"advertised host-ptr capability: the plan names the opening", d ? d->reason : "no decision");
|
|
}
|
|
|
|
// Everything below exercises the rule as it will behave once that precondition holds.
|
|
PlannerPolicy able = PlannerPolicy::defaults();
|
|
able.streamer_serves_device_memory = true;
|
|
|
|
// Unmeasured bandwidth: the mechanism is stated, the decision is not taken.
|
|
const Plan unmeasured = plan_run(base_cfg(), acc, model, Placement{}, PlanRequest{}, able);
|
|
const Decision * d = find(unmeasured, "experts");
|
|
check(d && d->source == Source::Unprobed, "host-buffer device, no bandwidth: stated, not decided");
|
|
|
|
// Measured and faster than the host: the experts stay rebindable and go there. (back(), not
|
|
// [0]: this fixture already carries a plain discrete device, and the one under test is ours.)
|
|
acc.devices.back().memory_bandwidth_gibs = 200.0;
|
|
const Plan p = plan_run(base_cfg(), acc, model, Placement{}, PlanRequest{}, able);
|
|
d = find(p, "experts");
|
|
check(d && d->value == "device" && d->source == Source::Measured,
|
|
"host-buffer device, faster than the host: experts computed there", d ? d->value : "none");
|
|
|
|
// A device that needs a repack is excluded by that property alone, however fast it is.
|
|
acc.devices.back().needs_repack = Tri::Yes;
|
|
const Plan r = plan_run(base_cfg(), acc, model, Placement{}, PlanRequest{}, able);
|
|
d = find(r, "experts");
|
|
check(d && d->value == "host", "repacking device: excluded whatever its bandwidth");
|
|
}
|
|
|
|
// ── a dense model has nothing to stream ────────────────────────────────────────
|
|
{
|
|
ModelProfile dense = model;
|
|
dense.is_moe = false;
|
|
const Plan p = plan_run(base_cfg(), desktop(), dense, PlanRequest{});
|
|
check(p.regime == Regime::NotMoe && !p.config.moe.enabled, "dense model: declined");
|
|
}
|
|
|
|
// ── an unprofiled machine declines instead of inventing numbers ────────────────
|
|
{
|
|
const Plan p = plan_run(base_cfg(), HardwareProfile{}, model, PlanRequest{});
|
|
check(!p.config.moe.enabled && p.streaming_declined, "blank machine: declined");
|
|
check(validate(p.config).ok, "blank machine: still a valid config");
|
|
}
|
|
|
|
// ── invariant: nothing lossy arms itself, at any quality budget ────────────────
|
|
{
|
|
PlanRequest req;
|
|
req.quality_budget = 0.05f; // the caller would accept 5% — the planner still may not guess
|
|
const Plan p = plan_run(base_cfg(), phone(), model, req);
|
|
check(p.config.moe.drop_cold_frac == 0.0f && p.config.moe.substitute_lambda == 0.0f &&
|
|
p.config.moe.route_ahead == 0,
|
|
"lossy levers stay off even with a quality budget");
|
|
const Decision * d = find(p, "lossy");
|
|
check(d && d->source == Source::Unprobed, "the unspent quality budget is recorded as unprobed");
|
|
}
|
|
|
|
// A caller that armed a lossy lever keeps it: the planner records it and does not touch it.
|
|
{
|
|
RunConfig c = base_cfg();
|
|
c.moe.drop_cold_frac = 0.75f;
|
|
PlanRequest req;
|
|
req.pinned.push_back("drop-cold-experts");
|
|
const Plan p = plan_run(c, phone(), model, req);
|
|
check(p.config.moe.drop_cold_frac == 0.75f, "an armed lossy lever survives the planner");
|
|
const Decision * d = find(p, "lossy");
|
|
check(d && d->source == Source::Operator, "an armed lossy lever is attributed to the operator");
|
|
}
|
|
|
|
// ── invariant: a pinned knob is never overwritten ──────────────────────────────
|
|
{
|
|
RunConfig c = base_cfg();
|
|
c.moe.dense_weights = DenseWeightsMode::Mmap;
|
|
c.moe.cache_mb = 1700;
|
|
c.moe.io_threads = 2;
|
|
PlanRequest req;
|
|
req.pinned = {"dense-weights", "cache-mb", "io-threads"};
|
|
const Plan p = plan_run(c, phone(), model, req);
|
|
check(p.config.moe.dense_weights == DenseWeightsMode::Mmap, "pinned dense policy survives");
|
|
check(p.config.moe.cache_mb == 1700, "pinned cache budget survives");
|
|
check(p.config.moe.io_threads == 2, "pinned lane count survives");
|
|
for (const char * k : {"dense-weights", "cache-mb", "io-threads"}) {
|
|
const Decision * d = find(p, k);
|
|
check(d && d->source == Source::Operator, std::string("pinned knob attributed to the operator: ") + k);
|
|
}
|
|
}
|
|
|
|
// ── the widest graph: its compute buffers are a row, and the width is chosen against it ──
|
|
{
|
|
// An unfitted placement, as on a machine with no device: the context and compute buffers
|
|
// the first stage projected for a graph as wide as the context, and what a token of width
|
|
// costs. Nothing here names a machine; the rule reads the buffers and the holdable figure.
|
|
auto wide = [&](uint32_t ctx, uint64_t per_token) {
|
|
Placement pl;
|
|
pl.n_ctx = ctx;
|
|
pl.raw_host_context_bytes = 256 * MiB;
|
|
pl.raw_host_compute_bytes = (uint64_t) ctx * per_token;
|
|
pl.compute_width = ctx;
|
|
pl.host_compute_per_token = per_token;
|
|
return pl;
|
|
};
|
|
const uint64_t per_token = 256 * KiB; // 16384 tokens wide is 4 GiB of buffers
|
|
const HardwareProfile hw = desktop();
|
|
const PlannerPolicy pol = PlannerPolicy::defaults();
|
|
const uint64_t cap =
|
|
(uint64_t) ((double) (hw.holdable_bytes ? hw.holdable_bytes : hw.residency_budget) * pol.compute_share_max);
|
|
|
|
// A small context fits its share whole: nothing changes and the knob keeps its default.
|
|
const uint32_t narrow_ctx = 1024;
|
|
const Plan fits = plan_run(base_cfg(), hw, model, wide(narrow_ctx, per_token), PlanRequest{});
|
|
const Decision * df = find(fits, "ubatch");
|
|
check((uint64_t) narrow_ctx * per_token <= cap, "width: the small context of this test is inside the share");
|
|
check(fits.config.n_ubatch == 0 && df && df->source == Source::Derived,
|
|
"width: buffers inside their share leave the graph as wide as the context");
|
|
|
|
// A large one does not: the width is halved until its buffers fit, and no further.
|
|
RunConfig big_ctx = base_cfg();
|
|
big_ctx.n_ctx = 16384;
|
|
const Plan cut = plan_run(big_ctx, hw, model, wide(16384, per_token), PlanRequest{});
|
|
const Decision * dc = find(cut, "ubatch");
|
|
const uint64_t w = (uint64_t) cut.config.n_ubatch;
|
|
check(w > 0 && w < 16384 && dc && dc->source == Source::Policy, "width: a large context is computed in pieces",
|
|
std::to_string(w));
|
|
check(w * per_token <= cap && (w * 2 * per_token > cap || w * 2 > 16384),
|
|
"width: the widest whose buffers fit, not a narrower one", std::to_string(w));
|
|
check(cut.placement.raw_host_compute_bytes == w * per_token && cut.placement.compute_width == w,
|
|
"width: the ledger is charged the buffers of the width chosen",
|
|
std::to_string(cut.placement.raw_host_compute_bytes / MiB));
|
|
// The same run with the graph held at full width by the caller: the buffers are charged in
|
|
// full, and the cache is what pays for them.
|
|
PlanRequest full_pin;
|
|
full_pin.pinned = {"ubatch"};
|
|
const Plan uncut = plan_run(big_ctx, hw, model, wide(16384, per_token), full_pin);
|
|
check(cut.cache_budget_bytes > uncut.cache_budget_bytes || !uncut.config.moe.enabled,
|
|
"width: what the buffers give back goes to the cache",
|
|
std::to_string(cut.cache_budget_bytes / MiB) + " vs " + std::to_string(uncut.cache_budget_bytes / MiB));
|
|
check(validate(cut.config).ok, "width: the plan is a valid config", validate(cut.config).error);
|
|
|
|
// The floor holds even when the buffers still do not fit there.
|
|
const Plan floor = plan_run(big_ctx, hw, model, wide(16384, 64 * MiB), PlanRequest{});
|
|
check((uint32_t) floor.config.n_ubatch == pol.ubatch_min, "width: never narrower than the floor",
|
|
std::to_string(floor.config.n_ubatch));
|
|
|
|
// A slope nobody measured decides nothing.
|
|
const Plan blind = plan_run(big_ctx, hw, model, wide(16384, 0), PlanRequest{});
|
|
const Decision * du = find(blind, "ubatch");
|
|
check(blind.config.n_ubatch == 0 && du && du->source == Source::Unprobed,
|
|
"width: an unmeasured slope leaves the knob at its default and says so");
|
|
|
|
// A width the caller set is the caller's, whatever it costs.
|
|
RunConfig by_hand = big_ctx;
|
|
by_hand.n_ubatch = 8192;
|
|
PlanRequest pr;
|
|
pr.pinned = {"ubatch"};
|
|
Placement at_hand = wide(16384, per_token).at_width(8192);
|
|
const Plan kept = plan_run(by_hand, hw, model, at_hand, pr);
|
|
const Decision * dk = find(kept, "ubatch");
|
|
check(kept.config.n_ubatch == 8192 && dk && dk->source == Source::Operator, "width: a pinned width is kept");
|
|
|
|
// A device's buffers count where they come out of this pool, and not where it has its own.
|
|
Placement with_dev = wide(4096, per_token / 2);
|
|
with_dev.device_compute_bytes = 4096ull * per_token;
|
|
with_dev.device_compute_per_token = per_token;
|
|
const Plan shared = plan_run(base_cfg(), unified(), model, with_dev, PlanRequest{});
|
|
HardwareProfile own = unified();
|
|
for (ComputeDevice & d : own.devices) {
|
|
d.memory_total = 24 * GiB;
|
|
d.memory_free = 24 * GiB;
|
|
d.shares_host_memory = Tri::No;
|
|
}
|
|
const Plan apart = plan_run(base_cfg(), own, model, with_dev, PlanRequest{});
|
|
const uint32_t ws = shared.config.n_ubatch ? (uint32_t) shared.config.n_ubatch : 4096;
|
|
const uint32_t wa = apart.config.n_ubatch ? (uint32_t) apart.config.n_ubatch : 4096;
|
|
check(ws <= wa, "width: a device that shares this pool narrows the graph at least as much as one that does not",
|
|
std::to_string(ws) + " vs " + std::to_string(wa));
|
|
|
|
// at_width is a line through the projected point, clamped at zero.
|
|
const Placement base_pl = wide(4096, per_token);
|
|
check(base_pl.at_width(2048).raw_host_compute_bytes == 2048ull * per_token &&
|
|
base_pl.at_width(8192).raw_host_compute_bytes == 8192ull * per_token &&
|
|
base_pl.at_width(4096).raw_host_compute_bytes == base_pl.raw_host_compute_bytes,
|
|
"width: buffers move along the measured slope, both ways");
|
|
}
|
|
|
|
// ── composed over a first stage: the fitter placed half the layers' experts on a device ─
|
|
{
|
|
Placement pl;
|
|
pl.fitted = true;
|
|
pl.outcome = "synthetic";
|
|
pl.n_gpu_layers = 24;
|
|
pl.n_ctx = 4096;
|
|
for (uint32_t il = 0; il < 24; ++il)
|
|
pl.host_expert_layers.push_back(il);
|
|
pl.host_resident_bytes = 4 * GiB; // the dense set the fitter left here, plus host KV/compute
|
|
pl.device_bytes = 8 * GiB;
|
|
const Plan p = plan_run(base_cfg(), desktop(), model, pl, PlanRequest{});
|
|
// 4 GiB dense + 9 GiB of host-side experts does not fit the desktop's budget, the dense part
|
|
// alone does: the host half streams.
|
|
check(p.config.moe.enabled, "placed: streaming still on for the host half");
|
|
check(p.config.n_gpu_layers == 24, "placed: n_gpu_layers carried into the config");
|
|
check(p.config.n_ctx == 4096, "placed: the fitter's context is honoured");
|
|
// The cache is capped at the HOST share of the experts, not the whole set.
|
|
check((uint64_t) p.config.moe.cache_mb * MiB <= model.expert_bytes / 2 + MiB,
|
|
"placed: cache capped at the host share of the experts", std::to_string(p.config.moe.cache_mb));
|
|
const Decision * d = find(p, "placement");
|
|
check(d && d->source == Source::Measured, "placed: the placement is recorded as measured");
|
|
check(validate(p.config).ok, "placed: the plan is a valid config", validate(p.config).error);
|
|
|
|
// What the fitter wrote is part of the plan, so it is part of the line that reproduces it.
|
|
// The patterns are regexes joined by ';': quoted, or a shell would end the command there.
|
|
Placement pinned_pl = pl;
|
|
pinned_pl.override_patterns = {"blk\\.(0|1)\\.ffn_.*_exps", "blk\\.2\\..*"};
|
|
const std::string placed_flags = plan_run(base_cfg(), desktop(), model, pinned_pl, PlanRequest{}).to_flags();
|
|
check(placed_flags.find("--gpu-layers 24") != std::string::npos &&
|
|
placed_flags.find("--tensor-overrides 'blk\\.(0|1)\\.ffn_.*_exps;blk\\.2\\..*'") != std::string::npos,
|
|
"placed: the reproduce line carries the fitter's tensor overrides, quoted", placed_flags);
|
|
check(placed_flags.find("model.gguf") == std::string::npos,
|
|
"placed: the reproduce line leaves the model to the caller", placed_flags);
|
|
|
|
// When the fitter's host residual FITS in RAM, the right answer is the classic offload -
|
|
// experts resident on the host, no streaming - and the plan must say so rather than stream.
|
|
Placement small = pl;
|
|
small.host_resident_bytes = 1 * GiB;
|
|
const Plan r = plan_run(base_cfg(), desktop(), model, small, PlanRequest{});
|
|
check(r.regime == Regime::Fits && !r.config.moe.enabled,
|
|
"placed, residual fits: experts stay resident, streaming declined (the -ot exps=CPU case)");
|
|
|
|
Placement all_dev = pl;
|
|
all_dev.host_expert_layers.clear();
|
|
all_dev.n_gpu_layers = 48;
|
|
const Plan q = plan_run(base_cfg(), desktop(), model, all_dev, PlanRequest{});
|
|
check(!q.config.moe.enabled && q.streaming_declined, "all on devices: nothing left to stream, declined");
|
|
}
|
|
|
|
// ── the rationale exists and names its confidence ──────────────────────────────
|
|
{
|
|
const Plan p = plan_run(base_cfg(), phone(), model, PlanRequest{});
|
|
const std::string text = p.explain();
|
|
check(text.find("experts-stream") != std::string::npos, "explain() names the regime");
|
|
check(text.find("[derived]") != std::string::npos, "explain() names each decision's source");
|
|
check(find(p, "threads") && find(p, "threads")->source == Source::Unprobed,
|
|
"a knob with no rule is recorded as unprobed rather than silently defaulted");
|
|
}
|
|
|
|
// ── the cost model: every verdict it encodes, as a case rather than as a comment ──
|
|
//
|
|
// These drive `allocate()` directly. It is a pure function over two structs, so a machine with
|
|
// a fast accelerator and a machine that lies about one are both a fixture, and the refutations
|
|
// this engine paid for stay checked instead of staying in a paragraph nobody runs.
|
|
{
|
|
// A machine with everything measured: rates for compute, refault and the expert lane.
|
|
auto measured_machine = [&]() {
|
|
HardwareProfile h = desktop();
|
|
h.host_bandwidth_gibs = 100.0;
|
|
h.storage.refault_mibs = 24.0; // the price of a page fault, two orders below sequential
|
|
h.devices.clear();
|
|
return h;
|
|
};
|
|
auto inputs = [&](uint64_t budget) {
|
|
AllocationInputs in;
|
|
in.budget_bytes = budget;
|
|
in.engine_cache_min = 0;
|
|
in.cacheable_bytes = model.expert_bytes;
|
|
return in;
|
|
};
|
|
auto fast_device = [](const char * name, double gibs) {
|
|
ComputeDevice d;
|
|
d.name = name;
|
|
d.memory_bandwidth_gibs = gibs;
|
|
d.memory_total = 16 * GiB;
|
|
d.memory_free = 15 * GiB;
|
|
d.identity_ok = Tri::Yes;
|
|
d.needs_repack = Tri::No;
|
|
return d;
|
|
};
|
|
const PlannerPolicy pol = PlannerPolicy::defaults();
|
|
|
|
// The read-whole groups outrank the cache: they are re-read every token while the cache
|
|
// serves one token cycle, so seconds saved per byte of RAM is orders apart. This is the
|
|
// measured behaviour ("pin the dense set, then give the cache the rest") arrived at by
|
|
// arithmetic rather than written down as an order.
|
|
{
|
|
const Allocation a = allocate(measured_machine(), model, inputs(6 * GiB), pol);
|
|
check(a.at(WeightGroup::Attention).lane == Lane::Resident, "read-whole groups win residency first");
|
|
check(a.cache_bytes > 0, "the cache still gets what remains after them");
|
|
check(a.at(WeightGroup::Embedding).resident_bytes == 0,
|
|
"a row-gathered table buys no residency, however large it is",
|
|
"it holds " + std::to_string(a.at(WeightGroup::Embedding).resident_bytes >> 20) + " MiB");
|
|
}
|
|
|
|
// Under the model's own token cycle a cache evicts what the same token still needs. The
|
|
// floor is the model's arithmetic, and a budget below it buys nothing at all.
|
|
{
|
|
AllocationInputs in = inputs(3 * GiB);
|
|
in.reserved_bytes = 3 * GiB - (100 * MiB); // leaves less than one token cycle
|
|
const Allocation a = allocate(measured_machine(), model, in, pol);
|
|
check(a.cache_bytes == 0, "a residual under the token cycle leaves the cache off",
|
|
std::to_string(a.cache_bytes >> 20) + " MiB");
|
|
}
|
|
|
|
// THE refutation. A device three times the host's bandwidth is still not used while the
|
|
// price of a graph crossing is unmeasured: dense and expert halves alternate per layer, and
|
|
// a bandwidth ratio cannot see that cost. Measured 20 against 12 GiB/s on a phone that then
|
|
// ran the move at 0.53x.
|
|
{
|
|
HardwareProfile h = measured_machine();
|
|
h.devices.push_back(fast_device("fast", 300.0));
|
|
const Allocation a = allocate(h, model, inputs(6 * GiB), pol);
|
|
check(a.at(WeightGroup::Attention).device < 0, "an unpriced graph crossing forbids a device placement");
|
|
check(a.device_applicable, "and the plan does not claim a placement it did not make");
|
|
}
|
|
|
|
// Priced, and worth it: the same device wins once a crossing has a number small enough.
|
|
{
|
|
HardwareProfile h = measured_machine();
|
|
h.devices.push_back(fast_device("fast", 300.0));
|
|
h.devices.back().split_seconds = 1e-7; // 96 crossings cost ~10 us against ~9 ms saved
|
|
const PlannerPolicy & priced = pol;
|
|
const Allocation a = allocate(h, model, inputs(6 * GiB), priced);
|
|
check(a.at(WeightGroup::Attention).device >= 0, "a priced crossing lets a real win through");
|
|
check(!a.device_applicable, "and it is reported as not applicable by this build");
|
|
}
|
|
|
|
// Correctness gates speed, and it is a fact rather than a promise: a device that does not
|
|
// reproduce the CPU's answer is excluded before its rate is looked at.
|
|
{
|
|
HardwareProfile h = measured_machine();
|
|
ComputeDevice d = fast_device("wrong", 300.0);
|
|
d.identity_ok = Tri::No;
|
|
h.devices.push_back(d);
|
|
h.devices.back().split_seconds = 1e-9;
|
|
const PlannerPolicy & priced = pol;
|
|
const Allocation a = allocate(h, model, inputs(6 * GiB), priced);
|
|
check(a.at(WeightGroup::Attention).device < 0, "a device that computes something else is excluded");
|
|
}
|
|
|
|
// A repacked layout replaces the very thing a rebind depends on, so such a device is out on
|
|
// a property rather than on a name.
|
|
{
|
|
HardwareProfile h = measured_machine();
|
|
ComputeDevice d = fast_device("repacker", 300.0);
|
|
d.needs_repack = Tri::Yes;
|
|
h.devices.push_back(d);
|
|
h.devices.back().split_seconds = 1e-9;
|
|
const PlannerPolicy & priced = pol;
|
|
const Allocation a = allocate(h, model, inputs(6 * GiB), priced);
|
|
check(a.at(WeightGroup::Attention).device < 0, "a device that only runs a repacked layout is excluded");
|
|
}
|
|
|
|
// Unified memory at equal bandwidth: nothing to win, and the plan says so instead of moving
|
|
// work for its own sake. Measured 19 against 19 on a phone, which is not a finding.
|
|
{
|
|
HardwareProfile h = measured_machine();
|
|
ComputeDevice d = fast_device("integrated", 100.0);
|
|
d.shares_host_memory = Tri::Yes;
|
|
h.devices.push_back(d);
|
|
h.devices.back().split_seconds = 1e-9;
|
|
const PlannerPolicy & priced = pol;
|
|
const Allocation a = allocate(h, model, inputs(6 * GiB), priced);
|
|
check(a.at(WeightGroup::Attention).device < 0, "an equal-bandwidth device wins nothing and is not used");
|
|
}
|
|
|
|
// A device whose memory topology was never settled must not be credited with memory of its
|
|
// own. The error is asymmetric and the safe direction is the one taken: counting a shared
|
|
// pool as extra capacity overcommits a machine - the accounting that took a phone down -
|
|
// while counting separate memory as shared merely leaves it unused. This is also the case a
|
|
// device TYPE gets wrong, since Metal reports GPU on unified-memory hardware.
|
|
{
|
|
HardwareProfile h = measured_machine();
|
|
ComputeDevice d = fast_device("unsettled", 300.0);
|
|
d.shares_host_memory = Tri::Unknown; // the probe could not run
|
|
d.memory_total = 16 * GiB;
|
|
h.devices.push_back(d);
|
|
check(h.device_local_memory() == 0, "an unsettled device is not counted as capacity of its own",
|
|
std::to_string(h.device_local_memory() >> 20) + " MiB");
|
|
check(!h.devices.back().has_own_memory() && h.devices.back().reads_host_memory(),
|
|
"and it reads as sharing the host's memory until measured");
|
|
}
|
|
|
|
// The crossing price is a per-device MEASUREMENT, not a policy constant: two devices on one
|
|
// machine can differ, and a device nobody measured declines on its own rather than
|
|
// inheriting a number from its neighbour.
|
|
{
|
|
HardwareProfile h = measured_machine();
|
|
ComputeDevice slow_link = fast_device("far", 300.0);
|
|
slow_link.split_seconds = 1e-3; // 96 crossings cost ~96 ms against ~9 ms saved
|
|
h.devices.push_back(slow_link);
|
|
const Allocation a = allocate(h, model, inputs(6 * GiB), pol);
|
|
check(a.at(WeightGroup::Attention).device < 0,
|
|
"a measured crossing price can refuse a device that wins on bandwidth");
|
|
}
|
|
|
|
// An unmeasured rate makes a candidate ineligible, never cheap.
|
|
{
|
|
HardwareProfile h = measured_machine();
|
|
h.storage.refault_mibs = 0.0;
|
|
const Allocation a = allocate(h, model, inputs(6 * GiB), pol);
|
|
check(a.at(WeightGroup::Attention).lane == Lane::Mmap,
|
|
"with the refault price unmeasured, residency is not bought");
|
|
}
|
|
}
|
|
|
|
// ── the memory ledger: one account, two ceilings ───────────────────────────────
|
|
// What is held and what is held reclaim-exempt are different totals, and the second is the
|
|
// smaller wherever it exists. Sized against the first alone, a plan asks for locks the machine
|
|
// stops granting part way - measured at a tenth of the throughput of the plan that closes.
|
|
{
|
|
const auto mibs_of = [](uint64_t b) { return std::to_string((unsigned long long) (b >> 20)) + " MiB"; };
|
|
|
|
// Room for both: the dense set is pinned whole, and the cache is what can be locked after
|
|
// it - less than what could merely be held.
|
|
const Plan p = plan_run(base_cfg(), unified(), model, PlanRequest{});
|
|
const Ledger & l = p.allocation.ledger;
|
|
check(p.config.moe.dense_weights == DenseWeightsMode::Pinned,
|
|
"ledger: the dense set fits the lock and is pinned");
|
|
check(l.lockable_cap > 0 && l.locked <= l.lockable_cap,
|
|
"ledger: what is locked stays under the lockable ceiling",
|
|
mibs_of(l.locked) + " of " + mibs_of(l.lockable_cap));
|
|
check(l.held <= l.holdable_cap, "ledger: what is held stays under the holdable ceiling",
|
|
mibs_of(l.held) + " of " + mibs_of(l.holdable_cap));
|
|
check(p.allocation.cache_locked_bytes == p.cache_budget_bytes && p.cache_budget_bytes > 0,
|
|
"ledger: the whole cache is the part that can be locked");
|
|
uint64_t rows = 0;
|
|
for (const LedgerRow & r : l.rows)
|
|
rows += r.bytes;
|
|
check(rows == l.held, "ledger: the rows add up to what is held");
|
|
check(find(p, "memory-ledger") != nullptr, "ledger: the plan states it");
|
|
check(validate(p.config).ok, "ledger: the plan is a valid config", validate(p.config).error);
|
|
|
|
// The same machine with nothing bounding the lock but what can be held: the cache is larger.
|
|
// It is the lockable total, and nothing else, that made the difference above.
|
|
HardwareProfile roomy = unified();
|
|
roomy.lockable_bytes = ~0ull;
|
|
roomy.lock_in_place_bytes = ~0ull;
|
|
const Plan q = plan_run(base_cfg(), roomy, model, PlanRequest{});
|
|
check(q.cache_budget_bytes > p.cache_budget_bytes, "ledger: the lockable total is what bounded the cache",
|
|
mibs_of(p.cache_budget_bytes) + " against " + mibs_of(q.cache_budget_bytes));
|
|
|
|
// A lock too small for the dense set. It is not pinned part way: it is held whole as
|
|
// ordinary memory, the plan says by how much it missed, and the lock goes to the cache.
|
|
HardwareProfile tight = unified();
|
|
tight.lockable_bytes = 3 * GiB; // the dense set is 3 GiB, and the margin comes off this
|
|
tight.lock_in_place_bytes = 3 * GiB;
|
|
const Plan t = plan_run(base_cfg(), tight, model, PlanRequest{});
|
|
check(t.config.moe.dense_weights == DenseWeightsMode::Anonymous,
|
|
"ledger: a dense set that does not fit the lock whole is not pinned at all");
|
|
check(t.allocation.pinned_shortfall_bytes > 0, "ledger: the shortfall is recorded");
|
|
const Decision * dw = find(t, "dense-weights");
|
|
check(dw && dw->value == "anon" && dw->reason.find("more than can still be held") != std::string::npos,
|
|
"ledger: the dense decision says why it was not pinned");
|
|
check(t.allocation.ledger.locked <= t.allocation.ledger.lockable_cap,
|
|
"ledger: the cache alone stays under the lockable ceiling");
|
|
check(t.cache_budget_bytes >= model.token_cycle_bytes, "ledger: the cache still clears the token cycle");
|
|
|
|
// The caller's pin is the caller's authority, including over a ledger that does not close:
|
|
// it is charged as asked, and the cache - which no longer has a cycle's worth of lock - is
|
|
// ordinary memory bounded by what can be held.
|
|
RunConfig forced = base_cfg();
|
|
forced.moe.dense_weights = DenseWeightsMode::Pinned;
|
|
PlanRequest fr;
|
|
fr.pinned = {"dense-weights"};
|
|
const Plan f = plan_run(forced, tight, model, fr);
|
|
check(f.config.moe.dense_weights == DenseWeightsMode::Pinned,
|
|
"ledger: a caller's pinned dense store is not overruled");
|
|
check(f.allocation.cache_locked_bytes == 0 && f.cache_budget_bytes >= model.token_cycle_bytes,
|
|
"ledger: with the lock spent, the cache is ordinary memory");
|
|
|
|
// The plan and the run agree about the cache's lock: sized to it, the run is told to keep
|
|
// it there; a caller's own budget is never one the run may shrink.
|
|
check(p.config.moe.cache_pin && p.config.moe.cache_pin_fit,
|
|
"ledger: a cache sized to the lock is fitted to it");
|
|
check(!f.config.moe.cache_pin && !f.config.moe.cache_pin_fit,
|
|
"ledger: a cache that cannot be locked asks for no lock");
|
|
RunConfig own = base_cfg();
|
|
own.moe.cache_mb = 12000;
|
|
PlanRequest orq;
|
|
orq.pinned = {"cache-mb"};
|
|
const Plan o = plan_run(own, unified(), model, orq);
|
|
const LedgerRow * oc = ledger_row(o, "expert cache");
|
|
check(!o.config.moe.cache_pin_fit && o.config.moe.cache_mb == 12000,
|
|
"ledger: a caller's cache budget is not fitted");
|
|
check(oc && oc->bytes == 12000 * MiB && o.allocation.ledger.held > o.allocation.ledger.holdable_cap,
|
|
"ledger: a caller's cache is the row, even over the ceiling");
|
|
|
|
// Two stores that are not one pool: no reclaim-exempt allocation, a generous lock in place.
|
|
// The ceiling reported is the one the locked rows were charged to.
|
|
HardwareProfile inplace = unified();
|
|
inplace.reclaim_exempt_max = 0;
|
|
inplace.lockable_bytes = 0;
|
|
inplace.lock_in_place_bytes = 8 * GiB;
|
|
const Plan ip = plan_run(base_cfg(), inplace, model, PlanRequest{});
|
|
check(ip.allocation.ledger.locked > 0 && ip.allocation.ledger.locked <= ip.allocation.ledger.lockable_cap,
|
|
"ledger: a lock-in-place store is the ceiling its rows are checked against",
|
|
mibs_of(ip.allocation.ledger.locked) + " of " + mibs_of(ip.allocation.ledger.lockable_cap));
|
|
|
|
// A machine that offers a reclaim-exempt allocation and no lock in place: the dense set is
|
|
// pinned and the cache is bounded only by what can be held, as it always was there.
|
|
const Plan ph = plan_run(base_cfg(), phone(), model, PlanRequest{});
|
|
const LedgerRow * pc = ledger_row(ph, "expert cache");
|
|
const LedgerRow * pd = ledger_row(ph, "dense");
|
|
check(pd && pd->locked && pc && !pc->locked, "ledger: two stores, and only the dense set is in the locked one");
|
|
check(!ph.config.moe.cache_pin, "ledger: and no lock is asked for the cache there");
|
|
check(ph.cache_budget_bytes > 1 * GiB, "ledger: a cache that cannot be locked is not sized to the lock",
|
|
mibs_of(ph.cache_budget_bytes));
|
|
|
|
// No lock at all: nothing is charged to a ceiling that does not exist.
|
|
const Plan dk = plan_run(base_cfg(), desktop(), model, PlanRequest{});
|
|
check(dk.allocation.ledger.locked == 0 && dk.allocation.ledger.lockable_cap == 0,
|
|
"ledger: a machine with no lock locks nothing");
|
|
|
|
// A device armed for prefill is a line, charged to both ceilings where its buffers are
|
|
// locked: two of the model's largest expert layer, and the cache gives up exactly that.
|
|
ModelProfile mm = model;
|
|
mm.largest_expert_layer_bytes = 384 * MiB;
|
|
RunConfig armed = base_cfg();
|
|
armed.prefill.device = "accel";
|
|
PlanRequest ar;
|
|
ar.pinned = {"prefill-device"};
|
|
const Plan off = plan_run(base_cfg(), unified(), mm, PlanRequest{});
|
|
const Plan on = plan_run(armed, unified(), mm, ar);
|
|
const LedgerRow * dev = ledger_row(on, "device");
|
|
check(dev && dev->locked && dev->bytes == 768 * MiB,
|
|
"ledger: an armed device is charged two expert layers, locked", dev ? mibs_of(dev->bytes) : "no row");
|
|
check(ledger_row(off, "device") == nullptr, "ledger: a device nobody armed is not a line");
|
|
check(off.cache_budget_bytes - on.cache_budget_bytes == 768 * MiB,
|
|
"ledger: the cache gives up what the device takes",
|
|
mibs_of(off.cache_budget_bytes) + " against " + mibs_of(on.cache_budget_bytes));
|
|
|
|
// The context and the compute buffers are a line even when the capacity fitter's answer
|
|
// was set aside - which on shared memory is always, and where they used to be charged nowhere.
|
|
Placement pl;
|
|
pl.shared_memory_placement = true;
|
|
pl.raw_host_context_bytes = 500 * MiB;
|
|
pl.raw_host_compute_bytes = 100 * MiB;
|
|
const Plan fx = plan_run(base_cfg(), phone(), model, pl, PlanRequest{});
|
|
const LedgerRow * fr_row = ledger_row(fx, "context and compute");
|
|
check(fr_row && fr_row->bytes == 600 * MiB,
|
|
"ledger: context and compute are charged without a fitted placement");
|
|
check(ph.cache_budget_bytes - fx.cache_budget_bytes == 600 * MiB, "ledger: and the cache is sized after them",
|
|
mibs_of(ph.cache_budget_bytes) + " against " + mibs_of(fx.cache_budget_bytes));
|
|
}
|
|
|
|
// ── the prefill plan: a second placement, decided at prefill width ─────────────
|
|
// A device that loses the one-token graph can win a wide one several times over, so the
|
|
// prefill is decided from the wide measurement and from nothing else - and armed only when
|
|
// the device also gave the host's answer and its memory still leaves the ledger closed.
|
|
{
|
|
ModelProfile mm = model;
|
|
mm.largest_expert_layer_bytes = 384 * MiB;
|
|
mm.largest_layer_dense_bytes = 64 * MiB;
|
|
const uint64_t two_layers = 2 * (384 + 64) * MiB;
|
|
|
|
// The accelerator of `unified()`, slower than the cores on one token and faster on many.
|
|
const auto machine = [](double wide, Tri same) {
|
|
HardwareProfile h = unified();
|
|
h.host_bandwidth_gibs = 100.0;
|
|
h.host_wide_matmul_gibs = 200.0;
|
|
h.wide_batch = 128;
|
|
h.devices[1].memory_bandwidth_gibs = 60.0; // loses the decode's graph
|
|
h.devices[1].wide_matmul_gibs = wide;
|
|
h.devices[1].wide_identity_ok = same;
|
|
return h;
|
|
};
|
|
const auto reason = [](const Plan & p) {
|
|
const Decision * d = find(p, "prefill-device");
|
|
return d ? d->reason : std::string();
|
|
};
|
|
|
|
const Plan off = plan_run(base_cfg(), unified(), mm, PlanRequest{});
|
|
check(!off.config.prefill.enabled() && find(off, "prefill-device") &&
|
|
find(off, "prefill-device")->source == Source::Unprobed,
|
|
"prefill: a device nobody measured at prefill width is not armed, and the plan says unprobed");
|
|
|
|
const Plan on = plan_run(base_cfg(), machine(1500.0, Tri::Yes), mm, PlanRequest{});
|
|
const LedgerRow * row = ledger_row(on, "device");
|
|
const LedgerRow * turn = ledger_row(on, "device slots");
|
|
const uint64_t stays = 2 * 64 * MiB; // the layer's other weights, held for the session
|
|
const uint64_t in_turn = 2 * 384 * MiB; // the expert slots, held only during a prefill
|
|
check(on.config.prefill.device == "accel",
|
|
"prefill: a device that wins wide and agrees is armed by its own name");
|
|
check(on.config.prefill.best_effort, "prefill: a device the plan armed is never a condition for the load");
|
|
// The expert slots take turns with the cache: it is idle exactly when they are busy, so
|
|
// they are not charged to decode. What the cache gives up is only what stays.
|
|
check(on.config.prefill.slots_on_demand, "prefill: a plan arms slots that take turns with the cache");
|
|
check(row && row->locked && row->bytes == stays && !row->in_turn,
|
|
"prefill: what stays for the session is a row",
|
|
row ? std::to_string((unsigned long long) (row->bytes >> 20)) + " MiB" : "no row");
|
|
check(turn && turn->in_turn && turn->bytes == in_turn,
|
|
"prefill: the expert slots are a row in turn with the cache",
|
|
turn ? std::to_string((unsigned long long) (turn->bytes >> 20)) + " MiB" : "no row");
|
|
check(off.cache_budget_bytes - on.cache_budget_bytes == stays,
|
|
"prefill: and the cache gives up only what stays");
|
|
uint64_t counted = 0;
|
|
for (const LedgerRow & r : on.allocation.ledger.rows)
|
|
if (!r.in_turn) counted += r.bytes;
|
|
check(counted == on.allocation.ledger.held, "prefill: a row in turn is not in what is held while decoding");
|
|
// A caller who wants them held for the session gets the old account: the cache makes room for all of it.
|
|
PlanRequest held_req;
|
|
held_req.pinned = {"prefill-slots-on-demand"};
|
|
const Plan held = plan_run(base_cfg(), machine(1500.0, Tri::Yes), mm, held_req);
|
|
check(!held.config.prefill.slots_on_demand && ledger_row(held, "device slots") == nullptr &&
|
|
off.cache_budget_bytes - held.cache_budget_bytes == two_layers,
|
|
"prefill: slots held for the session are charged whole, beside the cache");
|
|
check(on.allocation.ledger.locked <= on.allocation.ledger.lockable_cap, "prefill: the ledger still closes");
|
|
check(on.config.device_use != DeviceUse::CpuOnly, "prefill: an armed device is not taken out of the run");
|
|
check(validate(on.config).ok, "prefill: the armed plan is a valid config", validate(on.config).error);
|
|
check(on.to_flags().find("--prefill-device accel") != std::string::npos &&
|
|
on.to_flags().find("--prefill-best-effort") != std::string::npos &&
|
|
on.to_flags().find("--prefill-slots-on-demand") != std::string::npos,
|
|
"prefill: the reproduce line carries it");
|
|
|
|
const Plan wrong = plan_run(base_cfg(), machine(1500.0, Tri::No), mm, PlanRequest{});
|
|
check(!wrong.config.prefill.enabled() && reason(wrong).find("did not reproduce") != std::string::npos,
|
|
"prefill: a faster device with a different answer is refused on correctness");
|
|
|
|
const Plan slow = plan_run(base_cfg(), machine(205.0, Tri::Yes), mm, PlanRequest{});
|
|
check(!slow.config.prefill.enabled() && reason(slow).find("not a win") != std::string::npos,
|
|
"prefill: a device inside the margin is not a second placement");
|
|
|
|
// Two slots do not fit and one does: the device is armed with one, and the plan says so.
|
|
// The layer is sized so that the difference between the two is exactly what decides it.
|
|
ModelProfile mid = mm;
|
|
mid.largest_expert_layer_bytes = 4 * GiB; // the cache can yield one of these and not two
|
|
const Plan one = plan_run(base_cfg(), machine(1500.0, Tri::Yes), mid, PlanRequest{});
|
|
const LedgerRow * one_row = ledger_row(one, "device slots");
|
|
check(one.config.prefill.device == "accel" && one.config.prefill.slots == 1,
|
|
"prefill: where two expert slots do not fit and one does, one is used",
|
|
std::to_string(one.config.prefill.slots) + " slot(s)");
|
|
check(one_row && one_row->bytes == 4 * GiB, "prefill: and one layer of experts is what takes its turn");
|
|
check(reason(one).find("one expert slot") != std::string::npos,
|
|
"prefill: the plan says it chose the narrower scheme");
|
|
check(one.to_flags().find("--prefill-slots 1") != std::string::npos,
|
|
"prefill: the reproduce line carries the slot count");
|
|
check(on.config.prefill.slots == 2 && on.to_flags().find("--prefill-slots 1") == std::string::npos,
|
|
"prefill: two slots stay the default where they fit");
|
|
check(validate(one.config).ok, "prefill: the one-slot plan is a valid config", validate(one.config).error);
|
|
|
|
// A caller who fixed the count is not given the other one.
|
|
RunConfig two = base_cfg();
|
|
two.prefill.slots = 2;
|
|
PlanRequest tr;
|
|
tr.pinned = {"prefill-slots"};
|
|
check(!plan_run(two, machine(1500.0, Tri::Yes), mid, tr).config.prefill.enabled(),
|
|
"prefill: a caller's slot count is not traded down");
|
|
|
|
// Too large for what is left: refused, with the figures, and the plan is the one without it.
|
|
ModelProfile big = mm;
|
|
big.largest_expert_layer_bytes = 7 * GiB;
|
|
const Plan nofit = plan_run(base_cfg(), machine(1500.0, Tri::Yes), big, PlanRequest{});
|
|
const Plan nodev = plan_run(base_cfg(), unified(), big, PlanRequest{});
|
|
check(!nofit.config.prefill.enabled() && reason(nofit).find("refused on memory") != std::string::npos,
|
|
"prefill: a device that does not fit is refused with the reason");
|
|
check(nofit.cache_budget_bytes == nodev.cache_budget_bytes && ledger_row(nofit, "device") == nullptr,
|
|
"prefill: and a refusal leaves the ledger as it was");
|
|
|
|
// Something else in the run that cannot share a session with it.
|
|
RunConfig rows = base_cfg();
|
|
rows.moe.row_stream = true;
|
|
const Plan conflict = plan_run(rows, machine(1500.0, Tri::Yes), mm, PlanRequest{});
|
|
check(!conflict.config.prefill.enabled() && reason(conflict).find("not armed, because") != std::string::npos,
|
|
"prefill: a setting it cannot coexist with declines it instead of producing an invalid config");
|
|
|
|
// The caller's word, either way.
|
|
PlanRequest none;
|
|
none.pinned = {"prefill-device"};
|
|
const Plan pinned_off = plan_run(base_cfg(), machine(1500.0, Tri::Yes), mm, none);
|
|
check(!pinned_off.config.prefill.enabled() && find(pinned_off, "prefill-device")->source == Source::Operator,
|
|
"prefill: a caller who set it empty is not overruled");
|
|
|
|
// A library the host calls into is not somewhere a prefill can be placed, however it measures.
|
|
HardwareProfile helper = machine(0.0, Tri::Unknown);
|
|
ComputeDevice lib;
|
|
lib.name = "lib";
|
|
lib.is_host_helper = true;
|
|
lib.shares_host_memory = Tri::Yes;
|
|
lib.wide_matmul_gibs = 5000.0;
|
|
lib.wide_identity_ok = Tri::Yes;
|
|
helper.devices.push_back(lib);
|
|
check(!plan_run(base_cfg(), helper, mm, PlanRequest{}).config.prefill.enabled(),
|
|
"prefill: a host helper library is never the prefill device");
|
|
|
|
// A device with memory of its own costs this pool nothing, and is refused only by its own.
|
|
HardwareProfile disc = machine(1500.0, Tri::Yes);
|
|
disc.devices[1].shares_host_memory = Tri::No;
|
|
disc.devices[1].memory_total = 8 * GiB;
|
|
disc.devices[1].memory_free = 8 * GiB;
|
|
const Plan own = plan_run(base_cfg(), disc, mm, PlanRequest{});
|
|
check(own.config.prefill.device == "accel" && ledger_row(own, "device") == nullptr &&
|
|
own.cache_budget_bytes == off.cache_budget_bytes,
|
|
"prefill: a device with its own memory is armed and charged nothing here");
|
|
// "Its own memory" with no size is not a pool to budget against: charged to this one.
|
|
HardwareProfile sizeless = machine(1500.0, Tri::Yes);
|
|
sizeless.devices[1].shares_host_memory = Tri::No;
|
|
const Plan sz = plan_run(base_cfg(), sizeless, mm, PlanRequest{});
|
|
const LedgerRow * szr = ledger_row(sz, "device");
|
|
check(sz.config.prefill.device == "accel" && szr && szr->bytes == stays && ledger_row(sz, "device slots"),
|
|
"prefill: a device that claims its own memory and states no size is charged to the host");
|
|
|
|
disc.devices[1].memory_free = 256 * MiB;
|
|
check(!plan_run(base_cfg(), disc, mm, PlanRequest{}).config.prefill.enabled(),
|
|
"prefill: and refused when its own memory cannot take even one layer");
|
|
}
|
|
|
|
// ── a device probed where it cannot take the plan with it ──────────────────────
|
|
// A device is a claim by a driver, and a driver can end the process that uses it. So a device
|
|
// is asked in another process, and what comes back is its facts or the reason it has none.
|
|
{
|
|
// The facts survive the trip, whole.
|
|
ComputeDevice src;
|
|
src.name = "accel";
|
|
src.shares_host_memory = Tri::Yes;
|
|
src.host_buffer = Tri::Yes;
|
|
src.host_ptr_buffers = Tri::Unknown;
|
|
src.async_copies = Tri::No;
|
|
src.rebindable = true;
|
|
src.runs_expert_op = Tri::Yes;
|
|
src.needs_repack = Tri::No;
|
|
src.memory_bandwidth_gibs = 63.5;
|
|
src.identity_ok = Tri::Yes;
|
|
src.host_ptr_verified = Tri::No;
|
|
src.split_seconds = 0.000125;
|
|
src.graph_splits = 96;
|
|
src.wide_matmul_gibs = 1502.25;
|
|
src.wide_identity_ok = Tri::Yes;
|
|
src.buffers_locked = Tri::Yes;
|
|
ComputeDevice dst;
|
|
dst.name = "accel";
|
|
const std::string line = device_facts_to_text(src);
|
|
check(device_facts_from_text(line, dst), "isolated: a device's facts parse back", line);
|
|
check(dst.shares_host_memory == Tri::Yes && dst.host_ptr_buffers == Tri::Unknown &&
|
|
dst.async_copies == Tri::No && dst.rebindable && dst.runs_expert_op == Tri::Yes &&
|
|
dst.needs_repack == Tri::No && dst.memory_bandwidth_gibs == 63.5 && dst.identity_ok == Tri::Yes &&
|
|
dst.host_ptr_verified == Tri::No && dst.split_seconds == 0.000125 && dst.graph_splits == 96 &&
|
|
dst.wide_matmul_gibs == 1502.25 && dst.wide_identity_ok == Tri::Yes && dst.buffers_locked == Tri::Yes,
|
|
"isolated: and every one of them is what was sent");
|
|
// A line cut short - a child that died while printing - is applied not at all.
|
|
ComputeDevice untouched;
|
|
untouched.memory_bandwidth_gibs = 7.0;
|
|
check(!device_facts_from_text(line.substr(0, line.size() / 2), untouched) &&
|
|
untouched.memory_bandwidth_gibs == 7.0,
|
|
"isolated: half a line changes nothing");
|
|
check(!device_facts_from_text("", untouched) && !device_facts_from_text("garbage in the stream", untouched),
|
|
"isolated: nor does noise");
|
|
|
|
// The parent's side: one device answers, one ends its probe, the host is never asked.
|
|
HardwareProfile h = unified();
|
|
ComputeDevice bad;
|
|
bad.name = "broken";
|
|
bad.shares_host_memory = Tri::Yes;
|
|
h.devices.push_back(bad);
|
|
std::vector<std::string> asked;
|
|
probe_devices_isolated(h, [&](const std::string & dev, std::string & facts, std::string & why) {
|
|
asked.push_back(dev);
|
|
if (dev == "broken") {
|
|
why = "its probe ended the process that ran it (signal 11)";
|
|
return false;
|
|
}
|
|
ComputeDevice m = src;
|
|
facts = device_facts_to_text(m);
|
|
return true;
|
|
});
|
|
check(asked.size() == 2 && asked[0] == "accel" && asked[1] == "broken",
|
|
"isolated: every device but the host is asked");
|
|
check(h.devices[1].usable == Tri::Yes && h.devices[1].wide_matmul_gibs == 1502.25 && !h.devices[1].probe_here(),
|
|
"isolated: a device that answered carries its facts and is left alone by the probes here");
|
|
check(h.devices[2].usable == Tri::No && !h.devices[2].probe_here() &&
|
|
h.devices[2].unusable_reason.find("signal 11") != std::string::npos,
|
|
"isolated: a device that ended its probe is marked unusable, with the reason");
|
|
|
|
// And the plan: the broken device is kept out of the run, and the good one is still weighed.
|
|
h.host_wide_matmul_gibs = 200.0;
|
|
h.host_bandwidth_gibs = 100.0;
|
|
h.wide_batch = 128;
|
|
ModelProfile mm = model;
|
|
mm.largest_expert_layer_bytes = 384 * MiB;
|
|
const Plan with_good = plan_run(base_cfg(), h, mm, PlanRequest{});
|
|
check(with_good.config.prefill.device == "accel",
|
|
"isolated: the device that works is still armed for the prefill");
|
|
const Decision * dv = find(with_good, "devices");
|
|
check(dv && dv->source == Source::Measured && dv->reason.find("broken") != std::string::npos,
|
|
"isolated: and the plan names the one that is kept out");
|
|
HardwareProfile only_bad = unified();
|
|
only_bad.devices[1].usable = Tri::No;
|
|
only_bad.devices[1].unusable_reason = "its probe did not finish in 120 s and was stopped";
|
|
only_bad.devices[1].probed_out_of_process = true;
|
|
const Plan cpu_only = plan_run(base_cfg(), only_bad, mm, PlanRequest{});
|
|
check(cpu_only.config.device_use == DeviceUse::CpuOnly && !cpu_only.config.prefill.enabled() &&
|
|
cpu_only.config.moe.enabled,
|
|
"isolated: with nothing else to use the run is the host's, and it still streams");
|
|
check(validate(cpu_only.config).ok, "isolated: and that plan is a valid config",
|
|
validate(cpu_only.config).error);
|
|
check(cpu_only.to_flags().find("--devices cpu") != std::string::npos,
|
|
"isolated: the reproduce line carries the device choice", cpu_only.to_flags());
|
|
// No runner: nothing is marked, and the probes here do the asking as they always did.
|
|
HardwareProfile plain = unified();
|
|
probe_devices_isolated(plain, nullptr);
|
|
check(plain.devices[1].usable == Tri::Unknown && plain.devices[1].probe_here(),
|
|
"isolated: without a runner nothing changes");
|
|
}
|
|
|
|
// Informational: the rationale as a user would read it. Printed rather than asserted, because
|
|
// its wording is meant to change as the rules learn; what is asserted is that it exists, names
|
|
// the regime, and tags every decision with where its authority came from.
|
|
for (const auto & m : {std::make_pair("desktop", desktop()), std::make_pair("phone", phone()),
|
|
std::make_pair("hard-capped", hard_capped())}) {
|
|
std::printf("\n--- rationale on %s ---\n%s", m.first,
|
|
plan_run(base_cfg(), m.second, model, PlanRequest{}).explain().c_str());
|
|
}
|
|
|
|
std::printf(failures ? "\n%d check(s) failed\n" : "\nall checks passed\n", failures);
|
|
return failures ? 1 : 0;
|
|
}
|