{ "host": { "cpu_model": "Intel(R) Xeon(R) Platinum 8559C", "cpu_threads": 192, "mem_total_bytes": 2147448086528, "cgroup": { "memory.max": "104000000000", "memory.swap.max": "2936848384", "memory.high": "max" } }, "config": { "model": "models/Ling-3.0-tiny-Q4_K_M.gguf", "shards": [ "models/Ling-3.0-tiny-Q4_K_M.gguf" ], "bin": "llama.cpp/build/bin/sdq-chat", "levels_mb": [ 1024, 2048, 4096, 8192, 16384 ], "repeat": 2, "ctx": 256, "ubatch": 32, "threads": 4, "n_predict": 32, "reserve_mb": 1024, "timeout": 3600, "evict_helper": "./pagecache_resident", "reserve_map": "1024:700,2048:700,4096:700,8192:1000,16384:1400", "output_dir": "results/strict_runs", "out_json": "results/strict_memory_levels.json", "_reserve": { "1024": 700, "2048": 700, "4096": 700, "8192": 1000, "16384": 1400 } }, "runs": [ { "level_mb": 1024, "level_gib": 1.0, "repeat": 1, "wall_s": 7.63, "exit_code": 0, "peak_rss_mb": 904.9, "peak_anon_mb": 420.9, "peak_swap_mb": 0.0, "swap_free": true, "anon_within_limit": true, "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", "status": "OK", "decode_tok_s": 8.23, "decode_tok": 31, "prefill_tok_s": 17.7, "prefill_tok": 56, "final_rss_mb": 390, "selfpeak_rss_mb": 390, "self_swap_mb": 0, "rss_anon_mb": 382, "rss_file_mb": 8, "rss_total_mb": 390, "arena_gib": 0.3, "n_slots": 225, "total_ssd_gb": 10.33, "decode_ssd_mb": 6189.6, "ssd_mb_per_token": 199.7, "expert_requests": 8769, "expert_hits": 1470, "hot_rate_pct": 16.7636, "cold_rate_pct": 83.2364, "evictions": 7074, "per_layer_hit_min": 0, "per_layer_hit_max": 37.9121, "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" }, { "level_mb": 1024, "level_gib": 1.0, "repeat": 2, "wall_s": 7.65, "exit_code": 0, "peak_rss_mb": 904.3, "peak_anon_mb": 420.9, "peak_swap_mb": 0.0, "swap_free": true, "anon_within_limit": true, "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", "status": "OK", "decode_tok_s": 8.27, "decode_tok": 31, "prefill_tok_s": 17.7, "prefill_tok": 56, "final_rss_mb": 390, "selfpeak_rss_mb": 390, "self_swap_mb": 0, "rss_anon_mb": 382, "rss_file_mb": 7, "rss_total_mb": 390, "arena_gib": 0.3, "n_slots": 225, "total_ssd_gb": 10.34, "decode_ssd_mb": 6193.5, "ssd_mb_per_token": 199.8, "expert_requests": 8769, "expert_hits": 1467, "hot_rate_pct": 16.7294, "cold_rate_pct": 83.2706, "evictions": 7077, "per_layer_hit_min": 0, "per_layer_hit_max": 37.9121, "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" }, { "level_mb": 2048, "level_gib": 2.0, "repeat": 1, "wall_s": 7.13, "exit_code": 0, "peak_rss_mb": 1894.9, "peak_anon_mb": 1411.9, "peak_swap_mb": 0.0, "swap_free": true, "anon_within_limit": true, "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", "status": "OK", "decode_tok_s": 12.01, "decode_tok": 31, "prefill_tok_s": 17.7, "prefill_tok": 56, "final_rss_mb": 1381, "selfpeak_rss_mb": 1381, "self_swap_mb": 0, "rss_anon_mb": 1373, "rss_file_mb": 7, "rss_total_mb": 1381, "arena_gib": 1.3, "n_slots": 979, "total_ssd_gb": 6.46, "decode_ssd_mb": 2287.7, "ssd_mb_per_token": 73.8, "expert_requests": 8769, "expert_hits": 4338, "hot_rate_pct": 49.4697, "cold_rate_pct": 50.5303, "evictions": 3452, "per_layer_hit_min": 26.7157, "per_layer_hit_max": 75.0663, "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" }, { "level_mb": 2048, "level_gib": 2.0, "repeat": 2, "wall_s": 7.19, "exit_code": 0, "peak_rss_mb": 1895.3, "peak_anon_mb": 1411.9, "peak_swap_mb": 0.0, "swap_free": true, "anon_within_limit": true, "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", "status": "OK", "decode_tok_s": 11.75, "decode_tok": 31, "prefill_tok_s": 17.3, "prefill_tok": 56, "final_rss_mb": 1381, "selfpeak_rss_mb": 1381, "self_swap_mb": 0, "rss_anon_mb": 1373, "rss_file_mb": 7, "rss_total_mb": 1381, "arena_gib": 1.3, "n_slots": 979, "total_ssd_gb": 6.46, "decode_ssd_mb": 2287.5, "ssd_mb_per_token": 73.8, "expert_requests": 8769, "expert_hits": 4339, "hot_rate_pct": 49.4811, "cold_rate_pct": 50.5189, "evictions": 3451, "per_layer_hit_min": 26.7157, "per_layer_hit_max": 75.0663, "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" }, { "level_mb": 4096, "level_gib": 4.0, "repeat": 1, "wall_s": 7.9, "exit_code": 0, "peak_rss_mb": 3887.7, "peak_anon_mb": 3404.0, "peak_swap_mb": 0.0, "swap_free": true, "anon_within_limit": true, "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", "status": "OK", "decode_tok_s": 15.83, "decode_tok": 31, "prefill_tok_s": 17.0, "prefill_tok": 56, "final_rss_mb": 3362, "selfpeak_rss_mb": 3362, "self_swap_mb": 0, "rss_anon_mb": 3354, "rss_file_mb": 8, "rss_total_mb": 3362, "arena_gib": 3.3, "n_slots": 2486, "total_ssd_gb": 4.43, "decode_ssd_mb": 767.5, "ssd_mb_per_token": 24.8, "expert_requests": 8786, "expert_hits": 5864, "hot_rate_pct": 66.7425, "cold_rate_pct": 33.25749999999999, "evictions": 1231, "per_layer_hit_min": 40.9314, "per_layer_hit_max": 78.1003, "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" }, { "level_mb": 4096, "level_gib": 4.0, "repeat": 2, "wall_s": 7.83, "exit_code": 0, "peak_rss_mb": 3887.5, "peak_anon_mb": 3403.9, "peak_swap_mb": 0.0, "swap_free": true, "anon_within_limit": true, "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", "status": "OK", "decode_tok_s": 16.92, "decode_tok": 31, "prefill_tok_s": 17.0, "prefill_tok": 56, "final_rss_mb": 3362, "selfpeak_rss_mb": 3362, "self_swap_mb": 0, "rss_anon_mb": 3354, "rss_file_mb": 8, "rss_total_mb": 3362, "arena_gib": 3.3, "n_slots": 2486, "total_ssd_gb": 4.43, "decode_ssd_mb": 767.5, "ssd_mb_per_token": 24.8, "expert_requests": 8786, "expert_hits": 5864, "hot_rate_pct": 66.7425, "cold_rate_pct": 33.25749999999999, "evictions": 1231, "per_layer_hit_min": 40.9314, "per_layer_hit_max": 78.1003, "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" }, { "level_mb": 8192, "level_gib": 8.0, "repeat": 1, "wall_s": 11.19, "exit_code": 0, "peak_rss_mb": 7548.3, "peak_anon_mb": 7064.2, "peak_swap_mb": 0.0, "swap_free": true, "anon_within_limit": true, "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", "status": "OK", "decode_tok_s": 17.72, "decode_tok": 31, "prefill_tok_s": 14.0, "prefill_tok": 56, "final_rss_mb": 7034, "selfpeak_rss_mb": 7034, "self_swap_mb": 0, "rss_anon_mb": 7026, "rss_file_mb": 8, "rss_total_mb": 7034, "arena_gib": 7.01, "n_slots": 5279, "total_ssd_gb": 3.22, "decode_ssd_mb": 220.5, "ssd_mb_per_token": 7.1, "expert_requests": 8769, "expert_hits": 6689, "hot_rate_pct": 76.2801, "cold_rate_pct": 23.719899999999996, "evictions": 0, "per_layer_hit_min": 73.75, "per_layer_hit_max": 78.1003, "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" }, { "level_mb": 8192, "level_gib": 8.0, "repeat": 2, "wall_s": 10.57, "exit_code": 0, "peak_rss_mb": 7547.8, "peak_anon_mb": 7064.2, "peak_swap_mb": 0.0, "swap_free": true, "anon_within_limit": true, "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", "status": "OK", "decode_tok_s": 19.57, "decode_tok": 31, "prefill_tok_s": 15.7, "prefill_tok": 56, "final_rss_mb": 7034, "selfpeak_rss_mb": 7034, "self_swap_mb": 0, "rss_anon_mb": 7026, "rss_file_mb": 8, "rss_total_mb": 7034, "arena_gib": 7.01, "n_slots": 5279, "total_ssd_gb": 3.22, "decode_ssd_mb": 220.5, "ssd_mb_per_token": 7.1, "expert_requests": 8769, "expert_hits": 6689, "hot_rate_pct": 76.2801, "cold_rate_pct": 23.719899999999996, "evictions": 0, "per_layer_hit_min": 73.75, "per_layer_hit_max": 78.1003, "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" }, { "level_mb": 16384, "level_gib": 16.0, "repeat": 1, "wall_s": 17.55, "exit_code": 0, "peak_rss_mb": 15084.0, "peak_anon_mb": 14600.4, "peak_swap_mb": 0.0, "swap_free": true, "anon_within_limit": true, "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", "status": "OK", "decode_tok_s": 19.23, "decode_tok": 31, "prefill_tok_s": 10.4, "prefill_tok": 56, "final_rss_mb": 14571, "selfpeak_rss_mb": 14571, "self_swap_mb": 0, "rss_anon_mb": 14563, "rss_file_mb": 7, "rss_total_mb": 14571, "arena_gib": 14.62, "n_slots": 11013, "total_ssd_gb": 3.22, "decode_ssd_mb": 223.4, "ssd_mb_per_token": 7.2, "expert_requests": 8763, "expert_hits": 6683, "hot_rate_pct": 76.2638, "cold_rate_pct": 23.736199999999997, "evictions": 0, "per_layer_hit_min": 73.75, "per_layer_hit_max": 78.1003, "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" }, { "level_mb": 16384, "level_gib": 16.0, "repeat": 2, "wall_s": 17.45, "exit_code": 0, "peak_rss_mb": 15084.1, "peak_anon_mb": 14600.3, "peak_swap_mb": 0.0, "swap_free": true, "anon_within_limit": true, "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", "status": "OK", "decode_tok_s": 19.25, "decode_tok": 31, "prefill_tok_s": 10.3, "prefill_tok": 56, "final_rss_mb": 14570, "selfpeak_rss_mb": 14570, "self_swap_mb": 0, "rss_anon_mb": 14562, "rss_file_mb": 8, "rss_total_mb": 14570, "arena_gib": 14.62, "n_slots": 11012, "total_ssd_gb": 3.22, "decode_ssd_mb": 219.0, "ssd_mb_per_token": 7.1, "expert_requests": 8773, "expert_hits": 6692, "hot_rate_pct": 76.2795, "cold_rate_pct": 23.7205, "evictions": 0, "per_layer_hit_min": 73.75, "per_layer_hit_max": 78.1003, "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" } ] }