| { |
| "host": { |
| "cpu_model": "Intel(R) Xeon(R) Platinum 8559C", |
| "cpu_threads": 192, |
| "mem_total_bytes": 2147448086528, |
| "cgroup": { |
| "memory.max": "104000000000", |
| "memory.swap.max": "2936848384", |
| "memory.high": "max" |
| } |
| }, |
| "config": { |
| "model": "models/Ling-3.0-tiny-Q4_K_M.gguf", |
| "shards": [ |
| "models/Ling-3.0-tiny-Q4_K_M.gguf" |
| ], |
| "bin": "llama.cpp/build/bin/sdq-chat", |
| "levels_mb": [ |
| 1024, |
| 2048, |
| 4096, |
| 8192, |
| 16384 |
| ], |
| "repeat": 2, |
| "ctx": 256, |
| "ubatch": 32, |
| "threads": 4, |
| "n_predict": 32, |
| "reserve_mb": 1024, |
| "timeout": 3600, |
| "evict_helper": "./pagecache_resident", |
| "reserve_map": "1024:700,2048:700,4096:700,8192:1000,16384:1400", |
| "output_dir": "results/strict_runs", |
| "out_json": "results/strict_memory_levels.json", |
| "_reserve": { |
| "1024": 700, |
| "2048": 700, |
| "4096": 700, |
| "8192": 1000, |
| "16384": 1400 |
| } |
| }, |
| "runs": [ |
| { |
| "level_mb": 1024, |
| "level_gib": 1.0, |
| "repeat": 1, |
| "wall_s": 7.63, |
| "exit_code": 0, |
| "peak_rss_mb": 904.9, |
| "peak_anon_mb": 420.9, |
| "peak_swap_mb": 0.0, |
| "swap_free": true, |
| "anon_within_limit": true, |
| "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", |
| "status": "OK", |
| "decode_tok_s": 8.23, |
| "decode_tok": 31, |
| "prefill_tok_s": 17.7, |
| "prefill_tok": 56, |
| "final_rss_mb": 390, |
| "selfpeak_rss_mb": 390, |
| "self_swap_mb": 0, |
| "rss_anon_mb": 382, |
| "rss_file_mb": 8, |
| "rss_total_mb": 390, |
| "arena_gib": 0.3, |
| "n_slots": 225, |
| "total_ssd_gb": 10.33, |
| "decode_ssd_mb": 6189.6, |
| "ssd_mb_per_token": 199.7, |
| "expert_requests": 8769, |
| "expert_hits": 1470, |
| "hot_rate_pct": 16.7636, |
| "cold_rate_pct": 83.2364, |
| "evictions": 7074, |
| "per_layer_hit_min": 0, |
| "per_layer_hit_max": 37.9121, |
| "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" |
| }, |
| { |
| "level_mb": 1024, |
| "level_gib": 1.0, |
| "repeat": 2, |
| "wall_s": 7.65, |
| "exit_code": 0, |
| "peak_rss_mb": 904.3, |
| "peak_anon_mb": 420.9, |
| "peak_swap_mb": 0.0, |
| "swap_free": true, |
| "anon_within_limit": true, |
| "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", |
| "status": "OK", |
| "decode_tok_s": 8.27, |
| "decode_tok": 31, |
| "prefill_tok_s": 17.7, |
| "prefill_tok": 56, |
| "final_rss_mb": 390, |
| "selfpeak_rss_mb": 390, |
| "self_swap_mb": 0, |
| "rss_anon_mb": 382, |
| "rss_file_mb": 7, |
| "rss_total_mb": 390, |
| "arena_gib": 0.3, |
| "n_slots": 225, |
| "total_ssd_gb": 10.34, |
| "decode_ssd_mb": 6193.5, |
| "ssd_mb_per_token": 199.8, |
| "expert_requests": 8769, |
| "expert_hits": 1467, |
| "hot_rate_pct": 16.7294, |
| "cold_rate_pct": 83.2706, |
| "evictions": 7077, |
| "per_layer_hit_min": 0, |
| "per_layer_hit_max": 37.9121, |
| "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" |
| }, |
| { |
| "level_mb": 2048, |
| "level_gib": 2.0, |
| "repeat": 1, |
| "wall_s": 7.13, |
| "exit_code": 0, |
| "peak_rss_mb": 1894.9, |
| "peak_anon_mb": 1411.9, |
| "peak_swap_mb": 0.0, |
| "swap_free": true, |
| "anon_within_limit": true, |
| "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", |
| "status": "OK", |
| "decode_tok_s": 12.01, |
| "decode_tok": 31, |
| "prefill_tok_s": 17.7, |
| "prefill_tok": 56, |
| "final_rss_mb": 1381, |
| "selfpeak_rss_mb": 1381, |
| "self_swap_mb": 0, |
| "rss_anon_mb": 1373, |
| "rss_file_mb": 7, |
| "rss_total_mb": 1381, |
| "arena_gib": 1.3, |
| "n_slots": 979, |
| "total_ssd_gb": 6.46, |
| "decode_ssd_mb": 2287.7, |
| "ssd_mb_per_token": 73.8, |
| "expert_requests": 8769, |
| "expert_hits": 4338, |
| "hot_rate_pct": 49.4697, |
| "cold_rate_pct": 50.5303, |
| "evictions": 3452, |
| "per_layer_hit_min": 26.7157, |
| "per_layer_hit_max": 75.0663, |
| "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" |
| }, |
| { |
| "level_mb": 2048, |
| "level_gib": 2.0, |
| "repeat": 2, |
| "wall_s": 7.19, |
| "exit_code": 0, |
| "peak_rss_mb": 1895.3, |
| "peak_anon_mb": 1411.9, |
| "peak_swap_mb": 0.0, |
| "swap_free": true, |
| "anon_within_limit": true, |
| "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", |
| "status": "OK", |
| "decode_tok_s": 11.75, |
| "decode_tok": 31, |
| "prefill_tok_s": 17.3, |
| "prefill_tok": 56, |
| "final_rss_mb": 1381, |
| "selfpeak_rss_mb": 1381, |
| "self_swap_mb": 0, |
| "rss_anon_mb": 1373, |
| "rss_file_mb": 7, |
| "rss_total_mb": 1381, |
| "arena_gib": 1.3, |
| "n_slots": 979, |
| "total_ssd_gb": 6.46, |
| "decode_ssd_mb": 2287.5, |
| "ssd_mb_per_token": 73.8, |
| "expert_requests": 8769, |
| "expert_hits": 4339, |
| "hot_rate_pct": 49.4811, |
| "cold_rate_pct": 50.5189, |
| "evictions": 3451, |
| "per_layer_hit_min": 26.7157, |
| "per_layer_hit_max": 75.0663, |
| "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" |
| }, |
| { |
| "level_mb": 4096, |
| "level_gib": 4.0, |
| "repeat": 1, |
| "wall_s": 7.9, |
| "exit_code": 0, |
| "peak_rss_mb": 3887.7, |
| "peak_anon_mb": 3404.0, |
| "peak_swap_mb": 0.0, |
| "swap_free": true, |
| "anon_within_limit": true, |
| "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", |
| "status": "OK", |
| "decode_tok_s": 15.83, |
| "decode_tok": 31, |
| "prefill_tok_s": 17.0, |
| "prefill_tok": 56, |
| "final_rss_mb": 3362, |
| "selfpeak_rss_mb": 3362, |
| "self_swap_mb": 0, |
| "rss_anon_mb": 3354, |
| "rss_file_mb": 8, |
| "rss_total_mb": 3362, |
| "arena_gib": 3.3, |
| "n_slots": 2486, |
| "total_ssd_gb": 4.43, |
| "decode_ssd_mb": 767.5, |
| "ssd_mb_per_token": 24.8, |
| "expert_requests": 8786, |
| "expert_hits": 5864, |
| "hot_rate_pct": 66.7425, |
| "cold_rate_pct": 33.25749999999999, |
| "evictions": 1231, |
| "per_layer_hit_min": 40.9314, |
| "per_layer_hit_max": 78.1003, |
| "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" |
| }, |
| { |
| "level_mb": 4096, |
| "level_gib": 4.0, |
| "repeat": 2, |
| "wall_s": 7.83, |
| "exit_code": 0, |
| "peak_rss_mb": 3887.5, |
| "peak_anon_mb": 3403.9, |
| "peak_swap_mb": 0.0, |
| "swap_free": true, |
| "anon_within_limit": true, |
| "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", |
| "status": "OK", |
| "decode_tok_s": 16.92, |
| "decode_tok": 31, |
| "prefill_tok_s": 17.0, |
| "prefill_tok": 56, |
| "final_rss_mb": 3362, |
| "selfpeak_rss_mb": 3362, |
| "self_swap_mb": 0, |
| "rss_anon_mb": 3354, |
| "rss_file_mb": 8, |
| "rss_total_mb": 3362, |
| "arena_gib": 3.3, |
| "n_slots": 2486, |
| "total_ssd_gb": 4.43, |
| "decode_ssd_mb": 767.5, |
| "ssd_mb_per_token": 24.8, |
| "expert_requests": 8786, |
| "expert_hits": 5864, |
| "hot_rate_pct": 66.7425, |
| "cold_rate_pct": 33.25749999999999, |
| "evictions": 1231, |
| "per_layer_hit_min": 40.9314, |
| "per_layer_hit_max": 78.1003, |
| "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" |
| }, |
| { |
| "level_mb": 8192, |
| "level_gib": 8.0, |
| "repeat": 1, |
| "wall_s": 11.19, |
| "exit_code": 0, |
| "peak_rss_mb": 7548.3, |
| "peak_anon_mb": 7064.2, |
| "peak_swap_mb": 0.0, |
| "swap_free": true, |
| "anon_within_limit": true, |
| "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", |
| "status": "OK", |
| "decode_tok_s": 17.72, |
| "decode_tok": 31, |
| "prefill_tok_s": 14.0, |
| "prefill_tok": 56, |
| "final_rss_mb": 7034, |
| "selfpeak_rss_mb": 7034, |
| "self_swap_mb": 0, |
| "rss_anon_mb": 7026, |
| "rss_file_mb": 8, |
| "rss_total_mb": 7034, |
| "arena_gib": 7.01, |
| "n_slots": 5279, |
| "total_ssd_gb": 3.22, |
| "decode_ssd_mb": 220.5, |
| "ssd_mb_per_token": 7.1, |
| "expert_requests": 8769, |
| "expert_hits": 6689, |
| "hot_rate_pct": 76.2801, |
| "cold_rate_pct": 23.719899999999996, |
| "evictions": 0, |
| "per_layer_hit_min": 73.75, |
| "per_layer_hit_max": 78.1003, |
| "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" |
| }, |
| { |
| "level_mb": 8192, |
| "level_gib": 8.0, |
| "repeat": 2, |
| "wall_s": 10.57, |
| "exit_code": 0, |
| "peak_rss_mb": 7547.8, |
| "peak_anon_mb": 7064.2, |
| "peak_swap_mb": 0.0, |
| "swap_free": true, |
| "anon_within_limit": true, |
| "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", |
| "status": "OK", |
| "decode_tok_s": 19.57, |
| "decode_tok": 31, |
| "prefill_tok_s": 15.7, |
| "prefill_tok": 56, |
| "final_rss_mb": 7034, |
| "selfpeak_rss_mb": 7034, |
| "self_swap_mb": 0, |
| "rss_anon_mb": 7026, |
| "rss_file_mb": 8, |
| "rss_total_mb": 7034, |
| "arena_gib": 7.01, |
| "n_slots": 5279, |
| "total_ssd_gb": 3.22, |
| "decode_ssd_mb": 220.5, |
| "ssd_mb_per_token": 7.1, |
| "expert_requests": 8769, |
| "expert_hits": 6689, |
| "hot_rate_pct": 76.2801, |
| "cold_rate_pct": 23.719899999999996, |
| "evictions": 0, |
| "per_layer_hit_min": 73.75, |
| "per_layer_hit_max": 78.1003, |
| "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" |
| }, |
| { |
| "level_mb": 16384, |
| "level_gib": 16.0, |
| "repeat": 1, |
| "wall_s": 17.55, |
| "exit_code": 0, |
| "peak_rss_mb": 15084.0, |
| "peak_anon_mb": 14600.4, |
| "peak_swap_mb": 0.0, |
| "swap_free": true, |
| "anon_within_limit": true, |
| "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", |
| "status": "OK", |
| "decode_tok_s": 19.23, |
| "decode_tok": 31, |
| "prefill_tok_s": 10.4, |
| "prefill_tok": 56, |
| "final_rss_mb": 14571, |
| "selfpeak_rss_mb": 14571, |
| "self_swap_mb": 0, |
| "rss_anon_mb": 14563, |
| "rss_file_mb": 7, |
| "rss_total_mb": 14571, |
| "arena_gib": 14.62, |
| "n_slots": 11013, |
| "total_ssd_gb": 3.22, |
| "decode_ssd_mb": 223.4, |
| "ssd_mb_per_token": 7.2, |
| "expert_requests": 8763, |
| "expert_hits": 6683, |
| "hot_rate_pct": 76.2638, |
| "cold_rate_pct": 23.736199999999997, |
| "evictions": 0, |
| "per_layer_hit_min": 73.75, |
| "per_layer_hit_max": 78.1003, |
| "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" |
| }, |
| { |
| "level_mb": 16384, |
| "level_gib": 16.0, |
| "repeat": 2, |
| "wall_s": 17.45, |
| "exit_code": 0, |
| "peak_rss_mb": 15084.1, |
| "peak_anon_mb": 14600.3, |
| "peak_swap_mb": 0.0, |
| "swap_free": true, |
| "anon_within_limit": true, |
| "limit_enforced_by": "prlimit --data (RLIMIT_DATA)", |
| "status": "OK", |
| "decode_tok_s": 19.25, |
| "decode_tok": 31, |
| "prefill_tok_s": 10.3, |
| "prefill_tok": 56, |
| "final_rss_mb": 14570, |
| "selfpeak_rss_mb": 14570, |
| "self_swap_mb": 0, |
| "rss_anon_mb": 14562, |
| "rss_file_mb": 8, |
| "rss_total_mb": 14570, |
| "arena_gib": 14.62, |
| "n_slots": 11012, |
| "total_ssd_gb": 3.22, |
| "decode_ssd_mb": 219.0, |
| "ssd_mb_per_token": 7.1, |
| "expert_requests": 8773, |
| "expert_hits": 6692, |
| "hot_rate_pct": 76.2795, |
| "cold_rate_pct": 23.7205, |
| "evictions": 0, |
| "per_layer_hit_min": 73.75, |
| "per_layer_hit_max": 78.1003, |
| "assistant": "Alright, the user is asking about how a Mixture-of-Experts (MoE) model routes tokens to expert weights and why paging those weights from" |
| } |
| ] |
| } |
