WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel

断松 kilo-size-japanese-text-embedding-models

枯松や 庭にはいらぬ 御粗松の 我枝をかり 断松を聞く

(多分国内で極小の)20k~60kパラメータの日本語テキスト埋め込みモデルです。 ベンチマークも含まれています。 気になったら測ったり監査してみてね!!

20k model

architecture

rwkv7 hidden size 16dim 2 layers chr + bpe hybrid tokenizer (vocab size 512)

scores

JSTS

"pearson": 0.589115,
"spearman": 0.579923,
"cosine_pearson": 0.589115,
"cosine_spearman": 0.579923,
"manhattan_pearson": 0.601691,
"manhattan_spearman": 0.571704,
"euclidean_pearson": 0.60794,
"euclidean_spearman": 0.578109,
"main_score": 0.579923,

JaQuADRetrieval

"hit_rate_at_1": 0.07422,
"hit_rate_at_3": 0.14893,
"hit_rate_at_5": 0.2002,
"hit_rate_at_10": 0.27441,
"hit_rate_at_20": 0.3877,
"hit_rate_at_100": 0.67139,
"hit_rate_at_1000": 0.979,
"main_score": 0.10268,

---

40k model

architecture

rwkv7 hidden size 16dim 2 layers chr + bpe hybrid tokenizer (vocab size 512)

scores

JSTS

"pearson": 0.622386,
"spearman": 0.601339,
"cosine_pearson": 0.622386,
"cosine_spearman": 0.601339,
"manhattan_pearson": 0.626993,
"manhattan_spearman": 0.594981,
"euclidean_pearson": 0.634518,
"euclidean_spearman": 0.601892,
"main_score": 0.601339,

JaQuADRetrieval

"hit_rate_at_1": 0.12988,
"hit_rate_at_3": 0.23926,
"hit_rate_at_5"20k~60kパラメータの: 0.30713,
"hit_rate_at_10": 0.41113,
"hit_rate_at_20": 0.51318,
"hit_rate_at_100": 0.74707,
"hit_rate_at_1000": 0.98096,
"main_score": 0.16319,

---

60k model

architecture

rwkv7 hidden size 32dim 2 layers chr + bpe hybrid tokenizer (vocab size 512)

scores

JSTS

"pearson": 0.685757,
"spearman": 0.656014,
"cosine_pearson": 0.685757,
"cosine_spearman": 0.656014,
"manhattan_pearson": 0.679063,
"manhattan_spearman": 0.652057,
"euclidean_pearson": 0.682095,
"euclidean_spearman": 0.655796,
"main_score": 0.656014,

JaQuADRetrieval

"hit_rate_at_1": 0.1792,
"hit_rate_at_3": 0.31494,
"hit_rate_at_5": 0.39404,
"hit_rate_at_10": 0.49805,
"hit_rate_at_20": 0.60254,
"hit_rate_at_100": 0.79932,
"hit_rate_at_1000": 0.98682,
"main_score": 0.20629,