Model,task,0K,1K,2K,4K,8K,16K,32K,64K,128K,512K,1M,10M
GPT-2 (137M),avg,27,15,,,,,,,,,,
mamba-2.8b-hf,avg,70,52,35,9,0,,,,,,,
rwkv-6-world-7b,avg,56,55,48,35,7,,,,,,,
v5-Eagle-7B-HF,avg,62,54,48,41,2,,,,,,,
Meta-Llama-3-8B-Instruct,avg,64,60,58,50,44,,,,,,,
LLaMA-2-7B-32K,avg,41,53,45,40,39,32,3,,,,,
longchat-7b-v1.5-32k,avg,46,42,40,41,42,39,5,,,,,
LongAlpaca-13B,avg,48,47,46,43,40,36,4,,,,,
Llama-2-7B-32K-Instruct,avg,49,52,49,43,40,35,5,,,,,
01-ai/Yi-34B,avg,72,52,43,37,38,31,4,,,,,
Mistral-7b-Instruct-v0.2,avg,60,56,52,49,45,42,37,,,,,
Mixtral-8x7B-Instruct-v0.1,avg,65,63,60,55,50,46,40,,,,,
01-ai/Yi-34B-200k,avg,65,59,56,54,52,50,48,48,,,,
Mixtral-8x22B-Instruct-v0.1,avg,75,73,70,65,58,51,43,35,,,,
activation-beacon-llama2-7b-chat,avg,55,52,47,43,36,23,16,8,6,,,
Yarn-Mistral-7b-128k,avg,51,52,43,40,38,30,16,10,9,,,
chatglm3-6b-128k,avg,56,55,51,48,46,41,36,21,13,,,
activation-beacon-mistral-7b,avg,59,56,51,48,43,37,36,27,14,,,
01-ai/Yi-9B-200k,avg,52,55,48,46,45,36,37,29,24,,,
Phi-3-mini-128k-instruct,avg,64,57,55,51,50,46,42,37,7,,,
ai21labs/Jamba-v0.1,avg,65,53,50,48,46,45,41,40,34,,,
c4ai-command-r-v01,avg,64,64,63,61,59,52,51,46,38,,,
Meta-Llama-3.1-8B-Instruct,avg,67,68,66,66,62,60,56,49,39,,,
Phi-3-medium-128k-instruct,avg,72,70,67,62,60,57,53,45,30,,,
GPT-4,avg,87,81,77,74,71,64,53,43,36,,,
Meta-Llama-3.1-70B-Instruct,avg,85,81,78,74,70,65,59,53,45,,,
~ Mamba (130M) fine-tune,avg,,,,"98,7","98,5","98,5","98,1",97,"92,5",,,
Llama3-ChatQA-1.5-8B + RAG,avg,48,48,47,46,45,45,44,42,45,42,39,37
~ RMT (137M) fine-tune,avg,"99,36","97,4","94,66","92,32","89,9","85,62","77,88","69,86","58,52","46,36","42,84","33,78"
~ ARMT (137M) fine-tune,avg,"99,32",,,"98,1","98,2","98,1",98,"97,9","96,9","95,3","93,4","76,6"
GPT-2 (137M),qa1,35,13,,,,,,,,,,
mamba-2.8b-hf,qa1,65,56,40,7,1,,,,,,,
rwkv-6-world-7b,qa1,65,62,53,39,5,,,,,,,
v5-Eagle-7B-HF,qa1,68,58,52,36,3,,,,,,,
Meta-Llama-3-8B-Instruct,qa1,98,93,90,79,62,,,,,,,
LLaMA-2-7B-32K,qa1,54,57,33,26,34,32,3,,,,,
longchat-7b-v1.5-32k,qa1,52,60,56,55,50,42,4,,,,,
LongAlpaca-13B,qa1,58,55,58,50,37,23,2,,,,,
Llama-2-7B-32K-Instruct,qa1,65,61,52,41,35,23,3,,,,,
01-ai/Yi-34B,qa1,99,59,51,34,46,31,4,,,,,
Mistral-7b-Instruct-v0.2,qa1,92,86,75,64,63,57,45,,,,,
Mixtral-8x7B-Instruct-v0.1,qa1,99,92,84,77,65,53,49,,,,,
01-ai/Yi-34B-200k,qa1,85,73,68,66,63,65,62,60,,,,
Mixtral-8x22B-Instruct-v0.1,qa1,100,99,95,89,79,63,40,38,,,,
activation-beacon-llama2-7b-chat,qa1,85,81,67,65,48,21,16,6,5,,,
Yarn-Mistral-7b-128k,qa1,61,71,58,45,51,34,21,8,8,,,
chatglm3-6b-128k,qa1,82,77,74,72,67,56,47,13,13,,,
activation-beacon-mistral-7b,qa1,92,86,73,59,47,42,42,27,9,,,
01-ai/Yi-9B-200k,qa1,33,82,69,59,56,47,44,32,29,,,
Phi-3-mini-128k-instruct,qa1,97,84,72,69,70,60,53,38,1,,,
ai21labs/Jamba-v0.1,qa1,90,72,66,63,65,58,50,49,38,,,
c4ai-command-r-v01,qa1,98,95,94,91,89,68,70,50,30,,,
Phi-3-medium-128k-instruct,qa1,100,93,87,80,81,72,69,58,21,,,
GPT-4,qa1,99,100,100,95,93,84,62,58,39,,,
~ Mamba (130M) fine-tune,qa1,100,100,100,100,100,100,100,100,100,92,,
Llama3-ChatQA-1.5-8B + RAG,qa1,60,62,60,58,58,60,60,56,64,54,55,50
~ RMT (137M) fine-tune,qa1,100,100,100,100,100,100,99,96,94,87,84,66
~ ARMT (137M) fine-tune,qa1,100,,,100,100,100,100,100,100,99,99,89
GPT-2 (137M),qa2,21,17,,,,,,,,,,
mamba-2.8b-hf,qa2,68,47,24,8,0,,,,,,,
rwkv-6-world-7b,qa2,42,26,20,16,3,,,,,,,
v5-Eagle-7B-HF,qa2,43,27,25,19,2,,,,,,,
Meta-Llama-3-8B-Instruct,qa2,47,46,49,39,20,,,,,,,
LLaMA-2-7B-32K,qa2,37,40,37,26,27,21,2,,,,,
longchat-7b-v1.5-32k,qa2,31,19,14,13,18,14,2,,,,,
LongAlpaca-13B,qa2,30,30,28,29,26,18,2,,,,,
Llama-2-7B-32K-Instruct,qa2,41,40,36,27,20,16,3,,,,,
01-ai/Yi-34B,qa2,67,43,32,30,23,15,4,,,,,
Mistral-7b-Instruct-v0.2,qa2,46,34,32,22,11,9,7,,,,,
Mixtral-8x7B-Instruct-v0.1,qa2,58,51,46,35,27,18,14,,,,,
01-ai/Yi-34B-200k,qa2,62,51,46,42,35,32,27,30,,,,
Mixtral-8x22B-Instruct-v0.1,qa2,77,65,61,56,48,41,33,11,,,,
activation-beacon-llama2-7b-chat,qa2,52,37,28,25,23,11,9,4,2,,,
Yarn-Mistral-7b-128k,qa2,47,48,37,30,34,21,12,5,3,,,
chatglm3-6b-128k,qa2,51,46,45,39,38,31,24,6,5,,,
activation-beacon-mistral-7b,qa2,45,35,32,28,22,14,12,10,2,,,
01-ai/Yi-9B-200k,qa2,67,52,43,39,31,25,22,12,8,,,
Phi-3-mini-128k-instruct,qa2,57,38,38,36,34,23,22,15,2,,,
ai21labs/Jamba-v0.1,qa2,57,43,42,39,37,29,26,20,16,,,
c4ai-command-r-v01,qa2,64,58,56,54,50,39,37,32,16,,,
Phi-3-medium-128k-instruct,qa2,76,62,58,51,44,41,27,14,11,,,
GPT-4,qa2,88,79,72,68,65,59,42,25,25,,,
~ Mamba (130M) fine-tune,qa2,98,98,98,98,98,98,98,95,87,,,
Llama3-ChatQA-1.5-8B + RAG,qa2,28,25,22,19,14,13,9,7,6,6,2,2
~ RMT (137M) fine-tune,qa2,100,100,99,98,97,94,82,59,39,25,22,19
~ ARMT (137M) fine-tune,qa2,100,,,100,100,100,100,100,100,99,99,84
GPT-2 (137M),qa3,6,8,,,,,,,,,,
mamba-2.8b-hf,qa3,48,39,21,8,0,,,,,,,
rwkv-6-world-7b,qa3,40,45,28,24,4,,,,,,,
v5-Eagle-7B-HF,qa3,43,34,30,40,1,,,,,,,
Meta-Llama-3-8B-Instruct,qa3,33,28,30,26,11,,,,,,,
LLaMA-2-7B-32K,qa3,32,38,34,28,27,21,1,,,,,
longchat-7b-v1.5-32k,qa3,22,16,15,17,21,22,4,,,,,
LongAlpaca-13B,qa3,25,26,26,25,24,27,2,,,,,
Llama-2-7B-32K-Instruct,qa3,35,36,34,26,23,20,2,,,,,
01-ai/Yi-34B,qa3,45,34,24,20,17,12,4,,,,,
Mistral-7b-Instruct-v0.2,qa3,36,34,31,30,24,18,12,,,,,
Mixtral-8x7B-Instruct-v0.1,qa3,34,32,31,30,27,29,24,,,,,
01-ai/Yi-34B-200k,qa3,35,30,27,24,24,22,22,26,,,,
Mixtral-8x22B-Instruct-v0.1,qa3,53,56,49,39,31,27,26,26,,,,
activation-beacon-llama2-7b-chat,qa3,33,25,25,21,20,17,13,5,5,,,
Yarn-Mistral-7b-128k,qa3,31,36,33,32,27,25,9,13,7,,,
chatglm3-6b-128k,qa3,33,37,31,31,27,25,23,17,9,,,
activation-beacon-mistral-7b,qa3,36,33,25,21,18,15,15,15,16,,,
01-ai/Yi-9B-200k,qa3,34,33,29,24,25,21,20,20,8,,,
Phi-3-mini-128k-instruct,qa3,32,41,31,27,26,24,21,22,4,,,
ai21labs/Jamba-v0.1,qa3,32,31,29,26,24,22,22,21,26,,,
c4ai-command-r-v01,qa3,25,28,26,28,26,30,28,33,24,,,
Phi-3-medium-128k-instruct,qa3,53,51,45,35,30,30,27,25,17,,,
GPT-4,qa3,56,63,57,56,53,45,31,31,32,,,
~ Mamba (130M) fine-tune,qa3,97,97,97,97,97,96,95,92,81,,,
Llama3-ChatQA-1.5-8B + RAG,qa3,17,18,17,17,16,17,15,13,19,17,10,11
~ RMT (137M) fine-tune,qa3,97,94,88,81,73,66,55,55,36,25,22,21
~ ARMT (137M) fine-tune,qa3,97,,,92,92,92,91,90,86,80,72,37
GPT-2 (137M),qa4,29,18,,,,,,,,,,
mamba-2.8b-hf,qa4,96,59,47,12,0,,,,,,,
rwkv-6-world-7b,qa4,54,65,57,35,7,,,,,,,
v5-Eagle-7B-HF,qa4,79,74,63,55,3,,,,,,,
Meta-Llama-3-8B-Instruct,qa4,58,55,50,43,52,,,,,,,
LLaMA-2-7B-32K,qa4,26,54,51,51,46,36,3,,,,,
longchat-7b-v1.5-32k,qa4,60,55,52,57,57,49,4,,,,,
LongAlpaca-13B,qa4,65,61,58,52,50,44,4,,,,,
Llama-2-7B-32K-Instruct,qa4,39,52,54,56,55,52,6,,,,,
01-ai/Yi-34B,qa4,59,56,51,55,52,43,4,,,,,
Mistral-7b-Instruct-v0.2,qa4,54,58,58,60,60,58,54,,,,,
Mixtral-8x7B-Instruct-v0.1,qa4,55,60,59,61,63,61,58,,,,,
01-ai/Yi-34B-200k,qa4,64,65,64,63,61,56,54,44,,,,
Mixtral-8x22B-Instruct-v0.1,qa4,56,62,59,62,62,60,54,39,,,,
activation-beacon-llama2-7b-chat,qa4,40,50,52,43,34,22,14,9,10,,,
Yarn-Mistral-7b-128k,qa4,60,56,43,45,32,31,16,7,8,,,
chatglm3-6b-128k,qa4,45,48,42,38,35,32,27,13,11,,,
activation-beacon-mistral-7b,qa4,53,58,60,57,53,50,45,29,15,,,
01-ai/Yi-9B-200k,qa4,49,47,50,50,54,43,45,36,33,,,
Phi-3-mini-128k-instruct,qa4,54,56,56,50,49,50,45,47,5,,,
ai21labs/Jamba-v0.1,qa4,64,50,49,49,48,52,46,49,38,,,
c4ai-command-r-v01,qa4,46,58,59,54,56,46,46,47,52,,,
Phi-3-medium-128k-instruct,qa4,54,61,63,64,64,61,59,52,33,,,
GPT-4,qa4,98,70,63,60,52,47,46,40,32,,,
~ Mamba (130M) fine-tune,qa4,100,100,100,100,100,100,99,100,98,,,
Llama3-ChatQA-1.5-8B + RAG,qa4,53,58,56,59,57,60,60,59,60,59,54,56
~ RMT (137M) fine-tune,qa4,100,94,87,83,80,75,64,51,38,26,24,20
~ ARMT (137M) fine-tune,qa4,100,,,100,100,100,100,100,100,100,100,92
GPT-2 (137M),qa5,45,19,,,,,,,,,,
mamba-2.8b-hf,qa5,75,58,43,9,0,,,,,,,
rwkv-6-world-7b,qa5,79,77,80,61,14,,,,,,,
v5-Eagle-7B-HF,qa5,75,76,71,57,3,,,,,,,
Meta-Llama-3-8B-Instruct,qa5,85,78,73,65,73,,,,,,,
LLaMA-2-7B-32K,qa5,55,74,70,67,59,51,7,,,,,
longchat-7b-v1.5-32k,qa5,63,62,62,65,66,67,9,,,,,
LongAlpaca-13B,qa5,63,61,61,61,62,66,12,,,,,
Llama-2-7B-32K-Instruct,qa5,63,69,69,67,66,63,9,,,,,
01-ai/Yi-34B,qa5,88,70,59,48,53,55,4,,,,,
Mistral-7b-Instruct-v0.2,qa5,70,66,66,67,69,67,67,,,,,
Mixtral-8x7B-Instruct-v0.1,qa5,80,79,80,73,66,67,56,,,,,
01-ai/Yi-34B-200k,qa5,78,77,77,76,76,75,76,80,,,,
Mixtral-8x22B-Instruct-v0.1,qa5,87,84,84,79,69,64,63,63,,,,
activation-beacon-llama2-7b-chat,qa5,65,67,64,63,57,45,29,17,7,,,
Yarn-Mistral-7b-128k,qa5,58,47,45,47,47,38,23,17,19,,,
chatglm3-6b-128k,qa5,70,69,64,60,61,61,58,55,26,,,
activation-beacon-mistral-7b,qa5,68,66,66,74,74,66,67,55,28,,,
01-ai/Yi-9B-200k,qa5,76,59,50,57,57,45,52,47,40,,,
Phi-3-mini-128k-instruct,qa5,79,66,76,72,72,73,71,64,23,,,
ai21labs/Jamba-v0.1,qa5,83,70,64,62,58,64,63,60,50,,,
c4ai-command-r-v01,qa5,86,82,81,78,75,79,72,70,66,,,
Phi-3-medium-128k-instruct,qa5,77,85,84,81,82,82,81,78,69,,,
GPT-4,qa5,96,95,92,90,93,85,82,60,51,,,
~ Mamba (130M) fine-tune,qa5,98,99,98,99,99,99,98,99,98,,,
Llama3-ChatQA-1.5-8B + RAG,qa5,80,77,78,77,78,77,78,76,75,75,76,67
~ RMT (137M) fine-tune,qa5,100,100,99,99,99,94,90,89,86,69,63,44
~ ARMT (137M) fine-tune,qa5,"99,6",,,"98,1","98,2","98,1",98,"97,9","96,9","95,3","93,4","76,6"