97 lines
8.4 KiB
XML
97 lines
8.4 KiB
XML
<svg width="720" height="300" xmlns="http://www.w3.org/2000/svg">
|
||
<defs>
|
||
<marker id="mla-arrow" markerWidth="7" markerHeight="5" refX="7" refY="2.5" orient="auto">
|
||
<polygon points="0 0, 7 2.5, 0 5" fill="#555"/>
|
||
</marker>
|
||
</defs>
|
||
<text x="360" y="22" fill="#333" font-size="14" font-weight="bold" text-anchor="middle">Attention KV Cache Strategies</text>
|
||
|
||
<!-- MHA -->
|
||
<rect x="20" y="42" width="150" height="175" rx="10" fill="#e74c3c" opacity="0.05" stroke="#e74c3c" stroke-width="1.5"/>
|
||
<text x="95" y="62" fill="#e74c3c" font-size="11" font-weight="bold" text-anchor="middle">MHA (standard)</text>
|
||
<!-- Q heads -->
|
||
<rect x="35" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="56" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="77" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="98" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="119" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="140" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<text x="95" y="107" fill="#3498db" font-size="8" text-anchor="middle">Q heads</text>
|
||
<!-- K heads (one per Q) -->
|
||
<rect x="35" y="115" width="18" height="18" rx="3" fill="#e74c3c" opacity="0.3" stroke="#e74c3c" stroke-width="1"/>
|
||
<rect x="56" y="115" width="18" height="18" rx="3" fill="#e74c3c" opacity="0.3" stroke="#e74c3c" stroke-width="1"/>
|
||
<rect x="77" y="115" width="18" height="18" rx="3" fill="#e74c3c" opacity="0.3" stroke="#e74c3c" stroke-width="1"/>
|
||
<rect x="98" y="115" width="18" height="18" rx="3" fill="#e74c3c" opacity="0.3" stroke="#e74c3c" stroke-width="1"/>
|
||
<rect x="119" y="115" width="18" height="18" rx="3" fill="#e74c3c" opacity="0.3" stroke="#e74c3c" stroke-width="1"/>
|
||
<rect x="140" y="115" width="18" height="18" rx="3" fill="#e74c3c" opacity="0.3" stroke="#e74c3c" stroke-width="1"/>
|
||
<text x="95" y="148" fill="#e74c3c" font-size="8" text-anchor="middle">K heads (one per Q)</text>
|
||
<!-- V heads -->
|
||
<rect x="35" y="155" width="18" height="18" rx="3" fill="#27ae60" opacity="0.3" stroke="#27ae60" stroke-width="1"/>
|
||
<rect x="56" y="155" width="18" height="18" rx="3" fill="#27ae60" opacity="0.3" stroke="#27ae60" stroke-width="1"/>
|
||
<rect x="77" y="155" width="18" height="18" rx="3" fill="#27ae60" opacity="0.3" stroke="#27ae60" stroke-width="1"/>
|
||
<rect x="98" y="155" width="18" height="18" rx="3" fill="#27ae60" opacity="0.3" stroke="#27ae60" stroke-width="1"/>
|
||
<rect x="119" y="155" width="18" height="18" rx="3" fill="#27ae60" opacity="0.3" stroke="#27ae60" stroke-width="1"/>
|
||
<rect x="140" y="155" width="18" height="18" rx="3" fill="#27ae60" opacity="0.3" stroke="#27ae60" stroke-width="1"/>
|
||
<text x="95" y="188" fill="#27ae60" font-size="8" text-anchor="middle">V heads (one per Q)</text>
|
||
<text x="95" y="210" fill="#555" font-size="9" text-anchor="middle" font-weight="bold">KV cache: n·d per head</text>
|
||
|
||
<!-- GQA -->
|
||
<rect x="195" y="42" width="150" height="175" rx="10" fill="#f39c12" opacity="0.05" stroke="#f39c12" stroke-width="1.5"/>
|
||
<text x="270" y="62" fill="#f39c12" font-size="11" font-weight="bold" text-anchor="middle">GQA (Llama, Qwen)</text>
|
||
<!-- Q heads -->
|
||
<rect x="210" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="231" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="252" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="283" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="304" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="325" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<text x="270" y="107" fill="#3498db" font-size="8" text-anchor="middle">Q heads (6)</text>
|
||
<!-- KV heads (shared, 2 groups) -->
|
||
<rect x="220" y="115" width="30" height="18" rx="3" fill="#e74c3c" opacity="0.3" stroke="#e74c3c" stroke-width="1.5"/>
|
||
<rect x="293" y="115" width="30" height="18" rx="3" fill="#e74c3c" opacity="0.3" stroke="#e74c3c" stroke-width="1.5"/>
|
||
<text x="270" y="148" fill="#e74c3c" font-size="8" text-anchor="middle">K heads (2 shared)</text>
|
||
<rect x="220" y="155" width="30" height="18" rx="3" fill="#27ae60" opacity="0.3" stroke="#27ae60" stroke-width="1.5"/>
|
||
<rect x="293" y="155" width="30" height="18" rx="3" fill="#27ae60" opacity="0.3" stroke="#27ae60" stroke-width="1.5"/>
|
||
<text x="270" y="188" fill="#27ae60" font-size="8" text-anchor="middle">V heads (2 shared)</text>
|
||
<!-- Lines showing sharing -->
|
||
<line x1="235" y1="93" x2="235" y2="115" stroke="#999" stroke-width="0.8" stroke-dasharray="2,2"/>
|
||
<line x1="308" y1="93" x2="308" y2="115" stroke="#999" stroke-width="0.8" stroke-dasharray="2,2"/>
|
||
<text x="270" y="210" fill="#555" font-size="9" text-anchor="middle" font-weight="bold">KV cache: 3x reduction</text>
|
||
|
||
<!-- MLA -->
|
||
<rect x="370" y="42" width="170" height="175" rx="10" fill="#9b59b6" opacity="0.05" stroke="#9b59b6" stroke-width="1.5"/>
|
||
<text x="455" y="62" fill="#9b59b6" font-size="11" font-weight="bold" text-anchor="middle">MLA (DeepSeek, Kimi)</text>
|
||
<!-- Q heads -->
|
||
<rect x="385" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="406" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="427" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="458" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="479" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<rect x="500" y="75" width="18" height="18" rx="3" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
||
<text x="455" y="107" fill="#3498db" font-size="8" text-anchor="middle">Q heads (6)</text>
|
||
<!-- Compressed latent -->
|
||
<rect x="420" y="120" width="70" height="24" rx="5" fill="#9b59b6" opacity="0.2" stroke="#9b59b6" stroke-width="2"/>
|
||
<text x="455" y="136" fill="#9b59b6" font-size="9" text-anchor="middle" font-weight="bold">c = W↓h</text>
|
||
<text x="455" y="157" fill="#555" font-size="8" text-anchor="middle">Compressed latent (d_c=512)</text>
|
||
<!-- Up-project arrows -->
|
||
<line x1="430" y1="144" x2="415" y2="175" stroke="#e74c3c" stroke-width="1" marker-end="url(#mla-arrow)"/>
|
||
<line x1="480" y1="144" x2="495" y2="175" stroke="#27ae60" stroke-width="1" marker-end="url(#mla-arrow)"/>
|
||
<text x="410" y="190" fill="#e74c3c" font-size="8" text-anchor="middle">K=W↑c</text>
|
||
<text x="500" y="190" fill="#27ae60" font-size="8" text-anchor="middle">V=W↑c</text>
|
||
<text x="455" y="210" fill="#555" font-size="9" text-anchor="middle" font-weight="bold">KV cache: 93% reduction</text>
|
||
|
||
<!-- Decoupled RoPE note -->
|
||
<rect x="555" y="110" width="150" height="55" rx="8" fill="#f5f5f5" stroke="#ddd" stroke-width="1"/>
|
||
<text x="630" y="128" fill="#9b59b6" font-size="9" text-anchor="middle" font-weight="bold">Decoupled RoPE</text>
|
||
<text x="630" y="142" fill="#555" font-size="8" text-anchor="middle">Small Q,K stream (64d)</text>
|
||
<text x="630" y="155" fill="#555" font-size="8" text-anchor="middle">carries position via RoPE</text>
|
||
|
||
<!-- Bottom comparison -->
|
||
<rect x="60" y="232" width="600" height="55" rx="8" fill="#f5f5f5" stroke="#ddd" stroke-width="1"/>
|
||
<text x="160" y="252" fill="#e74c3c" font-size="9" text-anchor="middle" font-weight="bold">MHA: n_heads × d_head</text>
|
||
<text x="160" y="268" fill="#666" font-size="8" text-anchor="middle">e.g. 128×128 = 16,384 per token</text>
|
||
<text x="360" y="252" fill="#f39c12" font-size="9" text-anchor="middle" font-weight="bold">GQA: n_kv × d_head</text>
|
||
<text x="360" y="268" fill="#666" font-size="8" text-anchor="middle">e.g. 8×128 = 1,024 per token</text>
|
||
<text x="560" y="252" fill="#9b59b6" font-size="9" text-anchor="middle" font-weight="bold">MLA: d_c only</text>
|
||
<text x="560" y="268" fill="#666" font-size="8" text-anchor="middle">e.g. 512 per token</text>
|
||
</svg> |
