72 lines
5.0 KiB
XML
72 lines
5.0 KiB
XML
<svg width="700" height="300" xmlns="http://www.w3.org/2000/svg">
|
|
<defs>
|
|
<marker id="moe-arrow" markerWidth="8" markerHeight="6" refX="8" refY="3" orient="auto">
|
|
<path d="M0,0 L8,3 L0,6" fill="none" stroke="#555" stroke-width="1"/>
|
|
</marker>
|
|
</defs>
|
|
<text x="350" y="22" fill="#333" font-size="14" font-weight="bold" text-anchor="middle">Mixture of Experts: Sparse Activation</text>
|
|
|
|
<!-- Input tokens -->
|
|
<text x="100" y="50" fill="#666" font-size="11" text-anchor="middle">Input tokens</text>
|
|
<rect x="40" y="58" width="40" height="25" rx="4" fill="#3498db" opacity="0.3" stroke="#3498db" stroke-width="1"/>
|
|
<text x="60" y="76" fill="#333" font-size="9" text-anchor="middle">tok 1</text>
|
|
<rect x="85" y="58" width="40" height="25" rx="4" fill="#e74c3c" opacity="0.3" stroke="#e74c3c" stroke-width="1"/>
|
|
<text x="105" y="76" fill="#333" font-size="9" text-anchor="middle">tok 2</text>
|
|
<rect x="130" y="58" width="40" height="25" rx="4" fill="#27ae60" opacity="0.3" stroke="#27ae60" stroke-width="1"/>
|
|
<text x="150" y="76" fill="#333" font-size="9" text-anchor="middle">tok 3</text>
|
|
|
|
<!-- Gating network -->
|
|
<rect x="220" y="95" width="150" height="40" rx="6" fill="#f39c12" opacity="0.2" stroke="#f39c12" stroke-width="2"/>
|
|
<text x="295" y="112" fill="#f39c12" font-size="12" font-weight="bold" text-anchor="middle">Gating Network</text>
|
|
<text x="295" y="126" fill="#f39c12" font-size="9" text-anchor="middle">softmax router</text>
|
|
|
|
<!-- Arrows from tokens to gate -->
|
|
<line x1="60" y1="83" x2="220" y2="108" stroke="#555" stroke-width="1" marker-end="url(#moe-arrow)"/>
|
|
<line x1="105" y1="83" x2="230" y2="108" stroke="#555" stroke-width="1" marker-end="url(#moe-arrow)"/>
|
|
<line x1="150" y1="83" x2="240" y2="108" stroke="#555" stroke-width="1" marker-end="url(#moe-arrow)"/>
|
|
|
|
<!-- Experts -->
|
|
<rect x="80" y="175" width="90" height="50" rx="6" fill="#3498db" opacity="0.15" stroke="#3498db" stroke-width="1.5"/>
|
|
<text x="125" y="197" fill="#3498db" font-size="11" font-weight="bold" text-anchor="middle">Expert 1</text>
|
|
<text x="125" y="213" fill="#3498db" font-size="9" text-anchor="middle">FFN</text>
|
|
|
|
<rect x="200" y="175" width="90" height="50" rx="6" fill="#e74c3c" opacity="0.15" stroke="#e74c3c" stroke-width="1.5"/>
|
|
<text x="245" y="197" fill="#e74c3c" font-size="11" font-weight="bold" text-anchor="middle">Expert 2</text>
|
|
<text x="245" y="213" fill="#e74c3c" font-size="9" text-anchor="middle">FFN</text>
|
|
|
|
<rect x="320" y="175" width="90" height="50" rx="6" fill="#27ae60" opacity="0.15" stroke="#27ae60" stroke-width="1.5"/>
|
|
<text x="365" y="197" fill="#27ae60" font-size="11" font-weight="bold" text-anchor="middle">Expert 3</text>
|
|
<text x="365" y="213" fill="#27ae60" font-size="9" text-anchor="middle">FFN</text>
|
|
|
|
<rect x="440" y="175" width="90" height="50" rx="6" fill="#9b59b6" opacity="0.15" stroke="#9b59b6" stroke-width="1.5"/>
|
|
<text x="485" y="197" fill="#9b59b6" font-size="11" font-weight="bold" text-anchor="middle">Expert 4</text>
|
|
<text x="485" y="213" fill="#9b59b6" font-size="9" text-anchor="middle">FFN</text>
|
|
|
|
<!-- Routing arrows (sparse: each token goes to top-2) -->
|
|
<line x1="260" y1="135" x2="125" y2="175" stroke="#3498db" stroke-width="2" marker-end="url(#moe-arrow)"/>
|
|
<line x1="280" y1="135" x2="245" y2="175" stroke="#3498db" stroke-width="1.5" marker-end="url(#moe-arrow)"/>
|
|
|
|
<line x1="295" y1="135" x2="245" y2="175" stroke="#e74c3c" stroke-width="2" marker-end="url(#moe-arrow)"/>
|
|
<line x1="310" y1="135" x2="365" y2="175" stroke="#e74c3c" stroke-width="1.5" marker-end="url(#moe-arrow)"/>
|
|
|
|
<line x1="320" y1="135" x2="365" y2="175" stroke="#27ae60" stroke-width="2" marker-end="url(#moe-arrow)"/>
|
|
<line x1="330" y1="135" x2="485" y2="175" stroke="#27ae60" stroke-width="1.5" marker-end="url(#moe-arrow)"/>
|
|
|
|
<!-- Top-K annotation -->
|
|
<text x="380" y="152" fill="#f39c12" font-size="9" font-weight="bold">top-K=2</text>
|
|
|
|
<!-- Weighted sum output -->
|
|
<line x1="295" y1="225" x2="295" y2="255" stroke="#555" stroke-width="1.5" marker-end="url(#moe-arrow)"/>
|
|
<rect x="220" y="255" width="150" height="25" rx="6" fill="#f5f5f5" stroke="#333" stroke-width="1.5"/>
|
|
<text x="295" y="272" fill="#333" font-size="10" text-anchor="middle" font-weight="bold">Weighted sum of expert outputs</text>
|
|
|
|
<!-- Key insight box (right side) -->
|
|
<rect x="550" y="85" width="135" height="120" rx="6" fill="#f5f5f5" stroke="#ddd" stroke-width="1"/>
|
|
<text x="617" y="105" fill="#333" font-size="10" font-weight="bold" text-anchor="middle">Key Idea</text>
|
|
<text x="560" y="125" fill="#666" font-size="9">Total params: huge</text>
|
|
<text x="560" y="142" fill="#666" font-size="9">Active params: small</text>
|
|
<text x="560" y="162" fill="#27ae60" font-size="9" font-weight="bold">More capacity</text>
|
|
<text x="560" y="176" fill="#27ae60" font-size="9" font-weight="bold">without more FLOPs</text>
|
|
<text x="560" y="196" fill="#666" font-size="8">(only K of N experts</text>
|
|
<text x="560" y="208" fill="#666" font-size="8">run per token)</text>
|
|
</svg> |
