51 lines
3.7 KiB
XML
51 lines
3.7 KiB
XML
<svg width="700" height="300" xmlns="http://www.w3.org/2000/svg">
|
|
<text x="350" y="22" fill="#333" font-size="14" font-weight="bold" text-anchor="middle">Common Activation Functions</text>
|
|
|
|
<!-- === ReLU (top-left) === -->
|
|
<text x="130" y="48" fill="#3498db" font-size="12" font-weight="bold" text-anchor="middle">ReLU</text>
|
|
<!-- Axes -->
|
|
<line x1="50" y1="130" x2="210" y2="130" stroke="#999" stroke-width="0.8"/>
|
|
<line x1="130" y1="60" x2="130" y2="145" stroke="#999" stroke-width="0.8"/>
|
|
<!-- Curve: flat at 0 for x<0, linear for x>0 -->
|
|
<path d="M 50,130 L 130,130 L 210,60" fill="none" stroke="#3498db" stroke-width="2.2"/>
|
|
<text x="160" y="80" fill="#3498db" font-size="9">max(0, x)</text>
|
|
|
|
<!-- === Sigmoid (top-right) === -->
|
|
<text x="300" y="48" fill="#e74c3c" font-size="12" font-weight="bold" text-anchor="middle">Sigmoid</text>
|
|
<line x1="220" y1="130" x2="380" y2="130" stroke="#999" stroke-width="0.8"/>
|
|
<line x1="300" y1="60" x2="300" y2="145" stroke="#999" stroke-width="0.8"/>
|
|
<path d="M 220,128 C 240,128 260,127 270,122 C 280,115 290,100 300,95 C 310,90 320,78 330,72 C 340,67 360,63 380,62" fill="none" stroke="#e74c3c" stroke-width="2.2"/>
|
|
<text x="365" y="58" fill="#e74c3c" font-size="9">1</text>
|
|
<text x="365" y="133" fill="#e74c3c" font-size="9">0</text>
|
|
<line x1="220" y1="62" x2="380" y2="62" stroke="#e74c3c" stroke-width="0.5" stroke-dasharray="3,3" opacity="0.5"/>
|
|
|
|
<!-- === Tanh (bottom-left) === -->
|
|
<text x="130" y="175" fill="#27ae60" font-size="12" font-weight="bold" text-anchor="middle">Tanh</text>
|
|
<line x1="50" y1="240" x2="210" y2="240" stroke="#999" stroke-width="0.8"/>
|
|
<line x1="130" y1="185" x2="130" y2="280" stroke="#999" stroke-width="0.8"/>
|
|
<path d="M 50,275 C 70,274 85,272 95,265 C 105,255 115,245 130,240 C 145,235 155,225 165,215 C 175,208 190,206 210,205" fill="none" stroke="#27ae60" stroke-width="2.2"/>
|
|
<text x="45" y="279" fill="#27ae60" font-size="9">-1</text>
|
|
<text x="215" y="202" fill="#27ae60" font-size="9">1</text>
|
|
<line x1="50" y1="205" x2="210" y2="205" stroke="#27ae60" stroke-width="0.5" stroke-dasharray="3,3" opacity="0.5"/>
|
|
<line x1="50" y1="275" x2="210" y2="275" stroke="#27ae60" stroke-width="0.5" stroke-dasharray="3,3" opacity="0.5"/>
|
|
|
|
<!-- === GELU (bottom-right) === -->
|
|
<text x="300" y="175" fill="#9b59b6" font-size="12" font-weight="bold" text-anchor="middle">GELU</text>
|
|
<line x1="220" y1="240" x2="380" y2="240" stroke="#999" stroke-width="0.8"/>
|
|
<line x1="300" y1="185" x2="300" y2="270" stroke="#999" stroke-width="0.8"/>
|
|
<!-- GELU: smooth curve, slightly negative dip around x=-1 -->
|
|
<path d="M 220,243 C 240,245 255,246 270,244 C 280,243 290,241 300,240 C 310,237 320,228 340,215 C 355,205 370,195 380,185" fill="none" stroke="#9b59b6" stroke-width="2.2"/>
|
|
<text x="330" y="198" fill="#9b59b6" font-size="9">x·Φ(x)</text>
|
|
|
|
<!-- Summary boxes on the right -->
|
|
<rect x="420" y="55" width="260" height="105" rx="6" fill="#f5f5f5" stroke="#ddd" stroke-width="1"/>
|
|
<text x="550" y="75" fill="#333" font-size="11" text-anchor="middle" font-weight="bold">Properties</text>
|
|
<text x="435" y="95" fill="#3498db" font-size="10">ReLU: sparse, fast, can "die"</text>
|
|
<text x="435" y="112" fill="#e74c3c" font-size="10">Sigmoid: (0,1), vanishing gradients</text>
|
|
<text x="435" y="129" fill="#27ae60" font-size="10">Tanh: (-1,1), zero-centred</text>
|
|
<text x="435" y="146" fill="#9b59b6" font-size="10">GELU: smooth ReLU, used in GPT</text>
|
|
|
|
<!-- Bottom note -->
|
|
<rect x="140" y="280" width="420" height="18" rx="4" fill="#f5f5f5" stroke="#333" stroke-width="0.8"/>
|
|
<text x="350" y="293" fill="#333" font-size="10" text-anchor="middle">Nonlinearities are essential: without them, stacking layers collapses to one linear map</text>
|
|
</svg> |
