81 lines
5.8 KiB
XML
81 lines
5.8 KiB
XML
<svg width="700" height="250" xmlns="http://www.w3.org/2000/svg">
|
|
<defs>
|
|
<marker id="bert-arrow" markerWidth="7" markerHeight="5" refX="7" refY="2.5" orient="auto">
|
|
<polygon points="0 0, 7 2.5, 0 5" fill="#555"/>
|
|
</marker>
|
|
</defs>
|
|
<text x="350" y="22" fill="#333" font-size="14" font-weight="bold" text-anchor="middle">BERT: Masked Language Modelling</text>
|
|
|
|
<!-- Original sentence -->
|
|
<text x="50" y="55" fill="#666" font-size="10" font-weight="bold">Original:</text>
|
|
<g>
|
|
<rect x="120" y="40" width="50" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
|
|
<text x="145" y="58" fill="#333" font-size="11" text-anchor="middle">[CLS]</text>
|
|
<rect x="178" y="40" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
|
|
<text x="198" y="58" fill="#333" font-size="11" text-anchor="middle">the</text>
|
|
<rect x="226" y="40" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
|
|
<text x="246" y="58" fill="#333" font-size="11" text-anchor="middle">cat</text>
|
|
<rect x="274" y="40" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
|
|
<text x="294" y="58" fill="#333" font-size="11" text-anchor="middle">sat</text>
|
|
<rect x="322" y="40" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
|
|
<text x="342" y="58" fill="#333" font-size="11" text-anchor="middle">on</text>
|
|
<rect x="370" y="40" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
|
|
<text x="390" y="58" fill="#333" font-size="11" text-anchor="middle">the</text>
|
|
<rect x="418" y="40" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
|
|
<text x="438" y="58" fill="#333" font-size="11" text-anchor="middle">mat</text>
|
|
</g>
|
|
|
|
<!-- Arrow down -->
|
|
<text x="500" y="62" fill="#e74c3c" font-size="10" font-weight="bold">15% masked</text>
|
|
<line x1="350" y1="70" x2="350" y2="88" stroke="#e74c3c" stroke-width="1.5" marker-end="url(#bert-arrow)"/>
|
|
|
|
<!-- Masked input -->
|
|
<text x="50" y="110" fill="#666" font-size="10" font-weight="bold">Input:</text>
|
|
<g>
|
|
<rect x="120" y="95" width="50" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
|
|
<text x="145" y="113" fill="#333" font-size="11" text-anchor="middle">[CLS]</text>
|
|
<rect x="178" y="95" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
|
|
<text x="198" y="113" fill="#333" font-size="11" text-anchor="middle">the</text>
|
|
<!-- MASKED -->
|
|
<rect x="226" y="95" width="55" height="26" rx="4" fill="#e74c3c" opacity="0.2" stroke="#e74c3c" stroke-width="2"/>
|
|
<text x="253" y="113" fill="#e74c3c" font-size="10" text-anchor="middle" font-weight="bold">[MASK]</text>
|
|
<rect x="289" y="95" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
|
|
<text x="309" y="113" fill="#333" font-size="11" text-anchor="middle">sat</text>
|
|
<rect x="337" y="95" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
|
|
<text x="357" y="113" fill="#333" font-size="11" text-anchor="middle">on</text>
|
|
<rect x="385" y="95" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
|
|
<text x="405" y="113" fill="#333" font-size="11" text-anchor="middle">the</text>
|
|
<!-- MASKED -->
|
|
<rect x="433" y="95" width="55" height="26" rx="4" fill="#e74c3c" opacity="0.2" stroke="#e74c3c" stroke-width="2"/>
|
|
<text x="460" y="113" fill="#e74c3c" font-size="10" text-anchor="middle" font-weight="bold">[MASK]</text>
|
|
</g>
|
|
|
|
<!-- BERT Transformer -->
|
|
<rect x="140" y="135" width="370" height="40" rx="8" fill="#3498db" opacity="0.12" stroke="#3498db" stroke-width="2"/>
|
|
<text x="325" y="160" fill="#3498db" font-size="12" text-anchor="middle" font-weight="bold">BERT Transformer Encoder (bidirectional attention)</text>
|
|
|
|
<!-- Arrows up and down -->
|
|
<line x1="253" y1="121" x2="253" y2="132" stroke="#555" stroke-width="1" marker-end="url(#bert-arrow)"/>
|
|
<line x1="460" y1="121" x2="460" y2="132" stroke="#555" stroke-width="1" marker-end="url(#bert-arrow)"/>
|
|
|
|
<!-- Prediction outputs -->
|
|
<line x1="253" y1="175" x2="253" y2="195" stroke="#27ae60" stroke-width="1.5" marker-end="url(#bert-arrow)"/>
|
|
<line x1="460" y1="175" x2="460" y2="195" stroke="#27ae60" stroke-width="1.5" marker-end="url(#bert-arrow)"/>
|
|
|
|
<rect x="226" y="198" width="55" height="26" rx="4" fill="#27ae60" opacity="0.15" stroke="#27ae60" stroke-width="2"/>
|
|
<text x="253" y="216" fill="#27ae60" font-size="11" text-anchor="middle" font-weight="bold">cat</text>
|
|
|
|
<rect x="433" y="198" width="55" height="26" rx="4" fill="#27ae60" opacity="0.15" stroke="#27ae60" stroke-width="2"/>
|
|
<text x="460" y="216" fill="#27ae60" font-size="11" text-anchor="middle" font-weight="bold">mat</text>
|
|
|
|
<text x="350" y="216" fill="#27ae60" font-size="10" text-anchor="middle">predict masked tokens</text>
|
|
|
|
<!-- Right side: masking strategy breakdown -->
|
|
<rect x="530" y="88" width="155" height="80" rx="6" fill="#f5f5f5" stroke="#ccc" stroke-width="1"/>
|
|
<text x="607" y="105" fill="#333" font-size="10" text-anchor="middle" font-weight="bold">Of 15% selected:</text>
|
|
<text x="607" y="122" fill="#e74c3c" font-size="10" text-anchor="middle">80% → [MASK]</text>
|
|
<text x="607" y="138" fill="#f39c12" font-size="10" text-anchor="middle">10% → random word</text>
|
|
<text x="607" y="154" fill="#3498db" font-size="10" text-anchor="middle">10% → unchanged</text>
|
|
|
|
<text x="350" y="245" fill="#666" font-size="10" text-anchor="middle">Every token attends to every other token (bidirectional), enabling rich contextual representations.</text>
|
|
</svg> |
