Deployed 2536c93 with MkDocs version: 1.6.1

images/bert_mlm.svg

@@ -0,0 +1,81 @@
+<svg width="700" height="250" xmlns="http://www.w3.org/2000/svg">
+  <defs>
+    <marker id="bert-arrow" markerWidth="7" markerHeight="5" refX="7" refY="2.5" orient="auto">
+      <polygon points="0 0, 7 2.5, 0 5" fill="#555"/>
+    </marker>
+  </defs>
+  <text x="350" y="22" fill="#333" font-size="14" font-weight="bold" text-anchor="middle">BERT: Masked Language Modelling</text>
+
+  <!-- Original sentence -->
+  <text x="50" y="55" fill="#666" font-size="10" font-weight="bold">Original:</text>
+  <g>
+    <rect x="120" y="40" width="50" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
+    <text x="145" y="58" fill="#333" font-size="11" text-anchor="middle">[CLS]</text>
+    <rect x="178" y="40" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
+    <text x="198" y="58" fill="#333" font-size="11" text-anchor="middle">the</text>
+    <rect x="226" y="40" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
+    <text x="246" y="58" fill="#333" font-size="11" text-anchor="middle">cat</text>
+    <rect x="274" y="40" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
+    <text x="294" y="58" fill="#333" font-size="11" text-anchor="middle">sat</text>
+    <rect x="322" y="40" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
+    <text x="342" y="58" fill="#333" font-size="11" text-anchor="middle">on</text>
+    <rect x="370" y="40" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
+    <text x="390" y="58" fill="#333" font-size="11" text-anchor="middle">the</text>
+    <rect x="418" y="40" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
+    <text x="438" y="58" fill="#333" font-size="11" text-anchor="middle">mat</text>
+  </g>
+
+  <!-- Arrow down -->
+  <text x="500" y="62" fill="#e74c3c" font-size="10" font-weight="bold">15% masked</text>
+  <line x1="350" y1="70" x2="350" y2="88" stroke="#e74c3c" stroke-width="1.5" marker-end="url(#bert-arrow)"/>
+
+  <!-- Masked input -->
+  <text x="50" y="110" fill="#666" font-size="10" font-weight="bold">Input:</text>
+  <g>
+    <rect x="120" y="95" width="50" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
+    <text x="145" y="113" fill="#333" font-size="11" text-anchor="middle">[CLS]</text>
+    <rect x="178" y="95" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
+    <text x="198" y="113" fill="#333" font-size="11" text-anchor="middle">the</text>
+    <!-- MASKED -->
+    <rect x="226" y="95" width="55" height="26" rx="4" fill="#e74c3c" opacity="0.2" stroke="#e74c3c" stroke-width="2"/>
+    <text x="253" y="113" fill="#e74c3c" font-size="10" text-anchor="middle" font-weight="bold">[MASK]</text>
+    <rect x="289" y="95" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
+    <text x="309" y="113" fill="#333" font-size="11" text-anchor="middle">sat</text>
+    <rect x="337" y="95" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
+    <text x="357" y="113" fill="#333" font-size="11" text-anchor="middle">on</text>
+    <rect x="385" y="95" width="40" height="26" rx="4" fill="#95a5a6" opacity="0.15" stroke="#95a5a6" stroke-width="1"/>
+    <text x="405" y="113" fill="#333" font-size="11" text-anchor="middle">the</text>
+    <!-- MASKED -->
+    <rect x="433" y="95" width="55" height="26" rx="4" fill="#e74c3c" opacity="0.2" stroke="#e74c3c" stroke-width="2"/>
+    <text x="460" y="113" fill="#e74c3c" font-size="10" text-anchor="middle" font-weight="bold">[MASK]</text>
+  </g>
+
+  <!-- BERT Transformer -->
+  <rect x="140" y="135" width="370" height="40" rx="8" fill="#3498db" opacity="0.12" stroke="#3498db" stroke-width="2"/>
+  <text x="325" y="160" fill="#3498db" font-size="12" text-anchor="middle" font-weight="bold">BERT Transformer Encoder (bidirectional attention)</text>
+
+  <!-- Arrows up and down -->
+  <line x1="253" y1="121" x2="253" y2="132" stroke="#555" stroke-width="1" marker-end="url(#bert-arrow)"/>
+  <line x1="460" y1="121" x2="460" y2="132" stroke="#555" stroke-width="1" marker-end="url(#bert-arrow)"/>
+
+  <!-- Prediction outputs -->
+  <line x1="253" y1="175" x2="253" y2="195" stroke="#27ae60" stroke-width="1.5" marker-end="url(#bert-arrow)"/>
+  <line x1="460" y1="175" x2="460" y2="195" stroke="#27ae60" stroke-width="1.5" marker-end="url(#bert-arrow)"/>
+
+  <rect x="226" y="198" width="55" height="26" rx="4" fill="#27ae60" opacity="0.15" stroke="#27ae60" stroke-width="2"/>
+  <text x="253" y="216" fill="#27ae60" font-size="11" text-anchor="middle" font-weight="bold">cat</text>
+
+  <rect x="433" y="198" width="55" height="26" rx="4" fill="#27ae60" opacity="0.15" stroke="#27ae60" stroke-width="2"/>
+  <text x="460" y="216" fill="#27ae60" font-size="11" text-anchor="middle" font-weight="bold">mat</text>
+
+  <text x="350" y="216" fill="#27ae60" font-size="10" text-anchor="middle">predict masked tokens</text>
+
+  <!-- Right side: masking strategy breakdown -->
+  <rect x="530" y="88" width="155" height="80" rx="6" fill="#f5f5f5" stroke="#ccc" stroke-width="1"/>
+  <text x="607" y="105" fill="#333" font-size="10" text-anchor="middle" font-weight="bold">Of 15% selected:</text>
+  <text x="607" y="122" fill="#e74c3c" font-size="10" text-anchor="middle">80% → [MASK]</text>
+  <text x="607" y="138" fill="#f39c12" font-size="10" text-anchor="middle">10% → random word</text>
+  <text x="607" y="154" fill="#3498db" font-size="10" text-anchor="middle">10% → unchanged</text>
+
+  <text x="350" y="245" fill="#666" font-size="10" text-anchor="middle">Every token attends to every other token (bidirectional), enabling rich contextual representations.</text>
+</svg>