5953 lines
162 KiB
HTML
5953 lines
162 KiB
HTML
|
||
<!doctype html>
|
||
<html lang="zh" class="no-js">
|
||
<head>
|
||
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||
|
||
<meta name="description" content="一本开源的直觉优先教科书，从零开始覆盖数学、计算机科学和人工智能（中文翻译版）。">
|
||
|
||
|
||
<meta name="author" content="Henry Ndubuaku (flykhan 译)">
|
||
|
||
|
||
<link rel="canonical" href="https://flykhan.github.io/maths-cs-ai-compendium-zh/chapter%2010%3A%20multimodal%20learning/02.%20vision%20language%20models/">
|
||
|
||
|
||
<link rel="prev" href="../01.%20multimodal%20representations/">
|
||
|
||
|
||
<link rel="next" href="../03.%20image%20and%20video%20tokenisation/">
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="icon" href="../../assets/images/favicon.png">
|
||
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
|
||
|
||
|
||
|
||
<title>视觉语言模型 - 数学、计算机科学与 AI 百科全书</title>
|
||
|
||
|
||
|
||
<link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">
|
||
|
||
|
||
<link rel="stylesheet" href="../../assets/stylesheets/palette.ab4e12ef.min.css">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
|
||
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
|
||
|
||
|
||
|
||
<script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
|
||
|
||
|
||
|
||
|
||
|
||
</head>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="slate" data-md-color-accent="indigo">
|
||
|
||
|
||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||
<label class="md-overlay" for="__drawer"></label>
|
||
<div data-md-component="skip">
|
||
|
||
|
||
<a href="#_1" class="md-skip">
|
||
跳转至
|
||
</a>
|
||
|
||
</div>
|
||
<div data-md-component="announce">
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<header class="md-header" data-md-component="header">
|
||
<nav class="md-header__inner md-grid" aria-label="页眉">
|
||
<a href="../.." title="数学、计算机科学与 AI 百科全书" class="md-header__button md-logo" aria-label="数学、计算机科学与 AI 百科全书" data-md-component="logo">
|
||
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
|
||
|
||
</a>
|
||
<label class="md-header__button md-icon" for="__drawer">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
|
||
</label>
|
||
<div class="md-header__title" data-md-component="header-title">
|
||
<div class="md-header__ellipsis">
|
||
<div class="md-header__topic">
|
||
<span class="md-ellipsis">
|
||
数学、计算机科学与 AI 百科全书
|
||
</span>
|
||
</div>
|
||
<div class="md-header__topic" data-md-component="header-topic">
|
||
<span class="md-ellipsis">
|
||
|
||
视觉语言模型
|
||
|
||
</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<form class="md-header__option" data-md-component="palette">
|
||
|
||
|
||
|
||
|
||
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="slate" data-md-color-accent="indigo" aria-label="切换到深色模式" type="radio" name="__palette" id="__palette_0">
|
||
|
||
<label class="md-header__button md-icon" title="切换到深色模式" for="__palette_1" hidden>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
|
||
</label>
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="slate" data-md-color-accent="indigo" aria-label="切换到浅色模式" type="radio" name="__palette" id="__palette_1">
|
||
|
||
<label class="md-header__button md-icon" title="切换到浅色模式" for="__palette_0" hidden>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12s-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
|
||
</label>
|
||
|
||
|
||
</form>
|
||
|
||
|
||
|
||
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-header__button md-icon" for="__search">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
|
||
</label>
|
||
<div class="md-search" data-md-component="search" role="dialog">
|
||
<label class="md-search__overlay" for="__search"></label>
|
||
<div class="md-search__inner" role="search">
|
||
<form class="md-search__form" name="search">
|
||
<input type="text" class="md-search__input" name="query" aria-label="搜索" placeholder="搜索" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
|
||
<label class="md-search__icon md-icon" for="__search">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
|
||
</label>
|
||
<nav class="md-search__options" aria-label="查找">
|
||
|
||
<button type="reset" class="md-search__icon md-icon" title="清空当前内容" aria-label="清空当前内容" tabindex="-1">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
|
||
</button>
|
||
</nav>
|
||
|
||
<div class="md-search__suggest" data-md-component="search-suggest"></div>
|
||
|
||
</form>
|
||
<div class="md-search__output">
|
||
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
|
||
<div class="md-search-result" data-md-component="search-result">
|
||
<div class="md-search-result__meta">
|
||
正在初始化搜索引擎
|
||
</div>
|
||
<ol class="md-search-result__list" role="presentation"></ol>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-header__source">
|
||
<a href="https://github.com/flykhan/maths-cs-ai-compendium-zh" title="前往仓库" class="md-source" data-md-component="source">
|
||
<div class="md-source__icon md-icon">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
|
||
</div>
|
||
<div class="md-source__repository">
|
||
flykhan/maths-cs-ai-compendium-zh
|
||
</div>
|
||
</a>
|
||
</div>
|
||
|
||
</nav>
|
||
|
||
</header>
|
||
|
||
<div class="md-container" data-md-component="container">
|
||
|
||
|
||
|
||
|
||
|
||
<nav class="md-tabs" aria-label="标签" data-md-component="tabs">
|
||
<div class="md-grid">
|
||
<ul class="md-tabs__list">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../.." class="md-tabs__link">
|
||
|
||
|
||
|
||
|
||
|
||
首页
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2001%3A%20vectors/01.%20vector%20spaces/" class="md-tabs__link">
|
||
|
||
|
||
|
||
向量
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2002%3A%20matrices/01.%20matrix%20properties/" class="md-tabs__link">
|
||
|
||
|
||
|
||
矩阵
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2003%3A%20calculus/01.%20differential%20calculus/" class="md-tabs__link">
|
||
|
||
|
||
|
||
微积分
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2004%3A%20statistics/01.%20fundamentals/" class="md-tabs__link">
|
||
|
||
|
||
|
||
统计学
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2005%3A%20probability/01.%20counting/" class="md-tabs__link">
|
||
|
||
|
||
|
||
概率论
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2006%3A%20machine%20learning/01.%20classical%20machine%20learning/" class="md-tabs__link">
|
||
|
||
|
||
|
||
机器学习
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2007%3A%20computational%20linguistics/01.%20linguistic%20foundations/" class="md-tabs__link">
|
||
|
||
|
||
|
||
计算语言学
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2008%3A%20computer%20vision/01.%20image%20fundamentals/" class="md-tabs__link">
|
||
|
||
|
||
|
||
计算机视觉
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2009%3A%20audio%20and%20speech/01.%20digital%20signal%20processing/" class="md-tabs__link">
|
||
|
||
|
||
|
||
音频与语音
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item md-tabs__item--active">
|
||
<a href="../01.%20multimodal%20representations/" class="md-tabs__link">
|
||
|
||
|
||
|
||
多模态学习
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2011%3A%20autonomous%20systems/01.%20perception/" class="md-tabs__link">
|
||
|
||
|
||
|
||
自主系统
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2012%3A%20graph%20neural%20networks/01.%20geometric%20deep%20learning/" class="md-tabs__link">
|
||
|
||
|
||
|
||
图神经网络
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2013%3A%20computing%20and%20OS/01.%20discrete%20maths/" class="md-tabs__link">
|
||
|
||
|
||
|
||
计算机与操作系统
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2014%3A%20data%20structures%20and%20algorithms/00.%20foundations/" class="md-tabs__link">
|
||
|
||
|
||
|
||
数据结构与算法
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2015%3A%20production%20software%20engineering/01.%20linux%20and%20CMD/" class="md-tabs__link">
|
||
|
||
|
||
|
||
生产级软件工程
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2016%3A%20SIMD%20and%20GPU%20programming/00.%20why%20C%2B%2B%20and%20how%20ML%20frameworks%20work/" class="md-tabs__link">
|
||
|
||
|
||
|
||
SIMD 与 GPU 编程
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2017%3A%20AI%20inference/01.%20quantisation/" class="md-tabs__link">
|
||
|
||
|
||
|
||
AI 推理
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2018%3A%20ML%20systems%20design/01.%20systems%20design%20fundamentals/" class="md-tabs__link">
|
||
|
||
|
||
|
||
ML 系统设计
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2019%3A%20applied%20AI/01.%20AI%20for%20finance/" class="md-tabs__link">
|
||
|
||
|
||
|
||
应用 AI
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../../chapter%2020%3A%20bleeding%20edge%20AI/01.%20quantum%20machine%20learning/" class="md-tabs__link">
|
||
|
||
|
||
|
||
前沿 AI
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</div>
|
||
</nav>
|
||
|
||
|
||
|
||
<main class="md-main" data-md-component="main">
|
||
<div class="md-main__inner md-grid">
|
||
|
||
|
||
|
||
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
|
||
<div class="md-sidebar__scrollwrap">
|
||
<div class="md-sidebar__inner">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="导航栏" data-md-level="0">
|
||
<label class="md-nav__title" for="__drawer">
|
||
<a href="../.." title="数学、计算机科学与 AI 百科全书" class="md-nav__button md-logo" aria-label="数学、计算机科学与 AI 百科全书" data-md-component="logo">
|
||
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
|
||
|
||
</a>
|
||
数学、计算机科学与 AI 百科全书
|
||
</label>
|
||
|
||
<div class="md-nav__source">
|
||
<a href="https://github.com/flykhan/maths-cs-ai-compendium-zh" title="前往仓库" class="md-source" data-md-component="source">
|
||
<div class="md-source__icon md-icon">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
|
||
</div>
|
||
<div class="md-source__repository">
|
||
flykhan/maths-cs-ai-compendium-zh
|
||
</div>
|
||
</a>
|
||
</div>
|
||
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../.." class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
首页
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
向量
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
向量
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2001%3A%20vectors/01.%20vector%20spaces/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
向量空间
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2001%3A%20vectors/02.%20vector%20properties/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
向量性质
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2001%3A%20vectors/03.%20norms%20and%20metrics/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
范数与度量
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2001%3A%20vectors/04.%20products/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
向量积
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2001%3A%20vectors/05.%20basis%20and%20duality/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
基与对偶性
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_3" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
矩阵
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_3">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
矩阵
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2002%3A%20matrices/01.%20matrix%20properties/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
矩阵性质
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2002%3A%20matrices/02.%20matrix%20types/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
矩阵类型
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2002%3A%20matrices/03.%20operations/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
矩阵运算
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2002%3A%20matrices/04.%20linear%20transformations/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
线性变换
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2002%3A%20matrices/05.%20decompositions/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
矩阵分解
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_4" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
微积分
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
微积分
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2003%3A%20calculus/01.%20differential%20calculus/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
微分
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2003%3A%20calculus/02.%20integral%20calculus/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
积分
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2003%3A%20calculus/03.%20multivariate%20calculus/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
多元微积分
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2003%3A%20calculus/04.%20function%20approximation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
函数逼近
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2003%3A%20calculus/05.%20optimisation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
优化
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_5" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
统计学
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_5">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
统计学
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2004%3A%20statistics/01.%20fundamentals/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
基础
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2004%3A%20statistics/02.%20measures/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
统计量
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2004%3A%20statistics/03.%20sampling/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
抽样
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2004%3A%20statistics/04.%20hypothesis%20testing/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
假设检验
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2004%3A%20statistics/05.%20inference/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
推断
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_6" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
概率论
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_6">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
概率论
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2005%3A%20probability/01.%20counting/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
计数
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2005%3A%20probability/02.%20probability%20concepts/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
概率概念
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2005%3A%20probability/03.%20distributions/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
分布
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2005%3A%20probability/04.%20bayesian/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
贝叶斯
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2005%3A%20probability/05.%20information%20theory/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
信息论
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_7" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_7" id="__nav_7_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
机器学习
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_7">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
机器学习
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2006%3A%20machine%20learning/01.%20classical%20machine%20learning/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
经典机器学习
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2006%3A%20machine%20learning/02.%20gradient%20machine%20learning/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
梯度机器学习
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2006%3A%20machine%20learning/03.%20deep%20learning/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
深度学习
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2006%3A%20machine%20learning/04.%20reinforcement%20learning/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
强化学习
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2006%3A%20machine%20learning/05.%20distributed%20deep%20learning/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
分布式深度学习
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_8" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_8" id="__nav_8_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
计算语言学
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_8">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
计算语言学
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2007%3A%20computational%20linguistics/01.%20linguistic%20foundations/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
语言学基础
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2007%3A%20computational%20linguistics/02.%20text%20processing%20and%20classic%20NLP/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
文本处理与经典 NLP
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2007%3A%20computational%20linguistics/03.%20embeddings%20and%20sequence%20models/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
嵌入与序列模型
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2007%3A%20computational%20linguistics/04.%20transformers%20and%20language%20models/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Transformer 与语言模型
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2007%3A%20computational%20linguistics/05.%20advanced%20text%20generation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
高级文本生成
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_9" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_9" id="__nav_9_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
计算机视觉
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_9">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
计算机视觉
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2008%3A%20computer%20vision/01.%20image%20fundamentals/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
图像基础
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2008%3A%20computer%20vision/02.%20convolutional%20networks/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
卷积网络
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2008%3A%20computer%20vision/03.%20object%20detection%20and%20segmentation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
目标检测与分割
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2008%3A%20computer%20vision/04.%20vision%20transformers%20and%20generation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
ViT 与生成模型
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2008%3A%20computer%20vision/05.%20video%20and%203D%20vision/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
视频与 3D 视觉
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_10" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_10" id="__nav_10_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
音频与语音
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_10">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
音频与语音
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2009%3A%20audio%20and%20speech/01.%20digital%20signal%20processing/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
数字信号处理
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2009%3A%20audio%20and%20speech/02.%20automatic%20speech%20recognition/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
自动语音识别
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2009%3A%20audio%20and%20speech/03.%20text%20to%20speech%20and%20voice/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
语音合成
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2009%3A%20audio%20and%20speech/04.%20speaker%20and%20audio%20analysis/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
说话人与音频分析
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2009%3A%20audio%20and%20speech/05.%20source%20separation%20and%20noise/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
源分离与降噪
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" checked>
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_11" id="__nav_11_label" tabindex="">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
多模态学习
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="true">
|
||
<label class="md-nav__title" for="__nav_11">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
多模态学习
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../01.%20multimodal%20representations/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
多模态表征
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--active">
|
||
|
||
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
视觉语言模型
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<a href="./" class="md-nav__link md-nav__link--active">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
视觉语言模型
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
|
||
<nav class="md-nav md-nav--secondary" aria-label="目录">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__title" for="__toc">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
目录
|
||
</label>
|
||
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_2" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
视觉问答
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_3" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
图像描述
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_4" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
架构模式
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="架构模式">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_5" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
双编码器
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_6" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
融合编码器
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#-" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
编码器-解码器
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#flamingo" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Flamingo：少样本多模态学习
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#llava" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
LLaVA 与视觉指令微调
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_7" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
扩展视觉语言模型
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="扩展视觉语言模型">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#pali" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
PaLI
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#qwen-vl" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Qwen-VL
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#internvl" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
InternVL
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_8" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
定位与指代
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#ocr" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
免 OCR 文档理解
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#token" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
视觉 Token 流水线
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_9" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
训练目标
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#colab-notebook" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
编程练习（使用 CoLab 或 notebook）
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../03.%20image%20and%20video%20tokenisation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
图像与视频 Token 化
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../04.%20cross-modal%20generation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
跨模态生成
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../05.%20unified%20multimodal%20architectures/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
统一多模态架构
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_12" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_12" id="__nav_12_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
自主系统
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_12_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_12">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
自主系统
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2011%3A%20autonomous%20systems/01.%20perception/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
感知
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2011%3A%20autonomous%20systems/02.%20robot%20learning/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
机器人学习
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2011%3A%20autonomous%20systems/03.%20vision-language-action%20models/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
视觉-语言-动作模型
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2011%3A%20autonomous%20systems/04.%20self-driving/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
自动驾驶
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2011%3A%20autonomous%20systems/05.%20space%20and%20extreme%20robotics/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
太空与极端机器人
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_13" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_13" id="__nav_13_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
图神经网络
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_13_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_13">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
图神经网络
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2012%3A%20graph%20neural%20networks/01.%20geometric%20deep%20learning/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
几何深度学习
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2012%3A%20graph%20neural%20networks/02.%20graph%20theory/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
图论
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2012%3A%20graph%20neural%20networks/03.%20graph%20neural%20networks/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
图神经网络
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2012%3A%20graph%20neural%20networks/04.%20graph%20attention%20networks/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
图注意力网络
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2012%3A%20graph%20neural%20networks/05.%203d%20graph%20networks/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
3D 图网络
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_14" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_14" id="__nav_14_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
计算机与操作系统
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_14_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_14">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
计算机与操作系统
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2013%3A%20computing%20and%20OS/01.%20discrete%20maths/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
离散数学
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2013%3A%20computing%20and%20OS/02.%20computer%20architecture/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
计算机体系结构
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2013%3A%20computing%20and%20OS/03.%20operating%20systems/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
操作系统
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2013%3A%20computing%20and%20OS/04.%20concurrency%20and%20parallelism/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
并发与并行
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2013%3A%20computing%20and%20OS/05.%20programming%20languages/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
编程语言
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_15" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_15" id="__nav_15_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
数据结构与算法
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_15_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_15">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
数据结构与算法
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2014%3A%20data%20structures%20and%20algorithms/00.%20foundations/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
基础
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2014%3A%20data%20structures%20and%20algorithms/01.%20arrays%20and%20hashing/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
数组与哈希
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2014%3A%20data%20structures%20and%20algorithms/02.%20linked%20lists%2C%20stacks%2C%20and%20queues/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
链表、栈与队列
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2014%3A%20data%20structures%20and%20algorithms/03.%20trees/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
树
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2014%3A%20data%20structures%20and%20algorithms/04.%20graphs/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
图
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2014%3A%20data%20structures%20and%20algorithms/05.%20sorting%20and%20search/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
排序与搜索
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_16" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_16" id="__nav_16_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
生产级软件工程
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_16_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_16">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
生产级软件工程
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2015%3A%20production%20software%20engineering/01.%20linux%20and%20CMD/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Linux 与命令行
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2015%3A%20production%20software%20engineering/02.%20git%20and%20repository%20management/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Git 与仓库管理
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2015%3A%20production%20software%20engineering/03.%20codebase%20design/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
代码设计
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2015%3A%20production%20software%20engineering/04.%20testing%20and%20quality%20assurance/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
测试与质量保障
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2015%3A%20production%20software%20engineering/05.%20deployment%20and%20devops/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
部署与 DevOps
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_17" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_17" id="__nav_17_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
SIMD 与 GPU 编程
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_17_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_17">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
SIMD 与 GPU 编程
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2016%3A%20SIMD%20and%20GPU%20programming/00.%20why%20C%2B%2B%20and%20how%20ML%20frameworks%20work/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
为什么是 C++ 及 ML 框架原理
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2016%3A%20SIMD%20and%20GPU%20programming/01.%20hardware%20fundamentals/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
硬件基础
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2016%3A%20SIMD%20and%20GPU%20programming/02.%20ARM%20and%20NEON/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
ARM 与 NEON
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2016%3A%20SIMD%20and%20GPU%20programming/03.%20x86%20and%20AVX/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
x86 与 AVX
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2016%3A%20SIMD%20and%20GPU%20programming/04.%20GPU%20architecture%20and%20CUDA/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
GPU 架构与 CUDA
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2016%3A%20SIMD%20and%20GPU%20programming/05.%20triton%2C%20TPUs%20and%20pallax/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Triton、TPU 与 Pallas
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2016%3A%20SIMD%20and%20GPU%20programming/06.%20RISC-V%20and%20embedded%20systems/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
RISC-V 与嵌入式系统
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2016%3A%20SIMD%20and%20GPU%20programming/07.%20vulkan%20compute%20and%20cross-platform%20GPU/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Vulkan Compute 与跨平台 GPU
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_18" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_18" id="__nav_18_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
AI 推理
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_18_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_18">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
AI 推理
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2017%3A%20AI%20inference/01.%20quantisation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
量化
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2017%3A%20AI%20inference/02.%20efficient%20architectures/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
高效架构
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2017%3A%20AI%20inference/03.%20serving%20and%20batching/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
服务与批处理
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2017%3A%20AI%20inference/04.%20edge%20inference/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
边缘推理
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2017%3A%20AI%20inference/05.%20scaling%20and%20deployment/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
扩缩与部署
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_19" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_19" id="__nav_19_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
ML 系统设计
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_19_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_19">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
ML 系统设计
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2018%3A%20ML%20systems%20design/01.%20systems%20design%20fundamentals/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
系统设计基础
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2018%3A%20ML%20systems%20design/02.%20cloud%20computing/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
云计算
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2018%3A%20ML%20systems%20design/03.%20large%20scale%20infrastructure/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
大规模基础设施
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2018%3A%20ML%20systems%20design/04.%20ML%20systems%20design/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
ML 系统设计
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2018%3A%20ML%20systems%20design/05.%20ML%20design%20examples/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
ML 设计案例
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_20" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_20" id="__nav_20_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
应用 AI
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_20_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_20">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
应用 AI
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2019%3A%20applied%20AI/01.%20AI%20for%20finance/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
AI 金融
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2019%3A%20applied%20AI/02.%20protein%20design/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
蛋白质设计
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2019%3A%20applied%20AI/03.%20drug%20discovery/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
药物发现
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2019%3A%20applied%20AI/04.%20agentic%20systems/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
智能体系统
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2019%3A%20applied%20AI/05.%20healthcare/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
医疗健康
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_21" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_21" id="__nav_21_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
前沿 AI
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_21_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_21">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
前沿 AI
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2020%3A%20bleeding%20edge%20AI/01.%20quantum%20machine%20learning/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
量子机器学习
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2020%3A%20bleeding%20edge%20AI/02.%20neuromorphic%20computing/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
神经形态计算
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2020%3A%20bleeding%20edge%20AI/03.%20datacentres%20in%20space/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
太空数据中心
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2020%3A%20bleeding%20edge%20AI/04.%20decentralised%20AI/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
去中心化 AI
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../chapter%2020%3A%20bleeding%20edge%20AI/05.%20brain%20machine%20interfaces/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
脑机接口
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
|
||
<div class="md-sidebar__scrollwrap">
|
||
<div class="md-sidebar__inner">
|
||
|
||
|
||
<nav class="md-nav md-nav--secondary" aria-label="目录">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__title" for="__toc">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
目录
|
||
</label>
|
||
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_2" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
视觉问答
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_3" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
图像描述
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_4" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
架构模式
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="架构模式">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_5" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
双编码器
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_6" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
融合编码器
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#-" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
编码器-解码器
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#flamingo" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Flamingo：少样本多模态学习
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#llava" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
LLaVA 与视觉指令微调
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_7" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
扩展视觉语言模型
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="扩展视觉语言模型">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#pali" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
PaLI
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#qwen-vl" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Qwen-VL
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#internvl" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
InternVL
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_8" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
定位与指代
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#ocr" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
免 OCR 文档理解
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#token" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
视觉 Token 流水线
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#_9" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
训练目标
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#colab-notebook" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
编程练习（使用 CoLab 或 notebook）
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-content" data-md-component="content">
|
||
|
||
<article class="md-content__inner md-typeset">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<h1 id="_1">视觉语言模型<a class="headerlink" href="#_1" title="Permanent link">¶</a></h1>
|
||
<p><em>视觉语言模型共同理解图像和文本，实现视觉问答、图像描述和视觉推理。本文件涵盖 VQA、图像描述、视觉定位，以及 VisualBERT、BLIP、LLaVA、Flamingo、PaLI 和 Qwen-VL 等将视觉编码器与大型语言模型融合的架构。</em></p>
|
||
<ul>
|
||
<li>
|
||
<p>想象一位博物馆导览员，他能看着一幅画并清晰描述画中的一切：有哪些物体、讲述了什么故事、传达了怎样的情感，还能回答参观者的任何问题。<strong>视觉语言模型（VLM）</strong> 就是计算领域的等价物——一个能同时理解图像和文本的系统，能够描述视觉场景、回答相关问题、执行视觉指令，甚至根据自然语言查询在图像中定位特定物体。</p>
|
||
</li>
|
||
<li>
|
||
<p>VLM 位于你在第 8 章学到的视觉编码器和第 7 章的语言模型的交汇点。核心工程挑战在于桥接两个截然不同的表征世界：视觉骨干网络产生的空间化、连续的 feature map，与语言模型产生的序列化、离散的 token 嵌入。本文件中的每一种架构，本质上都是对同一个问题的不同回答：如何融合视觉和语言？</p>
|
||
</li>
|
||
</ul>
|
||
<p><img alt="VLM 高层次分类：双编码器、融合编码器和编码器-解码器家族及其输入与输出" src="../../images/vlm_taxonomy.svg" /></p>
|
||
<h2 id="_2">视觉问答<a class="headerlink" href="#_2" title="Permanent link">¶</a></h2>
|
||
<ul>
|
||
<li>
|
||
<p>想象有人向你展示一张照片并问："公园里有几只狗？"你毫不费力地解析图像、定位狗、数出数量并给出答案。<strong>视觉问答（VQA）</strong> 将这一过程形式化：给定一张图像 <span class="arithmatex">\(I\)</span> 和一个自然语言问题 <span class="arithmatex">\(q\)</span>，预测答案 <span class="arithmatex">\(a\)</span>。</p>
|
||
</li>
|
||
<li>
|
||
<p>该任务可以有多种定义方式。最常见的方式将 VQA 视为<strong>开放式分类</strong>：模型从最常见的答案构成的固定词汇表中选择（例如 VQA v2 中排名前 3,129 的答案）。另一种方式是<strong>生成式回答</strong>，模型生成自由形式的文本字符串——这是现代 VLM 采用的方法。</p>
|
||
</li>
|
||
<li>
|
||
<p>形式上，你需要学习一个最大化正确答案似然的函数 <span class="arithmatex">\(f(I, q) \to a\)</span>。在分类设置中，这变为：</p>
|
||
</li>
|
||
</ul>
|
||
<div class="arithmatex">\[p(a \mid I, q) = \text{softmax}(W \cdot g(v, h))\]</div>
|
||
<ul>
|
||
<li>
|
||
<p>其中 <span class="arithmatex">\(v\)</span> 是视觉特征向量（来自 CNN 或 ViT），<span class="arithmatex">\(h\)</span> 是问题编码（来自 LSTM 或 Transformer），<span class="arithmatex">\(g\)</span> 是融合函数。<span class="arithmatex">\(g\)</span> 的设计正是真正的架构创造力所在。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>VQA v1</strong>（Antol 等人，2015）引入了该基准，包含来自 MS COCO 的 204,000 张图像上的 614,000 个问题。研究人员很快发现，模型可以通过利用<strong>语言先验</strong>达到惊人高的准确率——对"多少个"问题回答"2"，对"有没有"问题回答"是"，甚至不需要看图像。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>VQA v2</strong>（Goyal 等人，2017）通过为每个问题配对不同答案的两张相似图像来解决这个问题。这迫使模型真正将其推理建立在视觉内容之上。平衡配对设置使数据集规模大约翻倍，并使纯语言捷径的效果大打折扣。</p>
|
||
</li>
|
||
<li>
|
||
<p>其他重要的 VQA 数据集包括 <strong>GQA</strong>（Hudson & Manning，2019），包含需要多步推理的组合性问题；<strong>OK-VQA</strong>（Marino 等人，2019），需要超出图像范围的外部知识；以及 <strong>TextVQA</strong>（Singh 等人，2019），答案依赖于读取图像中的文字。</p>
|
||
</li>
|
||
</ul>
|
||
<p><img alt="VQA 流水线：图像经视觉编码器编码，问题经文本编码器编码，两者表征融合后，融合向量被分类为答案" src="../../images/vqa_pipeline.svg" /></p>
|
||
<ul>
|
||
<li>
|
||
<p>早期的 VQA 模型使用简单策略：从预训练 CNN 中提取图像特征（通常是第 8 章中 ResNet 或 VGGNet 的倒数第二层），用 LSTM（第 6 章）对问题进行编码，然后将它们组合。组合函数 <span class="arithmatex">\(g\)</span> 演变迅速：从简单的逐元素乘法，到双线性池化，再到多模态 Tucker 分解。<strong>双线性注意力</strong>计算 <span class="arithmatex">\(v^T W h\)</span>，其中 <span class="arithmatex">\(W\)</span> 是可学习的交互矩阵，但完整的双线性形式有 <span class="arithmatex">\(O(d_v \times d_h)\)</span> 个参数，规模过大。<strong>MLB</strong>（多模态低秩双线性池化）将其分解为两个低秩投影，使其变得可行。</p>
|
||
</li>
|
||
<li>
|
||
<p>VQA 的突破是注意力机制。<strong>堆叠注意力网络</strong>（Yang 等人，2016）使用问题编码在空间图像区域上施加注意力，迭代式地精炼需要关注的图像部分。这个思想——让问题"关注"相关图像区域——成为了标准做法。</p>
|
||
</li>
|
||
</ul>
|
||
<h2 id="_3">图像描述<a class="headerlink" href="#_3" title="Permanent link">¶</a></h2>
|
||
<ul>
|
||
<li>
|
||
<p>想象一位朋友看着你的度假照片并叙述他们所看到的："一只金毛猎犬在阳光明媚的沙滩上接飞盘。"<strong>图像描述</strong>是生成图像的自然语言描述的任务。与 VQA 不同，这里没有提问——模型必须自行决定哪些内容值得描述。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>Show and Tell</strong>（Vinyals 等人，2015）建立了描述任务的标准编码器-解码器架构。CNN 编码器（如 Inception 或 ResNet）生成一个单一图像特征向量 <span class="arithmatex">\(v\)</span>。该向量被用作 LSTM 解码器的初始隐藏状态，然后逐词自回归地生成描述：</p>
|
||
</li>
|
||
</ul>
|
||
<div class="arithmatex">\[p(w_t \mid w_{1:t-1}, I) = \text{LSTM}(w_{t-1}, h_{t-1})\]</div>
|
||
<ul>
|
||
<li>
|
||
<p>整个模型通过最大化真实描述的对数似然进行端到端训练。推理时使用束搜索（第 7 章）来找到高概率的描述。</p>
|
||
</li>
|
||
<li>
|
||
<p>Show and Tell 的问题在于整张图像被压缩成一个单一向量。对于复杂场景，单一向量无法捕捉所有相关细节。你会丢失空间信息——模型在生成不同词语时无法"回看"图像的特定区域。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>Show, Attend and Tell</strong>（Xu 等人，2015）通过引入<strong>图像区域上的注意力</strong>解决了这个问题。模型不是将图像编码为一个向量，而是由 CNN 产生一个空间特征网格（例如来自 VGGNet 最后一个卷积层的 <span class="arithmatex">\(14 \times 14 \times 512\)</span>）。在每个解码步骤，模型计算这些空间位置上的注意力权重，生成一个突出当前词语最相关区域的上下文向量。</p>
|
||
</li>
|
||
<li>
|
||
<p>回顾第 6 章的注意力机制：解码器隐藏状态充当查询，空间特征充当键和值，注意力权重告诉模型应该看哪里。作者提出了两种变体：<strong>软注意力</strong>（可微分，所有区域的加权平均）和<strong>硬注意力</strong>（对单个区域进行随机采样，使用 REINFORCE 训练）。</p>
|
||
</li>
|
||
</ul>
|
||
<p><img alt="基于注意力的描述：在每个解码步骤，模型关注图像的不同空间区域，例如在生成"狗"这个词时聚焦于狗所在区域" src="../../images/attention_captioning.svg" /></p>
|
||
<ul>
|
||
<li>
|
||
<p>这些模型产生的注意力图具有显著的可解释性：生成"狗"时，注意力集中在狗的区域；生成"海滩"时，注意力转移到沙子和水面。这是注意力机制提供内置可解释性的最早令人信服的演示之一。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>CIDEr</strong>（Vedantam 等人，2015）、<strong>METEOR</strong>、<strong>BLEU</strong> 和 <strong>SPICE</strong> 是标准描述评估指标。CIDEr 计算生成描述与参考描述之间的 TF-IDF 加权 n-gram 相似度，专门为描述评估设计。现代 VLM 通常在 MS COCO Captions 和 NoCaps 等描述基准上用 CIDEr 进行评估。</p>
|
||
</li>
|
||
<li>
|
||
<p>后来的描述模型引入了<strong>自底向上注意力</strong>（Anderson 等人，2018），其中目标检测器（Faster R-CNN，第 8 章）首先提出显著的图像区域，然后描述模型在这些区域特征而非均匀网格上施加注意力。在基于 ViT 的编码器接管之前，这是主导方法。</p>
|
||
</li>
|
||
</ul>
|
||
<h2 id="_4">架构模式<a class="headerlink" href="#_4" title="Permanent link">¶</a></h2>
|
||
<ul>
|
||
<li>每个 VLM 都必须回答一个基本设计问题：视觉和语言在哪个节点交互？答案决定了模型的架构家族。有三种主要模式，各自具有不同的权衡。</li>
|
||
</ul>
|
||
<h3 id="_5">双编码器<a class="headerlink" href="#_5" title="Permanent link">¶</a></h3>
|
||
<ul>
|
||
<li>
|
||
<p>想象两位独立工作的译者——一位读法语文件，另一位读英语文件——他们各自用一种共享的"通用语言"生成摘要。他们在翻译过程中从不交流，但他们的摘要可以直接比较。这就是<strong>双编码器</strong>模式。</p>
|
||
</li>
|
||
<li>
|
||
<p>视觉编码器 <span class="arithmatex">\(f_v\)</span> 和文本编码器 <span class="arithmatex">\(f_t\)</span> 独立地将各自的输入映射到一个维度为 <span class="arithmatex">\(d\)</span> 的共享嵌入空间。图像嵌入为 <span class="arithmatex">\(v = f_v(I) \in \mathbb{R}^d\)</span>，文本嵌入为 <span class="arithmatex">\(t = f_t(q) \in \mathbb{R}^d\)</span>。相似度通过点积或余弦相似度计算：<span class="arithmatex">\(\text{sim}(I, q) = v^T t / (\|v\| \|t\|)\)</span>。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>CLIP</strong>（Radford 等人，2021），在前一篇关于多模态表示的文件中已介绍，是典型的双编码器。它在从互联网抓取的 4 亿图像-文本对上使用对比目标函数（InfoNCE）进行训练。由于编码器相互独立，你可以预计算并缓存所有图像嵌入，使检索极其高效——搜索时只需对查询文本进行编码。</p>
|
||
</li>
|
||
<li>
|
||
<p>双编码器的缺点在于视觉和语言从未在特征层面进行交互。模型无法进行细粒度的跨模态推理：例如，它无法确定描述中的特定词是否对应图像中的特定区域。这限制了它在 VQA 或 grounded 描述等任务中的实用性。</p>
|
||
</li>
|
||
</ul>
|
||
<h3 id="_6">融合编码器<a class="headerlink" href="#_6" title="Permanent link">¶</a></h3>
|
||
<ul>
|
||
<li>
|
||
<p>现在想象两位译者共处一室，积极讨论两篇文件。他们可以指向特定段落、互相提问，并建立共同的理解。这就是<strong>融合编码器</strong>模式。</p>
|
||
</li>
|
||
<li>
|
||
<p>两种模态都被编码，然后通过<strong>交叉注意力层</strong>进行融合，其中一种模态的 token 关注另一种模态的 token。图像首先由视觉编码器处理为一系列 patch 或区域 token <span class="arithmatex">\(V = [v_1, \ldots, v_N]\)</span>。文本被分词化为 <span class="arithmatex">\(T = [t_1, \ldots, t_M]\)</span>。在融合层中，文本 token 通过交叉注意力关注图像 token：</p>
|
||
</li>
|
||
</ul>
|
||
<div class="arithmatex">\[\text{CrossAttn}(T, V) = \text{softmax}\!\left(\frac{(TW_Q)(VW_K)^T}{\sqrt{d_k}}\right)(VW_V)\]</div>
|
||
<ul>
|
||
<li>这实现了细粒度的交互：每个文本 token 都可以关注其所需的特定图像区域。<strong>VisualBERT</strong>、<strong>VilBERT</strong> 和 <strong>UNITER</strong> 等模型使用这种模式。代价是你无法为检索预计算独立的嵌入——每个图像-文本对都需要通过融合层进行完整的前向传播。</li>
|
||
</ul>
|
||
<p><img alt="双编码器与融合编码器对比：双编码器计算独立嵌入和相似度得分，而融合编码器通过交叉注意力层合并模态" src="../../images/dual_vs_fusion_encoder.svg" /></p>
|
||
<h3 id="-">编码器-解码器<a class="headerlink" href="#-" title="Permanent link">¶</a></h3>
|
||
<ul>
|
||
<li>
|
||
<p><strong>编码器-解码器</strong>模式将视觉编码器与自回归生成输出 token 的文本解码器相结合，类似于第 7 章中的 seq2seq 模型。视觉编码器产生上下文图像表征，文本解码器在生成输出文本时对其执行交叉注意力。</p>
|
||
</li>
|
||
<li>
|
||
<p>这种模式天然支持生成式任务：图像描述、自由形式答案的 VQA 以及视觉对话。<strong>GIT</strong>（Generative Image-to-text Transformer，Wang 等人，2022）、<strong>CoCa</strong>（Contrastive Captioner，Yu 等人，2022）和 <strong>PaLI</strong> 使用这种架构。CoCa 巧妙地将双编码器和编码器-解码器模式结合起来：文本解码器的前半部分作为单模态文本编码器（用于对比学习），而后半部分对图像特征执行交叉注意力（用于生成式描述），兼得两者之优势。</p>
|
||
</li>
|
||
<li>
|
||
<p>这三种模式的选择取决于目标任务。双编码器最适合大规模检索。融合编码器最适合细粒度理解任务。编码器-解码器对于生成任务最为通用。现代最先进的 VLM 越来越多地采用编码器-解码器或仅解码器范式，将每项视觉语言任务都视为文本生成。</p>
|
||
</li>
|
||
</ul>
|
||
<h2 id="flamingo">Flamingo：少样本多模态学习<a class="headerlink" href="#flamingo" title="Permanent link">¶</a></h2>
|
||
<ul>
|
||
<li>
|
||
<p>想象一位经验丰富的专家，经过多年对艺术和文学的研究，只需要看一两个例子就能优雅地描述一种全新的绘画风格。<strong>Flamingo</strong>（Alonso 等人，2022，DeepMind）基于相同原理构建：它利用强大的预训练语言模型和预训练视觉编码器，通过轻量级架构组件将其连接，实现多模态任务上的少样本学习。</p>
|
||
</li>
|
||
<li>
|
||
<p>Flamingo 的设计理念保守而有效：保持预训练的视觉编码器（NFNet）和语言模型（Chinchilla）冻结，仅学习连接它们的"胶水"。这种胶水由两个组件组成：<strong>Perceiver 重采样器</strong>和<strong>门控交叉注意力层</strong>。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>Perceiver 重采样器</strong>将视觉编码器的变长输出（取决于图像分辨率）压缩为一组固定数量的 <span class="arithmatex">\(N\)</span> 个视觉 token（通常 <span class="arithmatex">\(N = 64\)</span>）。它的工作原理是初始化一组 <span class="arithmatex">\(N\)</span> 个可学习的查询向量，并使用交叉注意力让这些查询关注完整的视觉编码器输出。这本质上是 Perceiver 架构（Jaegle 等人，2021）作为瓶颈的应用——无论输入图像大小如何，它都能生成紧凑的、固定大小的视觉表示。</p>
|
||
</li>
|
||
</ul>
|
||
<div class="arithmatex">\[z = \text{CrossAttn}(Q_{\text{learned}}, V_{\text{image}}) \in \mathbb{R}^{N \times d}\]</div>
|
||
<ul>
|
||
<li><strong>门控交叉注意力层</strong>交错插入在冻结的语言模型层之间。在每个这样的层中，语言模型的文本 token 对 Perceiver 重采样器产生的视觉 token 执行交叉注意力。关键之处在于，每个门控交叉注意力层包含一个可学习的标量门控 <span class="arithmatex">\(\alpha\)</span>，初始化为零，将交叉注意力输出乘以 <span class="arithmatex">\(\alpha\)</span> 后再加到残差流中：</li>
|
||
</ul>
|
||
<div class="arithmatex">\[\hat{x} = x + \alpha \cdot \text{CrossAttn}(x, z)\]</div>
|
||
<ul>
|
||
<li>初始化 <span class="arithmatex">\(\alpha = 0\)</span> 意味着训练开始时交叉注意力不贡献任何信息，模型行为与原始的冻结语言模型完全相同。门控在训练过程中逐渐打开，平滑地整合视觉信息，同时不破坏语言模型的预训练表示。</li>
|
||
</ul>
|
||
<p><img alt="Flamingo 架构：冻结的视觉编码器输入 Perceiver 重采样器，生成固定长度的视觉 token，通过交错在 LM 块之间的门控交叉注意力层注入冻结的 LM" src="../../images/flamingo_architecture.svg" /></p>
|
||
<ul>
|
||
<li>
|
||
<p>Flamingo 原生支持<strong>交错图像-文本序列</strong>。你可以向它输入包含多张图像穿插文本的提示，例如："[图像 1] 这是一只猫。[图像 2] 这是一只狗。[图像 3] 这是一个 ___。"模型将每张图像通过视觉编码器和 Perceiver 重采样器处理，得到的视觉 token 插入到文本序列中的对应位置。语言模型的因果注意力掩码确保每个文本 token 只能关注当前及之前图像的视觉 token。</p>
|
||
</li>
|
||
<li>
|
||
<p>这种交错机制实现了强大的<strong>少样本多模态学习</strong>。通过在上下文中提供少量图像-文本示例，Flamingo 可以在没有任何梯度更新的情况下执行新任务。在 VQAv2、OK-VQA 和描述等基准上，具有 800 亿参数的 Flamingo 实现了最先进的少样本性能，仅需 4 到 32 个示例即可匹配甚至超越经过微调的专家模型。</p>
|
||
</li>
|
||
</ul>
|
||
<h2 id="llava">LLaVA 与视觉指令微调<a class="headerlink" href="#llava" title="Permanent link">¶</a></h2>
|
||
<ul>
|
||
<li>
|
||
<p>想象你有一位出色的语言专家（一个 LLM）和一位出色的艺术评论家（一个视觉编码器）。如果你能教会艺术评论家"说语言专家的语言"，他们就可以无缝协作。<strong>LLaVA</strong>（Large Language and Vision Assistant，Liu 等人，2023）正是这样做的：它使用一个简单的线性层将视觉特征投影到 LLM 的 token 嵌入空间，然后在指令遵循数据上微调整个系统。</p>
|
||
</li>
|
||
<li>
|
||
<p>LLaVA 的架构出奇地简单。图像由一个预训练的 CLIP ViT-L/14 视觉编码器编码为一个 patch 特征网格 <span class="arithmatex">\(V \in \mathbb{R}^{N \times d_v}\)</span>，其中 <span class="arithmatex">\(N = 256\)</span> 个 patch（对于 336px 图像和 14px patch）。一个<strong>投影层</strong> <span class="arithmatex">\(W\)</span> 将这些视觉特征映射到 LLM 的嵌入维度：</p>
|
||
</li>
|
||
</ul>
|
||
<div class="arithmatex">\[H_v = VW, \quad W \in \mathbb{R}^{d_v \times d_{\text{LLM}}}\]</div>
|
||
<ul>
|
||
<li>投影后的视觉 token <span class="arithmatex">\(H_v\)</span> 直接与文本 token 嵌入拼接，作为一个单一序列输入到 LLM（Vicuna，一个微调后的 LLaMA）。LLM 使用其标准因果自注意力处理它们——没有特殊的交叉注意力层，没有 perceiver，只有拼接。视觉 token 被当作恰好编码了视觉信息的文本 token 来处理。</li>
|
||
</ul>
|
||
<p><img alt="LLaVA 架构：CLIP ViT 将图像编码为 patch 特征，线性投影将其映射到 LLM 嵌入空间，投影后的视觉 token 拼接在文本 token 之前并输入到 LLM" src="../../images/llava_architecture.svg" /></p>
|
||
<ul>
|
||
<li>
|
||
<p><strong>视觉指令微调</strong>是 LLaVA 的关键训练创新。作者使用 GPT-4 从 COCO 图像生成了 158,000 个多模态指令遵循示例。每个示例包含一张图像和一个对话式指令（例如"详细描述这张图像"、"这张图像有什么不寻常之处？"、"如果我是一名游客参观这个地方，我应该知道什么？"）。模型接受训练，根据图像和指令生成 GPT-4 撰写的回答。</p>
|
||
</li>
|
||
<li>
|
||
<p>训练分为两个阶段。<strong>阶段 1（预训练）</strong>：仅训练投影层 <span class="arithmatex">\(W\)</span>，使用图像-描述对（来自 CC3M 的 595K 数据），视觉编码器和 LLM 都保持冻结。这教会 <span class="arithmatex">\(W\)</span> 将视觉特征与 LLM 的嵌入空间对齐。<strong>阶段 2（微调）</strong>：投影层和 LLM 在指令遵循数据上联合微调，视觉编码器保持冻结。这教会模型遵循复杂的视觉指令。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>LLaVA-1.5</strong> 通过三项关键更改改进了原始版本：将单层线性投影替换为两层 MLP（更具表现力的映射），使用更高分辨率的图像（336px 而非 224px，产生更多 patch token），以及在训练混合数据中加入学术 VQA 数据集。这些看似细微的修改带来了基准性能的大幅提升。</p>
|
||
</li>
|
||
<li>
|
||
<p>LLaVA 的方法证明，你不需要像 Flamingo 的 Perceiver 重采样器或门控交叉注意力那样复杂的架构创新。一个简单的线性投影，结合高质量的指令微调数据，就足以有效地将视觉编码器连接到 LLM。这种简洁性使得 LLaVA 极具影响力——后续大多数开源 VLM 都遵循类似的方案。</p>
|
||
</li>
|
||
</ul>
|
||
<h2 id="_7">扩展视觉语言模型<a class="headerlink" href="#_7" title="Permanent link">¶</a></h2>
|
||
<ul>
|
||
<li>该领域从概念验证型 VLM 迅速发展为在数十亿图像-文本对上训练的工业级系统。三个模型家族展示了不同的扩展方法。</li>
|
||
</ul>
|
||
<h3 id="pali">PaLI<a class="headerlink" href="#pali" title="Permanent link">¶</a></h3>
|
||
<ul>
|
||
<li>
|
||
<p><strong>PaLI</strong>（Pathways Language and Image model，Chen 等人，2022，Google）同时扩展视觉编码器和语言模型。PaLI 使用 ViT-e（40 亿参数）作为视觉编码器，mT5（130 亿参数）作为语言模型，总计 170 亿参数。图像被编码为一系列 patch token，拼接在文本 token 之前，输入到编码器-解码器架构的 mT5。</p>
|
||
</li>
|
||
<li>
|
||
<p>PaLI 的关键洞见是<strong>扩展视觉编码器与扩展语言模型同样重要</strong>。先前的工作通常使用固定的、中等规模的视觉骨干网络（如 ViT-B 或 ViT-L），将参数预算全部投入 LLM。PaLI 表明，一个 40 亿参数的 ViT-e，在 JFT-4B（40 亿张标注图像）上预训练后，能够显著提升 OCR 和空间推理等细粒度视觉任务的性能。</p>
|
||
</li>
|
||
<li>
|
||
<p>PaLI 在 WebLI（一个包含 109 种语言、100 亿图像-文本对的数据集）上训练，因此天然具备多语言能力。模型通过混合任务进行预训练：图像描述、VQA 和图像-文本匹配，全部作为文本到文本生成任务（遵循第 7 章的 T5 范式）。<strong>PaLI-X</strong>（550 亿参数）和 <strong>PaLI-3</strong>（50 亿，使用 SigLIP 作为视觉编码器）是后续迭代版本。</p>
|
||
</li>
|
||
</ul>
|
||
<h3 id="qwen-vl">Qwen-VL<a class="headerlink" href="#qwen-vl" title="Permanent link">¶</a></h3>
|
||
<ul>
|
||
<li>
|
||
<p><strong>Qwen-VL</strong>（Bai 等人，2023，阿里巴巴）在 Qwen LLM 基础上增加了一个 ViT 视觉编码器和一个单层交叉注意力模块（类似于 Flamingo 的 Perceiver 重采样器），将视觉编码器的输出压缩为一组固定的 256 个视觉 token。视觉 token 与文本 token 拼接后由 Qwen LLM 处理。</p>
|
||
</li>
|
||
<li>
|
||
<p>Qwen-VL 的训练采用三阶段方案。阶段 1：在 14 亿个弱监督图像-文本对上预训练，仅解冻视觉编码器。阶段 2：在更高质量的数据上进行多任务预训练，包括 VQA、描述、定位和 OCR 数据集，整个模型解冻。阶段 3：在指令遵循和对话数据上进行监督微调。这种从噪声网络数据到精选指令数据的渐进式精炼，是大多数现代 VLM 共享的模式。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>Qwen2-VL</strong>（2024）引入了<strong>动态分辨率</strong>支持：模型不是将所有图像缩放到固定大小，而是通过动态调整视觉 token 数量以原始分辨率处理图像。更高分辨率的图像产生更多 token，更低分辨率的图像产生更少 token。这在不浪费低分辨率输入计算量的前提下，提升了文档理解和细粒度识别等对细节敏感的任务的性能。</p>
|
||
</li>
|
||
</ul>
|
||
<h3 id="internvl">InternVL<a class="headerlink" href="#internvl" title="Permanent link">¶</a></h3>
|
||
<ul>
|
||
<li>
|
||
<p><strong>InternVL</strong>（Chen 等人，2024，上海人工智能实验室）激进地扩展了视觉编码器，使用 InternViT-6B——一个 60 亿参数的视觉 Transformer——与语言模型配对。关键的架构贡献是<strong>动态高分辨率处理</strong>：图像被分割为 448x448 像素的图块，每个图块由视觉编码器独立处理，得到的图块特征与完整图像的缩略图特征拼接。这使得模型能够处理任意宽高比和分辨率的图像。</p>
|
||
</li>
|
||
<li>
|
||
<p>InternVL-2 进一步引入了<strong>渐进对齐训练</strong>：首先用对比目标（如 CLIP）对齐视觉编码器，然后通过轻量级 MLP 连接器将其连接到 LLM，最后在指令数据上进行端到端微调。这种渐进策略防止了视觉编码器预训练表示的灾难性遗忘。</p>
|
||
</li>
|
||
</ul>
|
||
<p><img alt="扩展 VLM：PaLI、Qwen-VL 和 InternVL 的比较，展示了连接视觉编码器和语言模型的不同方法，包括其训练阶段" src="../../images/scaling_vlms_comparison.svg" /></p>
|
||
<ul>
|
||
<li>所有三个模型家族的一个共同主题是<strong>训练数据精选</strong>的重要性。从网络抓取的原始图像-文本对是噪声大且常常不对齐的。后续的训练阶段逐步过滤和精炼数据，从数十亿噪声对过渡到数百万高质量指令示例。最终微调数据的质量往往比模型的原始参数数量更为重要。</li>
|
||
</ul>
|
||
<h2 id="_8">定位与指代<a class="headerlink" href="#_8" title="Permanent link">¶</a></h2>
|
||
<ul>
|
||
<li>
|
||
<p>想象你在人群中指着一个人说"戴红帽子的女士"。你在用语言指代一个特定的空间区域。<strong>视觉定位</strong>是相反的过程：给定一张图像和一个自然语言表述，模型必须识别（定位）所指的对象。<strong>指代表达理解</strong>产生边界框；<strong>指代表达分割</strong>产生像素掩码。</p>
|
||
</li>
|
||
<li>
|
||
<p>形式上，给定一张图像 <span class="arithmatex">\(I\)</span> 和一个指代表达 <span class="arithmatex">\(r\)</span>（例如"左边那只大型棕色狗"），模型预测一个边界框 <span class="arithmatex">\(b = (x, y, w, h)\)</span> 或一组定位所引用对象的坐标。数据集包括 <strong>RefCOCO</strong>、<strong>RefCOCO+</strong> 和 <strong>RefCOCOg</strong>，每个数据集包含具有多个对象的图像以及每个对象的明确指代表达。</p>
|
||
</li>
|
||
<li>
|
||
<p>早期的定位模型使用两阶段方法：首先生成区域提议（使用 Faster R-CNN 或类似方法），然后使用融合模型对每个提议与语言查询进行评分。评分最高的区域即为预测结果。这种方法计算代价高昂，且受限于提议的质量。</p>
|
||
</li>
|
||
<li>
|
||
<p>现代 VLM 将定位直接整合到生成式框架中。关键思想是将边界框坐标表示为<strong>文本 token</strong>。你将连续的坐标空间离散化为槽位（例如 <span class="arithmatex">\(x, y, w, h\)</span> 各 1000 个槽位），并向词汇表中添加特殊的位置 token，如 <code><loc_342></code>。然后模型通过输出一系列位置 token 来生成边界框：</p>
|
||
</li>
|
||
</ul>
|
||
<div class="arithmatex">\[\text{输出: } \texttt{<loc\_102><loc\_215><loc\_487><loc\_398>}\]</div>
|
||
<ul>
|
||
<li>
|
||
<p>这种 token 化技巧使得任何自回归语言模型无需架构更改即可执行定位——它只需学会"说坐标"。<strong>Pix2Seq</strong>（Chen 等人，2022）率先将这种方法用于目标检测，而 Qwen-VL、Ferret 和 Kosmos-2 等模型将其扩展到指代表达理解和短语定位。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>Kosmos-2</strong>（Peng 等人，2023，Microsoft）通过将空间位置表示为嵌入在生成文本中的特殊 token，为多模态 LLM 增加了定位能力。例如，它可以生成："一只 <code><phrase></code> 金毛猎犬 <code></phrase></code> <code><box></code> <code><loc_102></code> <code><loc_215></code> <code><loc_487></code> <code><loc_398></code> <code></box></code> 正在接飞盘。"这种文本和空间 token 的交错融合实现了同步描述和定位。</p>
|
||
</li>
|
||
</ul>
|
||
<p><img alt="通过坐标 token 化实现定位：模型生成文本 token 与离散化的边界框坐标 token 交错在一起，定位描述中提到的物体" src="../../images/grounding_coordinate_tokens.svg" /></p>
|
||
<ul>
|
||
<li><strong>定点指向</strong>将定位更进一步：模型不再输出边界框，而是预测一个单一的点（通常是指代物体的中心）。这对于交互式应用非常有用，例如用户问"最近的出口在哪里？"，模型返回一个叠加在图像上的坐标。<strong>Shikra</strong> 和 <strong>Ferret</strong> 等模型支持基于点的指代以及基于框的定位。</li>
|
||
</ul>
|
||
<h2 id="ocr">免 OCR 文档理解<a class="headerlink" href="#ocr" title="Permanent link">¶</a></h2>
|
||
<ul>
|
||
<li>
|
||
<p>传统的文档理解流水线很复杂：首先运行 OCR 引擎提取文本和布局，然后将提取的文本输入语言模型。这种多阶段方法很脆弱——OCR 错误向下游传播，空间布局信息常常丢失或表征不良。如果模型能像人类一样直接从像素中读取信息呢？</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>Donut</strong>（Document Understanding Transformer，Kim 等人，2022）完全消除了 OCR。它使用 Swin Transformer（第 8 章）作为视觉编码器处理文档图像，并使用 BART 风格的 Transformer 解码器直接从视觉特征生成结构化文本输出。解码器可以根据任务生成 JSON、键值对或纯文本。</p>
|
||
</li>
|
||
<li>
|
||
<p>Donut 的训练分为两个阶段。<strong>预训练</strong>：模型通过执行合成 OCR 来学习阅读——给定一张文档图像，生成完整的文本内容。这在从文本语料库渲染的数百万张合成文档图像上进行训练，教会视觉编码器识别字符、字体和布局。<strong>微调</strong>：模型通过训练生成特定于任务的结构化输出，适应特定的下游任务，如收据解析、表格理解或文档分类。</p>
|
||
</li>
|
||
<li>
|
||
<p>Donut 解码器使用特殊的提示方案：任务由提示 token 指定（例如分类用 <code><doc_class></code>，收据解析用 <code><parse_receipt></code>），模型根据此提示生成输出。这种统一接口使得单个模型可以处理多种文档理解任务。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>Pix2Struct</strong>（Lee 等人，2023，Google）将免 OCR 思想应用于网页理解和图表/图形理解。关键的预训练目标是<strong>截图解析</strong>：给定一个网页的带掩码截图，模型生成产生可见区域的底层 HTML。这教会模型理解视觉呈现与结构化标记之间的关系。</p>
|
||
</li>
|
||
<li>
|
||
<p>Pix2Struct 引入了<strong>可变分辨率输入处理</strong>：它并不是将所有图像缩放到固定大小（这会扭曲宽高比并破坏精细文字），而是在保持原始宽高比的同时将图像打包为固定数量的 patch。一个高而窄的文档产生一个高而窄的 patch 网格。这对于文档理解至关重要，因为宽高比携带着语义信息（收据窄而高；表格宽而短）。</p>
|
||
</li>
|
||
</ul>
|
||
<p><img alt="免 OCR 文档理解：Donut 和 Pix2Struct 通过视觉编码器直接处理文档图像，无需任何 OCR 预处理即可生成结构化文本输出" src="../../images/ocr_free_document_understanding.svg" /></p>
|
||
<ul>
|
||
<li>
|
||
<p><strong>Nougat</strong>（Blecher 等人，2023，Meta）将 Donut 架构专门应用于学术论文，直接从 PDF 页面图像生成完整的 LaTeX 标记。它可以处理复杂的数学方程、表格和图形——这些任务正是传统 OCR 流水线难以应付的。该模型在 PDF 页面图像及其对应的 LaTeX 源代码对上进行训练。</p>
|
||
</li>
|
||
<li>
|
||
<p>免 OCR 模型的成功展示了深度学习中的一个更广泛原则：直接从原始输入（像素）学习的端到端模型通常优于复杂的多阶段流水线，因为它们可以联合优化所有组件，并学习专门针对最终任务定制的表示。中间的 OCR 步骤是一个瓶颈，限制了模型能够学习的内容。</p>
|
||
</li>
|
||
</ul>
|
||
<h2 id="token">视觉 Token 流水线<a class="headerlink" href="#token" title="Permanent link">¶</a></h2>
|
||
<ul>
|
||
<li>
|
||
<p>无论架构家族如何，每个 VLM 都必须将图像转换为语言模型可以处理的一系列 token。理解这一流水线至关重要。不同模型的处理过程有所差异，但总体流程如下：</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>第 1 步：Patch 提取。</strong> 图像（高度 <span class="arithmatex">\(H\)</span>，宽度 <span class="arithmatex">\(W\)</span>）被划分为不重叠的、大小为 <span class="arithmatex">\(P \times P\)</span> 的 patch，产生 <span class="arithmatex">\(N = HW / P^2\)</span> 个 patch。对于 336x336 图像和 14x14 patch，<span class="arithmatex">\(N = 576\)</span>。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>第 2 步：视觉编码。</strong> 每个 patch 经过线性投影并通过视觉编码器（通常是 ViT）。输出是一系列上下文 patch 嵌入 <span class="arithmatex">\(V = [v_1, \ldots, v_N] \in \mathbb{R}^{N \times d_v}\)</span>。这些嵌入既携带局部外观信息，也携带全局上下文（来自自注意力）。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>第 3 步：Token 压缩（可选）。</strong> 一些模型将 <span class="arithmatex">\(N\)</span> 个视觉 token 压缩为更少的 <span class="arithmatex">\(M \ll N\)</span> 个 token，以减少语言模型的计算负担。Flamingo 使用 Perceiver 重采样器（<span class="arithmatex">\(M = 64\)</span>）；Qwen-VL 使用交叉注意力（<span class="arithmatex">\(M = 256\)</span>）；<strong>Q-Former</strong>（在 BLIP-2 中使用，Li 等人，2023）使用一组 <span class="arithmatex">\(M = 32\)</span> 个可学习查询 token，对视觉编码器的输出执行交叉注意力。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>第 4 步：投影。</strong> 视觉 token（全部或压缩后的集合）通过线性层或 MLP 投影到语言模型的嵌入空间。投影后，视觉 token 与文本 token 嵌入具有相同维度，可以与它们拼接。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>第 5 步：注入 LLM。</strong> 投影后的视觉 token 在特殊 <code><image></code> 占位符 token 的位置插入到 token 序列中，组合后的序列由语言模型处理。LLM 的自注意力使文本 token 能够关注视觉 token，反之亦然。</p>
|
||
</li>
|
||
</ul>
|
||
<p><img alt="视觉 token 流水线：图像 patch 被提取，由 ViT 编码，可选地由 Perceiver 或 Q-Former 压缩，投影到 LLM 维度，并与文本 token 拼接" src="../../images/visual_token_pipeline.svg" /></p>
|
||
<ul>
|
||
<li>
|
||
<p>视觉 token 的数量直接影响计算成本。每个视觉 token 参与 LLM 的自注意力，其复杂度与序列长度的平方成正比。具有多个 patch 的高分辨率图像可能产生数百或数千个视觉 token，占据 LLM 上下文窗口的主导地位。这就是 token 压缩的重要性所在：将 576 个视觉 token 减少到 64 个，可将视觉部分在注意力中的贡献减少约 9 倍。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>BLIP-2</strong>（Li 等人，2023）以其高效的桥接策略而闻名。它引入了一个轻量级的 <strong>Q-Former</strong>（一个带有可学习查询的小型 Transformer），位于冻结的视觉编码器和冻结的 LLM 之间。Q-Former 是唯一可训练的组件——视觉编码器和 LLM 都保持冻结。它的预训练分为两个阶段：首先是图像-文本对比学习、匹配和描述目标（连接视觉编码器），然后是语言生成目标（连接 LLM）。这种模块化设计使得 BLIP-2 可以将任何视觉编码器插入到任何 LLM 中。</p>
|
||
</li>
|
||
</ul>
|
||
<h2 id="_9">训练目标<a class="headerlink" href="#_9" title="Permanent link">¶</a></h2>
|
||
<ul>
|
||
<li>
|
||
<p>VLM 使用多种目标的组合进行训练，具体取决于架构模式：</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>图像-文本对比损失（ITC）：</strong> 在共享嵌入空间中对齐图像和文本表示，如 CLIP 中所示。这是双编码器的主要目标，也常被用作融合模型的预训练目标。该损失就是上一篇文件中的 InfoNCE 损失。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>图像-文本匹配（ITM）：</strong> 一个二分类目标——给定图像和文本，预测它们是否匹配。困难负样本（与不同图像配对的相似文本）使这项任务具有挑战性，迫使模型学习细粒度的对齐。</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>语言建模（LM）：</strong> 标准的自回归语言建模目标——给定之前的所有 token 预测下一个 token。对于 VLM，"之前的 token" 包括视觉 token，因此模型学习在视觉输入条件下生成文本。这是编码器-解码器和仅解码器 VLM 的主要目标。</p>
|
||
</li>
|
||
</ul>
|
||
<div class="arithmatex">\[\mathcal{L}_{\text{LM}} = -\sum_{t=1}^{T} \log p(w_t \mid w_{<t}, V)\]</div>
|
||
<ul>
|
||
<li>
|
||
<p><strong>前缀语言建模：</strong> 一种变体，其中图像和文本前缀作为上下文提供（不进行训练），模型仅训练生成后续部分。这用于 PaLI 和 SimVLM 等模型。</p>
|
||
</li>
|
||
<li>
|
||
<p>大多数现代 VLM 在预训练期间结合多个目标（例如 BLIP 中的 ITC + ITM + LM，CoCa 中的 ITC + LM），然后在指令数据上使用纯 LM 目标进行微调。</p>
|
||
</li>
|
||
</ul>
|
||
<h2 id="colab-notebook">编程练习（使用 CoLab 或 notebook）<a class="headerlink" href="#colab-notebook" title="Permanent link">¶</a></h2>
|
||
<ol>
|
||
<li>
|
||
<p>实现一个简单的基于注意力的图像描述解码器。使用随机的"图像特征"作为编码器输出，训练解码器生成固定的描述，观察注意力权重在每个解码步骤如何跨空间位置移动。
|
||
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="kn">import</span><span class="w"> </span><span class="nn">jax</span>
|
||
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="kn">import</span><span class="w"> </span><span class="nn">jax.numpy</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">jnp</span>
|
||
<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="kn">import</span><span class="w"> </span><span class="nn">matplotlib.pyplot</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">plt</span>
|
||
<a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a>
|
||
<a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a><span class="c1"># 模拟 4x4 空间网格的图像特征（16 个区域，dim=32）</span>
|
||
<a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a><span class="n">key</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">PRNGKey</span><span class="p">(</span><span class="mi">42</span><span class="p">)</span>
|
||
<a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a><span class="n">k1</span><span class="p">,</span> <span class="n">k2</span><span class="p">,</span> <span class="n">k3</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
|
||
<a id="__codelineno-0-8" name="__codelineno-0-8" href="#__codelineno-0-8"></a><span class="n">img_features</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">k1</span><span class="p">,</span> <span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">32</span><span class="p">))</span> <span class="c1"># 16 个空间区域，32 维</span>
|
||
<a id="__codelineno-0-9" name="__codelineno-0-9" href="#__codelineno-0-9"></a>
|
||
<a id="__codelineno-0-10" name="__codelineno-0-10" href="#__codelineno-0-10"></a><span class="c1"># 词汇表：0=<start>, 1="a", 2="red", 3="car", 4=<end></span>
|
||
<a id="__codelineno-0-11" name="__codelineno-0-11" href="#__codelineno-0-11"></a><span class="n">vocab_size</span><span class="p">,</span> <span class="n">embed_dim</span><span class="p">,</span> <span class="n">hidden_dim</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">32</span>
|
||
<a id="__codelineno-0-12" name="__codelineno-0-12" href="#__codelineno-0-12"></a><span class="n">W_embed</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">k2</span><span class="p">,</span> <span class="p">(</span><span class="n">vocab_size</span><span class="p">,</span> <span class="n">embed_dim</span><span class="p">))</span> <span class="o">*</span> <span class="mf">0.1</span>
|
||
<a id="__codelineno-0-13" name="__codelineno-0-13" href="#__codelineno-0-13"></a><span class="n">W_attn_q</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">k3</span><span class="p">,</span> <span class="p">(</span><span class="n">hidden_dim</span><span class="p">,</span> <span class="mi">32</span><span class="p">))</span> <span class="o">*</span> <span class="mf">0.1</span> <span class="c1"># 查询投影</span>
|
||
<a id="__codelineno-0-14" name="__codelineno-0-14" href="#__codelineno-0-14"></a>
|
||
<a id="__codelineno-0-15" name="__codelineno-0-15" href="#__codelineno-0-15"></a><span class="k">def</span><span class="w"> </span><span class="nf">attend</span><span class="p">(</span><span class="n">h</span><span class="p">,</span> <span class="n">img_feats</span><span class="p">,</span> <span class="n">W_q</span><span class="p">):</span>
|
||
<a id="__codelineno-0-16" name="__codelineno-0-16" href="#__codelineno-0-16"></a><span class="w"> </span><span class="sd">"""在给定解码器状态 h 的情况下计算图像特征上的软注意力。"""</span>
|
||
<a id="__codelineno-0-17" name="__codelineno-0-17" href="#__codelineno-0-17"></a> <span class="n">query</span> <span class="o">=</span> <span class="n">h</span> <span class="o">@</span> <span class="n">W_q</span> <span class="c1"># (32,)</span>
|
||
<a id="__codelineno-0-18" name="__codelineno-0-18" href="#__codelineno-0-18"></a> <span class="n">scores</span> <span class="o">=</span> <span class="n">img_feats</span> <span class="o">@</span> <span class="n">query</span> <span class="c1"># (16,)</span>
|
||
<a id="__codelineno-0-19" name="__codelineno-0-19" href="#__codelineno-0-19"></a> <span class="n">weights</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">scores</span><span class="p">)</span> <span class="c1"># (16,)</span>
|
||
<a id="__codelineno-0-20" name="__codelineno-0-20" href="#__codelineno-0-20"></a> <span class="n">context</span> <span class="o">=</span> <span class="n">weights</span> <span class="o">@</span> <span class="n">img_feats</span> <span class="c1"># (32,)</span>
|
||
<a id="__codelineno-0-21" name="__codelineno-0-21" href="#__codelineno-0-21"></a> <span class="k">return</span> <span class="n">context</span><span class="p">,</span> <span class="n">weights</span>
|
||
<a id="__codelineno-0-22" name="__codelineno-0-22" href="#__codelineno-0-22"></a>
|
||
<a id="__codelineno-0-23" name="__codelineno-0-23" href="#__codelineno-0-23"></a><span class="c1"># 简单的 GRU 风格步骤（为说明目的，仅用线性 + tanh）</span>
|
||
<a id="__codelineno-0-24" name="__codelineno-0-24" href="#__codelineno-0-24"></a><span class="n">W_h</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">PRNGKey</span><span class="p">(</span><span class="mi">0</span><span class="p">),</span> <span class="p">(</span><span class="n">embed_dim</span> <span class="o">+</span> <span class="mi">32</span><span class="p">,</span> <span class="n">hidden_dim</span><span class="p">))</span> <span class="o">*</span> <span class="mf">0.1</span>
|
||
<a id="__codelineno-0-25" name="__codelineno-0-25" href="#__codelineno-0-25"></a>
|
||
<a id="__codelineno-0-26" name="__codelineno-0-26" href="#__codelineno-0-26"></a><span class="k">def</span><span class="w"> </span><span class="nf">decode_step</span><span class="p">(</span><span class="n">h</span><span class="p">,</span> <span class="n">word_idx</span><span class="p">,</span> <span class="n">img_feats</span><span class="p">):</span>
|
||
<a id="__codelineno-0-27" name="__codelineno-0-27" href="#__codelineno-0-27"></a> <span class="n">context</span><span class="p">,</span> <span class="n">attn_weights</span> <span class="o">=</span> <span class="n">attend</span><span class="p">(</span><span class="n">h</span><span class="p">,</span> <span class="n">img_feats</span><span class="p">,</span> <span class="n">W_attn_q</span><span class="p">)</span>
|
||
<a id="__codelineno-0-28" name="__codelineno-0-28" href="#__codelineno-0-28"></a> <span class="n">word_emb</span> <span class="o">=</span> <span class="n">W_embed</span><span class="p">[</span><span class="n">word_idx</span><span class="p">]</span> <span class="c1"># (16,)</span>
|
||
<a id="__codelineno-0-29" name="__codelineno-0-29" href="#__codelineno-0-29"></a> <span class="n">inp</span> <span class="o">=</span> <span class="n">jnp</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">word_emb</span><span class="p">,</span> <span class="n">context</span><span class="p">])</span> <span class="c1"># (48,)</span>
|
||
<a id="__codelineno-0-30" name="__codelineno-0-30" href="#__codelineno-0-30"></a> <span class="n">h_new</span> <span class="o">=</span> <span class="n">jnp</span><span class="o">.</span><span class="n">tanh</span><span class="p">(</span><span class="n">inp</span> <span class="o">@</span> <span class="n">W_h</span><span class="p">)</span> <span class="c1"># (32,)</span>
|
||
<a id="__codelineno-0-31" name="__codelineno-0-31" href="#__codelineno-0-31"></a> <span class="k">return</span> <span class="n">h_new</span><span class="p">,</span> <span class="n">attn_weights</span>
|
||
<a id="__codelineno-0-32" name="__codelineno-0-32" href="#__codelineno-0-32"></a>
|
||
<a id="__codelineno-0-33" name="__codelineno-0-33" href="#__codelineno-0-33"></a><span class="c1"># 运行解码序列：<start> -> "a" -> "red" -> "car" -> <end></span>
|
||
<a id="__codelineno-0-34" name="__codelineno-0-34" href="#__codelineno-0-34"></a><span class="n">target_seq</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]</span>
|
||
<a id="__codelineno-0-35" name="__codelineno-0-35" href="#__codelineno-0-35"></a><span class="n">h</span> <span class="o">=</span> <span class="n">jnp</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">hidden_dim</span><span class="p">)</span>
|
||
<a id="__codelineno-0-36" name="__codelineno-0-36" href="#__codelineno-0-36"></a><span class="n">all_attn</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<a id="__codelineno-0-37" name="__codelineno-0-37" href="#__codelineno-0-37"></a><span class="k">for</span> <span class="n">word_idx</span> <span class="ow">in</span> <span class="n">target_seq</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]:</span>
|
||
<a id="__codelineno-0-38" name="__codelineno-0-38" href="#__codelineno-0-38"></a> <span class="n">h</span><span class="p">,</span> <span class="n">attn_w</span> <span class="o">=</span> <span class="n">decode_step</span><span class="p">(</span><span class="n">h</span><span class="p">,</span> <span class="n">word_idx</span><span class="p">,</span> <span class="n">img_features</span><span class="p">)</span>
|
||
<a id="__codelineno-0-39" name="__codelineno-0-39" href="#__codelineno-0-39"></a> <span class="n">all_attn</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">attn_w</span><span class="p">)</span>
|
||
<a id="__codelineno-0-40" name="__codelineno-0-40" href="#__codelineno-0-40"></a>
|
||
<a id="__codelineno-0-41" name="__codelineno-0-41" href="#__codelineno-0-41"></a><span class="c1"># 可视化每一步的注意力图（重塑为 4x4 网格）</span>
|
||
<a id="__codelineno-0-42" name="__codelineno-0-42" href="#__codelineno-0-42"></a><span class="n">words</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"<start>"</span><span class="p">,</span> <span class="s2">"a"</span><span class="p">,</span> <span class="s2">"red"</span><span class="p">,</span> <span class="s2">"car"</span><span class="p">]</span>
|
||
<a id="__codelineno-0-43" name="__codelineno-0-43" href="#__codelineno-0-43"></a><span class="n">fig</span><span class="p">,</span> <span class="n">axes</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">14</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>
|
||
<a id="__codelineno-0-44" name="__codelineno-0-44" href="#__codelineno-0-44"></a><span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="p">(</span><span class="n">ax</span><span class="p">,</span> <span class="n">w</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">axes</span><span class="p">,</span> <span class="n">words</span><span class="p">)):</span>
|
||
<a id="__codelineno-0-45" name="__codelineno-0-45" href="#__codelineno-0-45"></a> <span class="n">ax</span><span class="o">.</span><span class="n">imshow</span><span class="p">(</span><span class="n">all_attn</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">4</span><span class="p">),</span> <span class="n">cmap</span><span class="o">=</span><span class="s1">'viridis'</span><span class="p">)</span>
|
||
<a id="__codelineno-0-46" name="__codelineno-0-46" href="#__codelineno-0-46"></a> <span class="n">ax</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="sa">f</span><span class="s1">'生成"</span><span class="si">{</span><span class="n">w</span><span class="si">}</span><span class="s1">"后</span><span class="se">\n</span><span class="s1">关注的区域'</span><span class="p">)</span>
|
||
<a id="__codelineno-0-47" name="__codelineno-0-47" href="#__codelineno-0-47"></a> <span class="n">ax</span><span class="o">.</span><span class="n">axis</span><span class="p">(</span><span class="s1">'off'</span><span class="p">)</span>
|
||
<a id="__codelineno-0-48" name="__codelineno-0-48" href="#__codelineno-0-48"></a><span class="n">plt</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s1">'每个解码步骤的图像区域注意力'</span><span class="p">)</span>
|
||
<a id="__codelineno-0-49" name="__codelineno-0-49" href="#__codelineno-0-49"></a><span class="n">plt</span><span class="o">.</span><span class="n">tight_layout</span><span class="p">();</span> <span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
<a id="__codelineno-0-50" name="__codelineno-0-50" href="#__codelineno-0-50"></a><span class="c1"># 尝试修改 img_features，观察注意力模式如何变化！</span>
|
||
</code></pre></div></p>
|
||
</li>
|
||
<li>
|
||
<p>模拟视觉 token 流水线：将图像划分为 patch，将 patch 投影到嵌入空间，与文本 token 嵌入拼接，并在组合序列上运行单层自注意力。
|
||
<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="kn">import</span><span class="w"> </span><span class="nn">jax</span>
|
||
<a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a><span class="kn">import</span><span class="w"> </span><span class="nn">jax.numpy</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">jnp</span>
|
||
<a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a><span class="kn">import</span><span class="w"> </span><span class="nn">matplotlib.pyplot</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">plt</span>
|
||
<a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a>
|
||
<a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a><span class="n">key</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">PRNGKey</span><span class="p">(</span><span class="mi">7</span><span class="p">)</span>
|
||
<a id="__codelineno-1-6" name="__codelineno-1-6" href="#__codelineno-1-6"></a>
|
||
<a id="__codelineno-1-7" name="__codelineno-1-7" href="#__codelineno-1-7"></a><span class="c1"># 创建一个合成的 8x8 "图像"，3 个通道</span>
|
||
<a id="__codelineno-1-8" name="__codelineno-1-8" href="#__codelineno-1-8"></a><span class="n">k1</span><span class="p">,</span> <span class="n">k2</span><span class="p">,</span> <span class="n">k3</span><span class="p">,</span> <span class="n">k4</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
|
||
<a id="__codelineno-1-9" name="__codelineno-1-9" href="#__codelineno-1-9"></a><span class="n">image</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="n">k1</span><span class="p">,</span> <span class="p">(</span><span class="mi">8</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>
|
||
<a id="__codelineno-1-10" name="__codelineno-1-10" href="#__codelineno-1-10"></a>
|
||
<a id="__codelineno-1-11" name="__codelineno-1-11" href="#__codelineno-1-11"></a><span class="c1"># 第 1 步：划分为 4x4 patch -> 4 个 patch</span>
|
||
<a id="__codelineno-1-12" name="__codelineno-1-12" href="#__codelineno-1-12"></a><span class="n">patch_size</span> <span class="o">=</span> <span class="mi">4</span>
|
||
<a id="__codelineno-1-13" name="__codelineno-1-13" href="#__codelineno-1-13"></a><span class="n">patches</span> <span class="o">=</span> <span class="n">image</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
|
||
<a id="__codelineno-1-14" name="__codelineno-1-14" href="#__codelineno-1-14"></a><span class="n">patches</span> <span class="o">=</span> <span class="n">patches</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="n">patch_size</span> <span class="o">*</span> <span class="n">patch_size</span> <span class="o">*</span> <span class="mi">3</span><span class="p">)</span> <span class="c1"># (4, 48)</span>
|
||
<a id="__codelineno-1-15" name="__codelineno-1-15" href="#__codelineno-1-15"></a><span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Patch 数量: </span><span class="si">{</span><span class="n">patches</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="si">}</span><span class="s2">, Patch 维度: </span><span class="si">{</span><span class="n">patches</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<a id="__codelineno-1-16" name="__codelineno-1-16" href="#__codelineno-1-16"></a>
|
||
<a id="__codelineno-1-17" name="__codelineno-1-17" href="#__codelineno-1-17"></a><span class="c1"># 第 2 步：将 patch 投影到嵌入维度 (d=16)</span>
|
||
<a id="__codelineno-1-18" name="__codelineno-1-18" href="#__codelineno-1-18"></a><span class="n">d_model</span> <span class="o">=</span> <span class="mi">16</span>
|
||
<a id="__codelineno-1-19" name="__codelineno-1-19" href="#__codelineno-1-19"></a><span class="n">W_patch</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">k2</span><span class="p">,</span> <span class="p">(</span><span class="n">patches</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">d_model</span><span class="p">))</span> <span class="o">*</span> <span class="mf">0.1</span>
|
||
<a id="__codelineno-1-20" name="__codelineno-1-20" href="#__codelineno-1-20"></a><span class="n">visual_tokens</span> <span class="o">=</span> <span class="n">patches</span> <span class="o">@</span> <span class="n">W_patch</span> <span class="c1"># (4, 16)</span>
|
||
<a id="__codelineno-1-21" name="__codelineno-1-21" href="#__codelineno-1-21"></a>
|
||
<a id="__codelineno-1-22" name="__codelineno-1-22" href="#__codelineno-1-22"></a><span class="c1"># 第 3 步：创建文本 token 嵌入（模拟 3 个文本 token）</span>
|
||
<a id="__codelineno-1-23" name="__codelineno-1-23" href="#__codelineno-1-23"></a><span class="n">text_tokens</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">k3</span><span class="p">,</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">d_model</span><span class="p">))</span> <span class="o">*</span> <span class="mf">0.1</span>
|
||
<a id="__codelineno-1-24" name="__codelineno-1-24" href="#__codelineno-1-24"></a>
|
||
<a id="__codelineno-1-25" name="__codelineno-1-25" href="#__codelineno-1-25"></a><span class="c1"># 第 4 步：拼接视觉 + 文本 token</span>
|
||
<a id="__codelineno-1-26" name="__codelineno-1-26" href="#__codelineno-1-26"></a><span class="n">combined</span> <span class="o">=</span> <span class="n">jnp</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">visual_tokens</span><span class="p">,</span> <span class="n">text_tokens</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span> <span class="c1"># (7, 16)</span>
|
||
<a id="__codelineno-1-27" name="__codelineno-1-27" href="#__codelineno-1-27"></a><span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"组合序列长度: </span><span class="si">{</span><span class="n">combined</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="si">}</span><span class="s2"> (4 个视觉 + 3 个文本)"</span><span class="p">)</span>
|
||
<a id="__codelineno-1-28" name="__codelineno-1-28" href="#__codelineno-1-28"></a>
|
||
<a id="__codelineno-1-29" name="__codelineno-1-29" href="#__codelineno-1-29"></a><span class="c1"># 第 5 步：在组合序列上运行单头自注意力</span>
|
||
<a id="__codelineno-1-30" name="__codelineno-1-30" href="#__codelineno-1-30"></a><span class="n">W_Q</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">k4</span><span class="p">,</span> <span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">d_model</span><span class="p">))</span> <span class="o">*</span> <span class="mf">0.1</span>
|
||
<a id="__codelineno-1-31" name="__codelineno-1-31" href="#__codelineno-1-31"></a><span class="n">k5</span><span class="p">,</span> <span class="n">k6</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">k4</span><span class="p">)</span>
|
||
<a id="__codelineno-1-32" name="__codelineno-1-32" href="#__codelineno-1-32"></a><span class="n">W_K</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">k5</span><span class="p">,</span> <span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">d_model</span><span class="p">))</span> <span class="o">*</span> <span class="mf">0.1</span>
|
||
<a id="__codelineno-1-33" name="__codelineno-1-33" href="#__codelineno-1-33"></a><span class="n">W_V</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">k6</span><span class="p">,</span> <span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">d_model</span><span class="p">))</span> <span class="o">*</span> <span class="mf">0.1</span>
|
||
<a id="__codelineno-1-34" name="__codelineno-1-34" href="#__codelineno-1-34"></a>
|
||
<a id="__codelineno-1-35" name="__codelineno-1-35" href="#__codelineno-1-35"></a><span class="n">Q</span> <span class="o">=</span> <span class="n">combined</span> <span class="o">@</span> <span class="n">W_Q</span>
|
||
<a id="__codelineno-1-36" name="__codelineno-1-36" href="#__codelineno-1-36"></a><span class="n">K</span> <span class="o">=</span> <span class="n">combined</span> <span class="o">@</span> <span class="n">W_K</span>
|
||
<a id="__codelineno-1-37" name="__codelineno-1-37" href="#__codelineno-1-37"></a><span class="n">V</span> <span class="o">=</span> <span class="n">combined</span> <span class="o">@</span> <span class="n">W_V</span>
|
||
<a id="__codelineno-1-38" name="__codelineno-1-38" href="#__codelineno-1-38"></a><span class="n">attn_scores</span> <span class="o">=</span> <span class="p">(</span><span class="n">Q</span> <span class="o">@</span> <span class="n">K</span><span class="o">.</span><span class="n">T</span><span class="p">)</span> <span class="o">/</span> <span class="n">jnp</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">d_model</span><span class="p">)</span>
|
||
<a id="__codelineno-1-39" name="__codelineno-1-39" href="#__codelineno-1-39"></a><span class="n">attn_weights</span> <span class="o">=</span> <span class="n">jax</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">attn_scores</span><span class="p">,</span> <span class="n">axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span> <span class="c1"># (7, 7)</span>
|
||
<a id="__codelineno-1-40" name="__codelineno-1-40" href="#__codelineno-1-40"></a>
|
||
<a id="__codelineno-1-41" name="__codelineno-1-41" href="#__codelineno-1-41"></a><span class="n">output</span> <span class="o">=</span> <span class="n">attn_weights</span> <span class="o">@</span> <span class="n">V</span> <span class="c1"># (7, 16)</span>
|
||
<a id="__codelineno-1-42" name="__codelineno-1-42" href="#__codelineno-1-42"></a>
|
||
<a id="__codelineno-1-43" name="__codelineno-1-43" href="#__codelineno-1-43"></a><span class="c1"># 可视化跨模态注意力模式</span>
|
||
<a id="__codelineno-1-44" name="__codelineno-1-44" href="#__codelineno-1-44"></a><span class="n">labels</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'V1'</span><span class="p">,</span> <span class="s1">'V2'</span><span class="p">,</span> <span class="s1">'V3'</span><span class="p">,</span> <span class="s1">'V4'</span><span class="p">,</span> <span class="s1">'T1'</span><span class="p">,</span> <span class="s1">'T2'</span><span class="p">,</span> <span class="s1">'T3'</span><span class="p">]</span>
|
||
<a id="__codelineno-1-45" name="__codelineno-1-45" href="#__codelineno-1-45"></a><span class="n">fig</span><span class="p">,</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">6</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span>
|
||
<a id="__codelineno-1-46" name="__codelineno-1-46" href="#__codelineno-1-46"></a><span class="n">im</span> <span class="o">=</span> <span class="n">ax</span><span class="o">.</span><span class="n">imshow</span><span class="p">(</span><span class="n">attn_weights</span><span class="p">,</span> <span class="n">cmap</span><span class="o">=</span><span class="s1">'Blues'</span><span class="p">)</span>
|
||
<a id="__codelineno-1-47" name="__codelineno-1-47" href="#__codelineno-1-47"></a><span class="n">ax</span><span class="o">.</span><span class="n">set_xticks</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">7</span><span class="p">));</span> <span class="n">ax</span><span class="o">.</span><span class="n">set_xticklabels</span><span class="p">(</span><span class="n">labels</span><span class="p">)</span>
|
||
<a id="__codelineno-1-48" name="__codelineno-1-48" href="#__codelineno-1-48"></a><span class="n">ax</span><span class="o">.</span><span class="n">set_yticks</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">7</span><span class="p">));</span> <span class="n">ax</span><span class="o">.</span><span class="n">set_yticklabels</span><span class="p">(</span><span class="n">labels</span><span class="p">)</span>
|
||
<a id="__codelineno-1-49" name="__codelineno-1-49" href="#__codelineno-1-49"></a><span class="n">ax</span><span class="o">.</span><span class="n">set_xlabel</span><span class="p">(</span><span class="s1">'键'</span><span class="p">);</span> <span class="n">ax</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s1">'查询'</span><span class="p">)</span>
|
||
<a id="__codelineno-1-50" name="__codelineno-1-50" href="#__codelineno-1-50"></a><span class="n">ax</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s1">'自注意力：视觉（V）和文本（T）Token'</span><span class="p">)</span>
|
||
<a id="__codelineno-1-51" name="__codelineno-1-51" href="#__codelineno-1-51"></a><span class="n">plt</span><span class="o">.</span><span class="n">colorbar</span><span class="p">(</span><span class="n">im</span><span class="p">,</span> <span class="n">ax</span><span class="o">=</span><span class="n">ax</span><span class="p">);</span> <span class="n">plt</span><span class="o">.</span><span class="n">tight_layout</span><span class="p">();</span> <span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
<a id="__codelineno-1-52" name="__codelineno-1-52" href="#__codelineno-1-52"></a><span class="c1"># 观察：文本 token 关注视觉 token（跨模态注意力）！</span>
|
||
</code></pre></div></p>
|
||
</li>
|
||
<li>
|
||
<p>实现用于视觉定位的坐标 token 化。给定一个边界框，将其转换为离散 token；给定离散 token，重构边界框。在不同槽位分辨率下可视化量化误差。
|
||
<div class="highlight"><pre><span></span><code><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="kn">import</span><span class="w"> </span><span class="nn">jax.numpy</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">jnp</span>
|
||
<a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a><span class="kn">import</span><span class="w"> </span><span class="nn">matplotlib.pyplot</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">plt</span>
|
||
<a id="__codelineno-2-3" name="__codelineno-2-3" href="#__codelineno-2-3"></a>
|
||
<a id="__codelineno-2-4" name="__codelineno-2-4" href="#__codelineno-2-4"></a><span class="k">def</span><span class="w"> </span><span class="nf">encode_bbox</span><span class="p">(</span><span class="n">bbox</span><span class="p">,</span> <span class="n">num_bins</span><span class="o">=</span><span class="mi">1000</span><span class="p">):</span>
|
||
<a id="__codelineno-2-5" name="__codelineno-2-5" href="#__codelineno-2-5"></a><span class="w"> </span><span class="sd">"""将连续的边界框 (x, y, w, h)（在 [0,1] 范围内）转换为离散 token。"""</span>
|
||
<a id="__codelineno-2-6" name="__codelineno-2-6" href="#__codelineno-2-6"></a> <span class="n">tokens</span> <span class="o">=</span> <span class="n">jnp</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="n">jnp</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">bbox</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="n">num_bins</span> <span class="o">-</span> <span class="mi">1</span><span class="p">))</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">jnp</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span>
|
||
<a id="__codelineno-2-7" name="__codelineno-2-7" href="#__codelineno-2-7"></a> <span class="k">return</span> <span class="n">tokens</span>
|
||
<a id="__codelineno-2-8" name="__codelineno-2-8" href="#__codelineno-2-8"></a>
|
||
<a id="__codelineno-2-9" name="__codelineno-2-9" href="#__codelineno-2-9"></a><span class="k">def</span><span class="w"> </span><span class="nf">decode_bbox</span><span class="p">(</span><span class="n">tokens</span><span class="p">,</span> <span class="n">num_bins</span><span class="o">=</span><span class="mi">1000</span><span class="p">):</span>
|
||
<a id="__codelineno-2-10" name="__codelineno-2-10" href="#__codelineno-2-10"></a><span class="w"> </span><span class="sd">"""将离散 token 转换回连续的边界框。"""</span>
|
||
<a id="__codelineno-2-11" name="__codelineno-2-11" href="#__codelineno-2-11"></a> <span class="k">return</span> <span class="n">tokens</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">jnp</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="n">num_bins</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
|
||
<a id="__codelineno-2-12" name="__codelineno-2-12" href="#__codelineno-2-12"></a>
|
||
<a id="__codelineno-2-13" name="__codelineno-2-13" href="#__codelineno-2-13"></a><span class="c1"># 真实边界框（归一化到 [0, 1]）</span>
|
||
<a id="__codelineno-2-14" name="__codelineno-2-14" href="#__codelineno-2-14"></a><span class="n">gt_bbox</span> <span class="o">=</span> <span class="n">jnp</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">0.123</span><span class="p">,</span> <span class="mf">0.456</span><span class="p">,</span> <span class="mf">0.333</span><span class="p">,</span> <span class="mf">0.222</span><span class="p">])</span>
|
||
<a id="__codelineno-2-15" name="__codelineno-2-15" href="#__codelineno-2-15"></a>
|
||
<a id="__codelineno-2-16" name="__codelineno-2-16" href="#__codelineno-2-16"></a><span class="c1"># 测试不同槽位分辨率下的量化</span>
|
||
<a id="__codelineno-2-17" name="__codelineno-2-17" href="#__codelineno-2-17"></a><span class="n">bin_sizes</span> <span class="o">=</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">500</span><span class="p">,</span> <span class="mi">1000</span><span class="p">]</span>
|
||
<a id="__codelineno-2-18" name="__codelineno-2-18" href="#__codelineno-2-18"></a><span class="n">errors</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<a id="__codelineno-2-19" name="__codelineno-2-19" href="#__codelineno-2-19"></a><span class="k">for</span> <span class="n">n_bins</span> <span class="ow">in</span> <span class="n">bin_sizes</span><span class="p">:</span>
|
||
<a id="__codelineno-2-20" name="__codelineno-2-20" href="#__codelineno-2-20"></a> <span class="n">tokens</span> <span class="o">=</span> <span class="n">encode_bbox</span><span class="p">(</span><span class="n">gt_bbox</span><span class="p">,</span> <span class="n">n_bins</span><span class="p">)</span>
|
||
<a id="__codelineno-2-21" name="__codelineno-2-21" href="#__codelineno-2-21"></a> <span class="n">reconstructed</span> <span class="o">=</span> <span class="n">decode_bbox</span><span class="p">(</span><span class="n">tokens</span><span class="p">,</span> <span class="n">n_bins</span><span class="p">)</span>
|
||
<a id="__codelineno-2-22" name="__codelineno-2-22" href="#__codelineno-2-22"></a> <span class="n">error</span> <span class="o">=</span> <span class="n">jnp</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">jnp</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="n">gt_bbox</span> <span class="o">-</span> <span class="n">reconstructed</span><span class="p">))</span>
|
||
<a id="__codelineno-2-23" name="__codelineno-2-23" href="#__codelineno-2-23"></a> <span class="n">errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">float</span><span class="p">(</span><span class="n">error</span><span class="p">))</span>
|
||
<a id="__codelineno-2-24" name="__codelineno-2-24" href="#__codelineno-2-24"></a> <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"槽位数=</span><span class="si">{</span><span class="n">n_bins</span><span class="si">:</span><span class="s2">>5d</span><span class="si">}</span><span class="s2"> | Token=</span><span class="si">{</span><span class="n">tokens</span><span class="si">}</span><span class="s2"> | "</span>
|
||
<a id="__codelineno-2-25" name="__codelineno-2-25" href="#__codelineno-2-25"></a> <span class="sa">f</span><span class="s2">"重构=</span><span class="si">{</span><span class="n">reconstructed</span><span class="si">}</span><span class="s2"> | 最大误差=</span><span class="si">{</span><span class="n">error</span><span class="si">:</span><span class="s2">.6f</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<a id="__codelineno-2-26" name="__codelineno-2-26" href="#__codelineno-2-26"></a>
|
||
<a id="__codelineno-2-27" name="__codelineno-2-27" href="#__codelineno-2-27"></a><span class="n">fig</span><span class="p">,</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">8</span><span class="p">,</span> <span class="mi">4</span><span class="p">))</span>
|
||
<a id="__codelineno-2-28" name="__codelineno-2-28" href="#__codelineno-2-28"></a><span class="n">ax</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">bin_sizes</span><span class="p">,</span> <span class="n">errors</span><span class="p">,</span> <span class="s1">'o-'</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s1">'#e74c3c'</span><span class="p">,</span> <span class="n">linewidth</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">markersize</span><span class="o">=</span><span class="mi">8</span><span class="p">)</span>
|
||
<a id="__codelineno-2-29" name="__codelineno-2-29" href="#__codelineno-2-29"></a><span class="n">ax</span><span class="o">.</span><span class="n">set_xlabel</span><span class="p">(</span><span class="s1">'槽位数'</span><span class="p">);</span> <span class="n">ax</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s1">'最大量化误差'</span><span class="p">)</span>
|
||
<a id="__codelineno-2-30" name="__codelineno-2-30" href="#__codelineno-2-30"></a><span class="n">ax</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s1">'边界框量化误差 vs 槽位分辨率'</span><span class="p">)</span>
|
||
<a id="__codelineno-2-31" name="__codelineno-2-31" href="#__codelineno-2-31"></a><span class="n">ax</span><span class="o">.</span><span class="n">set_xscale</span><span class="p">(</span><span class="s1">'log'</span><span class="p">);</span> <span class="n">ax</span><span class="o">.</span><span class="n">set_yscale</span><span class="p">(</span><span class="s1">'log'</span><span class="p">)</span>
|
||
<a id="__codelineno-2-32" name="__codelineno-2-32" href="#__codelineno-2-32"></a><span class="n">ax</span><span class="o">.</span><span class="n">grid</span><span class="p">(</span><span class="kc">True</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.3</span><span class="p">);</span> <span class="n">plt</span><span class="o">.</span><span class="n">tight_layout</span><span class="p">();</span> <span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
<a id="__codelineno-2-33" name="__codelineno-2-33" href="#__codelineno-2-33"></a><span class="c1"># 尝试：槽位非常少时（如 5）会发生什么？误差在何时是可接受的？</span>
|
||
</code></pre></div></p>
|
||
</li>
|
||
</ol>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</article>
|
||
</div>
|
||
|
||
|
||
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
|
||
</div>
|
||
|
||
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
|
||
回到页面顶部
|
||
</button>
|
||
|
||
</main>
|
||
|
||
<footer class="md-footer">
|
||
|
||
|
||
|
||
<nav class="md-footer__inner md-grid" aria-label="页脚" >
|
||
|
||
|
||
<a href="../01.%20multimodal%20representations/" class="md-footer__link md-footer__link--prev" aria-label="上一页: 多模态表征">
|
||
<div class="md-footer__button md-icon">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
|
||
</div>
|
||
<div class="md-footer__title">
|
||
<span class="md-footer__direction">
|
||
上一页
|
||
</span>
|
||
<div class="md-ellipsis">
|
||
多模态表征
|
||
</div>
|
||
</div>
|
||
</a>
|
||
|
||
|
||
|
||
<a href="../03.%20image%20and%20video%20tokenisation/" class="md-footer__link md-footer__link--next" aria-label="下一页: 图像与视频 Token 化">
|
||
<div class="md-footer__title">
|
||
<span class="md-footer__direction">
|
||
下一页
|
||
</span>
|
||
<div class="md-ellipsis">
|
||
图像与视频 Token 化
|
||
</div>
|
||
</div>
|
||
<div class="md-footer__button md-icon">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11z"/></svg>
|
||
</div>
|
||
</a>
|
||
|
||
</nav>
|
||
|
||
|
||
<div class="md-footer-meta md-typeset">
|
||
<div class="md-footer-meta__inner md-grid">
|
||
<div class="md-copyright">
|
||
|
||
|
||
Made with
|
||
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
|
||
Material for MkDocs
|
||
</a>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="md-social">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<a href="https://github.com/flykhan/maths-cs-ai-compendium-zh" target="_blank" rel="noopener" title="github.com" class="md-social__link">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M173.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6m-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3m44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9M252.8 8C114.1 8 8 113.3 8 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C436.2 457.8 504 362.9 504 252 504 113.3 391.5 8 252.8 8M105.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1m-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7m32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1m-11.4-14.7c-1.6 1-1.6 3.6 0 5.9s4.3 3.3 5.6 2.3c1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2"/></svg>
|
||
</a>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
<div class="md-dialog" data-md-component="dialog">
|
||
<div class="md-dialog__inner md-typeset"></div>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
<script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": ["navigation.tabs", "navigation.sections", "navigation.expand", "navigation.top", "navigation.footer", "search.suggest", "search.highlight", "content.code.copy", "toc.follow"], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "\u5df2\u590d\u5236", "clipboard.copy": "\u590d\u5236", "search.result.more.one": "\u5728\u8be5\u9875\u4e0a\u8fd8\u6709 1 \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.more.other": "\u5728\u8be5\u9875\u4e0a\u8fd8\u6709 # \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.none": "\u6ca1\u6709\u627e\u5230\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.one": "\u627e\u5230 1 \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.other": "# \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.placeholder": "\u952e\u5165\u4ee5\u5f00\u59cb\u641c\u7d22", "search.result.term.missing": "\u7f3a\u5c11", "select.version": "\u9009\u62e9\u5f53\u524d\u7248\u672c"}, "version": null}</script>
|
||
|
||
|
||
<script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>
|
||
|
||
<script src="../../javascripts/mathjax.js"></script>
|
||
|
||
<script src="https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
|
||
|
||
</body>
|
||
</html> |