Integer Quantization for Deep Learning Inference: Principle and Empirical Evaluation

CodiMD – Collaborative markdown notes @import url(https://fonts.googleapis.com/css?family=Source+Sans+Pro:400,400italic,600,600italic,300italic,300|Source+Serif+Pro|Source+Code+Pro:400,300,500&subset=latin,latin-ext);.markdown-body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica,Arial,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol;font-size:16px;line-height:1.5;word-wrap:break-word}.markdown-body:after,.markdown-body:before{display:table;content:””}.markdown-body:after{clear:both}.markdown-body>:first-child{margin-top:0!important}.markdown-body>:last-child{margin-bottom:0!important}.markdown-body a:not([href]){color:inherit;text-decoration:none}.markdown-body .absent{color:#c00}.markdown-body .anchor{float:left;padding-right:4px;margin-left:-20px;line-height:1}.markdown-body .anchor:focus{outline:none}.markdown-body blockquote,.markdown-body dl,.markdown-body ol,.markdown-body p,.markdown-body pre,.markdown-body table,.markdown-body ul{margin-top:0;margin-bottom:16px}.markdown-body hr{height:.25em;padding:0;margin:24px 0;background-color:#e7e7e7;border:0}.markdown-body blockquote{padding:0 1em;color:#777;border-left:.25em solid #ddd}.night .markdown-body blockquote{color:#bcbcbc}.markdown-body blockquote>:first-child{margin-top:0}.markdown-body blockquote>:last-child{margin-bottom:0}.markdown-body .loweralpha{list-style-type:lower-alpha}.markdown-body h1,.markdown-body h2,.markdown-body h3,.markdown-body h4,.markdown-body h5,.markdown-body h6{margin-top:24px;margin-bottom:16px;font-weight:600;line-height:1.25}.night .markdown-body h1,.night .markdown-body h2,.night .markdown-body h3,.night .markdown-body h4,.night .markdown-body h5,.night .markdown-body h6{color:#ddd}.markdown-body h1 .fa-link,.markdown-body h2 .fa-link,.markdown-body h3 .fa-link,.markdown-body h4 .fa-link,.markdown-body h5 .fa-link,.markdown-body h6 .fa-link{color:#000;vertical-align:middle;visibility:hidden;font-size:16px}.night .markdown-body h1 .fa-link,.night .markdown-body h2 .fa-link,.night .markdown-body h3 .fa-link,.night .markdown-body h4 .fa-link,.night .markdown-body h5 .fa-link,.night .markdown-body h6 .fa-link{color:#fff}.markdown-body h1:hover .anchor,.markdown-body h2:hover .anchor,.markdown-body h3:hover .anchor,.markdown-body h4:hover .anchor,.markdown-body h5:hover .anchor,.markdown-body h6:hover .anchor{text-decoration:none}.markdown-body h1:hover .anchor .fa-link,.markdown-body h2:hover .anchor .fa-link,.markdown-body h3:hover .anchor .fa-link,.markdown-body h4:hover .anchor .fa-link,.markdown-body h5:hover .anchor .fa-link,.markdown-body h6:hover .anchor .fa-link{visibility:visible}.markdown-body h1 code,.markdown-body h1 tt,.markdown-body h2 code,.markdown-body h2 tt,.markdown-body h3 code,.markdown-body h3 tt,.markdown-body h4 code,.markdown-body h4 tt,.markdown-body h5 code,.markdown-body h5 tt,.markdown-body h6 code,.markdown-body h6 tt{font-size:inherit}.markdown-body h1{font-size:2em}.markdown-body h1,.markdown-body h2{padding-bottom:.3em;border-bottom:1px solid #eee}.markdown-body h2{font-size:1.5em}.markdown-body h3{font-size:1.25em}.markdown-body h4{font-size:1em}.markdown-body h5{font-size:.875em}.markdown-body h6{font-size:.85em;color:#777}.markdown-body ol,.markdown-body ul{padding-left:2em}.markdown-body ol.no-list,.markdown-body ul.no-list{padding:0;list-style-type:none}.markdown-body ol ol,.markdown-body ol ul,.markdown-body ul ol,.markdown-body ul ul{margin-top:0;margin-bottom:0}.markdown-body li>p{margin-top:16px}.markdown-body li+li{margin-top:.25em}.markdown-body dl{padding:0}.markdown-body dl dt{padding:0;margin-top:16px;font-size:1em;font-style:italic;font-weight:700}.markdown-body dl dd{padding:0 16px;margin-bottom:16px}.markdown-body table{display:block;width:100%;overflow:auto;word-break:normal;word-break:keep-all}.markdown-body table th{font-weight:700}.markdown-body table td,.markdown-body table th{padding:6px 13px;border:1px solid #ddd}.markdown-body table tr{background-color:#fff;border-top:1px solid #ccc}.night .markdown-body table tr{background-color:#5f5f5f}.markdown-body table tr:nth-child(2n){background-color:#f8f8f8}.night .markdown-body table tr:nth-child(2n){background-color:#4f4f4f}.markdown-body img{max-width:100%;box-sizing:content-box;background-color:#fff}.markdown-body img[align=right]{padding-left:20px}.markdown-body img[align=left]{padding-right:20px}.markdown-body .emoji{max-width:none;vertical-align:text-top;background-color:transparent}.markdown-body span.frame{display:block;overflow:hidden}.markdown-body span.frame>span{display:block;float:left;width:auto;padding:7px;margin:13px 0 0;overflow:hidden;border:1px solid #ddd}.markdown-body span.frame span img{display:block;float:left}.markdown-body span.frame span span{display:block;padding:5px 0 0;clear:both;color:#333}.markdown-body span.align-center{display:block;overflow:hidden;clear:both}.markdown-body span.align-center>span{display:block;margin:13px auto 0;overflow:hidden;text-align:center}.markdown-body span.align-center span img{margin:0 auto;text-align:center}.markdown-body span.align-right{display:block;overflow:hidden;clear:both}.markdown-body span.align-right>span{display:block;margin:13px 0 0;overflow:hidden;text-align:right}.markdown-body span.align-right span img{margin:0;text-align:right}.markdown-body span.float-left{display:block;float:left;margin-right:13px;overflow:hidden}.markdown-body span.float-left span{margin:13px 0 0}.markdown-body span.float-right{display:block;float:right;margin-left:13px;overflow:hidden}.markdown-body span.float-right>span{display:block;margin:13px auto 0;overflow:hidden;text-align:right}.markdown-body code,.markdown-body tt{padding:.2em 0;margin:0;font-size:85%;background-color:rgba(0,0,0,.04);border-radius:3px}.night .markdown-body code,.night .markdown-body tt{color:#eee;background-color:hsla(0,0%,90.2%,.36)}.markdown-body code:after,.markdown-body code:before,.markdown-body tt:after,.markdown-body tt:before{letter-spacing:-.2em;content:”\A0″}.markdown-body code br,.markdown-body tt br{display:none}.markdown-body del code{text-decoration:inherit}.markdown-body pre{word-wrap:normal}.markdown-body pre>code{padding:0;margin:0;font-size:100%;word-break:normal;white-space:pre;background:transparent;border:0}.markdown-body .highlight{margin-bottom:16px}.markdown-body .highlight pre{margin-bottom:0;word-break:normal}.markdown-body .highlight pre,.markdown-body pre{padding:16px;overflow:auto;font-size:85%;line-height:1.45;background-color:#f7f7f7;border-radius:3px}.markdown-body pre code,.markdown-body pre tt{display:inline;max-width:auto;padding:0;margin:0;overflow:visible;line-height:inherit;word-wrap:normal;background-color:transparent;border:0}.markdown-body pre code:after,.markdown-body pre code:before,.markdown-body pre tt:after,.markdown-body pre tt:before{content:normal}.markdown-body .csv-data td,.markdown-body .csv-data th{padding:5px;overflow:hidden;font-size:12px;line-height:1;text-align:left;white-space:nowrap}.markdown-body .csv-data .blob-line-num{padding:10px 8px 9px;text-align:right;background:#fff;border:0}.markdown-body .csv-data tr{border-top:0}.markdown-body .csv-data th{font-weight:700;background:#f8f8f8;border-top:0}.markdown-body kbd{display:inline-block;padding:3px 5px;font-size:11px;line-height:10px;color:#555;vertical-align:middle;background-color:#fcfcfc;border:1px solid;border-color:#ccc #ccc #bbb;border-radius:3px;box-shadow:inset 0 -1px 0 #bbb}.news .alert .markdown-body blockquote{padding:0 0 0 40px;border:0}.activity-tab .news .alert .commits,.activity-tab .news .markdown-body blockquote{padding-left:0}.task-list-item{list-style-type:none}.task-list-item label{font-weight:400}.task-list-item.enabled label{cursor:pointer}.task-list-item+.task-list-item{margin-top:3px}.task-list-item-checkbox{float:left;margin:.31em 0 .2em -1.3em!important;vertical-align:middle;cursor:default!important}.markdown-body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Helvetica,Arial,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol;padding-top:40px;padding-bottom:40px;max-width:758px;overflow:visible!important}.markdown-body pre{border:inherit!important}.night .markdown-body pre{filter:invert(100%)}.markdown-body code{color:inherit!important}.markdown-body pre code .wrapper{display:-webkit-inline-flex;display:-moz-inline-flex;display:-ms-inline-flex;display:-o-inline-flex;display:inline-flex}.markdown-body pre code .gutter{float:left;overflow:hidden;-webkit-user-select:none;user-select:none}.markdown-body pre code .gutter.linenumber{text-align:right;position:relative;display:inline-block;cursor:default;z-index:4;padding:0 8px 0 0;min-width:20px;box-sizing:content-box;color:#afafaf!important;border-right:3px solid #6ce26c!important}.markdown-body pre code .gutter.linenumber>span:before{content:attr(data-linenumber)}.markdown-body pre code .code{float:left;margin:0 0 0 16px}.markdown-body .gist .line-numbers{border-left:none;border-top:none;border-bottom:none}.markdown-body .gist .line-data{border:none}.markdown-body .gist table{border-spacing:0;border-collapse:inherit!important}.night .markdown-body .gist table tr:nth-child(2n){background-color:#ddd}.markdown-body code[data-gist-id]{background:none;padding:0;filter:invert(100%)}.markdown-body code[data-gist-id]:after,.markdown-body code[data-gist-id]:before{content:””}.markdown-body code[data-gist-id] .blob-num{border:unset}.markdown-body code[data-gist-id] table{overflow:unset;margin-bottom:unset}.markdown-body code[data-gist-id] table tr{background:unset}.markdown-body[dir=rtl] pre{direction:ltr}.markdown-body[dir=rtl] code{direction:ltr;unicode-bidi:embed}.markdown-body .alert>p{margin-bottom:0}.markdown-body pre.abc,.markdown-body pre.flow-chart,.markdown-body pre.geo,.markdown-body pre.graphviz,.markdown-body pre.mermaid,.markdown-body pre.sequence-diagram,.markdown-body pre.vega{text-align:center;background-color:inherit;border-radius:0;white-space:inherit}.night .markdown-body pre.graphviz .graph>polygon{fill:#333}.night .markdown-body pre.mermaid .sectionTitle,.night .markdown-body pre.mermaid .titleText,.night .markdown-body pre.mermaid text{fill:#fff}.markdown-body pre.abc>code,.markdown-body pre.flow-chart>code,.markdown-body pre.graphviz>code,.markdown-body pre.mermaid>code,.markdown-body pre.sequence-diagram>code,.markdown-body pre.vega>code{text-align:left}.markdown-body pre.abc>svg,.markdown-body pre.flow-chart>svg,.markdown-body pre.graphviz>svg,.markdown-body pre.mermaid>svg,.markdown-body pre.sequence-diagram>svg,.markdown-body pre.vega>svg{max-width:100%;height:100%}.night .markdown-body .abc path{fill:#eee}.night .markdown-body .abc path.note_selected{fill:##4DD0E1}.night tspan{fill:#fefefe}.night pre rect{fill:transparent}.night pre.flow-chart path,.night pre.flow-chart rect{stroke:#fff}.markdown-body pre>code.wrap{white-space:pre-wrap;white-space:-moz-pre-wrap;white-space:-pre-wrap;white-space:-o-pre-wrap;word-wrap:break-word}.markdown-body .alert>p,.markdown-body .alert>ul{margin-bottom:0}.markdown-body summary{display:list-item}.markdown-body summary:focus{outline:none}.markdown-body details summary{cursor:pointer}.markdown-body details:not([open])>:not(summary){display:none}.markdown-body figure{margin:1em 40px}.markdown-body img{background-color:transparent}.vimeo,.youtube{cursor:pointer;display:table;text-align:center;background-position:50%;background-repeat:no-repeat;background-size:contain;background-color:#000;overflow:hidden}.vimeo,.youtube{position:relative;width:100%}.youtube{padding-bottom:56.25%}.vimeo img{width:100%;object-fit:contain;z-index:0}.youtube img{object-fit:cover;z-index:0}.vimeo iframe,.youtube iframe,.youtube img{width:100%;height:100%;position:absolute;top:0;left:0}.vimeo iframe,.youtube iframe{vertical-align:middle;z-index:1}.vimeo .icon,.youtube .icon{position:absolute;height:auto;width:auto;top:50%;left:50%;transform:translate(-50%,-50%);color:#fff;opacity:.3;-webkit-transition:opacity .2s;transition:opacity .2s;z-index:0}.vimeo:hover .icon,.youtube:hover .icon{opacity:.6;-webkit-transition:opacity .2s;transition:opacity .2s}.slideshare .inner,.speakerdeck .inner{position:relative;width:100%}.slideshare .inner iframe,.speakerdeck .inner iframe{position:absolute;top:0;bottom:0;left:0;right:0;width:100%;height:100%}.geo-map{width:100%;height:250px}.MJX_Assistive_MathML{display:none}.ui-infobar{position:relative;z-index:2;max-width:758px;margin-top:25px;margin-bottom:-25px;color:#777}.toc .invisable-node{list-style-type:none}.ui-toc{position:fixed;bottom:20px;z-index:10000}.ui-toc-label{opacity:.3;background-color:#ccc;border:none}.ui-toc-label,.ui-toc .open .ui-toc-label{-webkit-transition:opacity .2s;transition:opacity .2s}.ui-toc .open .ui-toc-label{opacity:1;color:#fff}.ui-toc-label:focus{opacity:.3;background-color:#ccc;color:#000}.ui-toc-label:hover{opacity:1;background-color:#ccc;-webkit-transition:opacity .2s;transition:opacity .2s}.ui-toc-dropdown{margin-top:23px;margin-bottom:20px;padding-left:10px;padding-right:10px;max-width:45vw;width:25vw;max-height:70vh;overflow:auto;text-align:inherit}.ui-toc-dropdown>.toc{max-height:calc(70vh – 100px);overflow:auto}.ui-toc-dropdown[dir=rtl] .nav{padding-right:0;letter-spacing:.0029em}.ui-toc-dropdown a{overflow:hidden;text-overflow:ellipsis;white-space:pre}.ui-toc-dropdown .nav>li>a{display:block;padding:4px 20px;font-size:13px;font-weight:500;color:#767676}.ui-toc-dropdown .nav>li:first-child:last-child>ul,.ui-toc-dropdown .toc.expand ul{display:block}.ui-toc-dropdown .nav>li>a:focus,.ui-toc-dropdown .nav>li>a:hover{padding-left:19px;color:#000;text-decoration:none;background-color:transparent;border-left:1px solid #000}.night .ui-toc-dropdown .nav>li>a:focus,.night .ui-toc-dropdown .nav>li>a:hover{color:#fff;border-left-color:#fff}.ui-toc-dropdown[dir=rtl] .nav>li>a:focus,.ui-toc-dropdown[dir=rtl] .nav>li>a:hover{padding-right:19px;border-left:none;border-right:1px solid #000}.ui-toc-dropdown .nav>.active:focus>a,.ui-toc-dropdown .nav>.active:hover>a,.ui-toc-dropdown .nav>.active>a{padding-left:18px;font-weight:700;color:#000;background-color:transparent;border-left:2px solid #000}.night .ui-toc-dropdown .nav>.active:focus>a,.night .ui-toc-dropdown .nav>.active:hover>a,.night .ui-toc-dropdown .nav>.active>a{color:#fff;border-left:2px solid #fff}.ui-toc-dropdown[dir=rtl] .nav>.active:focus>a,.ui-toc-dropdown[dir=rtl] .nav>.active:hover>a,.ui-toc-dropdown[dir=rtl] .nav>.active>a{padding-right:18px;border-left:none;border-right:2px solid #000}.ui-toc-dropdown .nav .nav{display:none;padding-bottom:10px}.ui-toc-dropdown .nav>.active>ul{display:block}.ui-toc-dropdown .nav .nav>li>a{padding-top:1px;padding-bottom:1px;padding-left:30px;font-size:12px;font-weight:400}.night .ui-toc-dropdown .nav>li>a{color:#aaa}.ui-toc-dropdown[dir=rtl] .nav .nav>li>a{padding-right:30px}.ui-toc-dropdown .nav .nav>li>ul>li>a{padding-top:1px;padding-bottom:1px;padding-left:40px;font-size:12px;font-weight:400}.ui-toc-dropdown[dir=rtl] .nav .nav>li>ul>li>a{padding-right:40px}.ui-toc-dropdown .nav .nav>li>a:focus,.ui-toc-dropdown .nav .nav>li>a:hover{padding-left:29px}.ui-toc-dropdown[dir=rtl] .nav .nav>li>a:focus,.ui-toc-dropdown[dir=rtl] .nav .nav>li>a:hover{padding-right:29px}.ui-toc-dropdown .nav .nav>li>ul>li>a:focus,.ui-toc-dropdown .nav .nav>li>ul>li>a:hover{padding-left:39px}.ui-toc-dropdown[dir=rtl] .nav .nav>li>ul>li>a:focus,.ui-toc-dropdown[dir=rtl] .nav .nav>li>ul>li>a:hover{padding-right:39px}.ui-toc-dropdown .nav .nav>.active:focus>a,.ui-toc-dropdown .nav .nav>.active:hover>a,.ui-toc-dropdown .nav .nav>.active>a{padding-left:28px;font-weight:500}.ui-toc-dropdown[dir=rtl] .nav .nav>.active:focus>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active:hover>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active>a{padding-right:28px}.ui-toc-dropdown .nav .nav>.active>.nav>.active:focus>a,.ui-toc-dropdown .nav .nav>.active>.nav>.active:hover>a,.ui-toc-dropdown .nav .nav>.active>.nav>.active>a{padding-left:38px;font-weight:500}.ui-toc-dropdown[dir=rtl] .nav .nav>.active>.nav>.active:focus>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active>.nav>.active:hover>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active>.nav>.active>a{padding-right:38px}.markdown-body[lang^=ja]{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Helvetica,Arial,Hiragino Kaku Gothic Pro,”\30D2\30E9\30AE\30CE\89D2\30B4 Pro W3″,Osaka,Meiryo,”\30E1\30A4\30EA\30AA”,MS Gothic,”\FF2D\FF33 \30B4\30B7\30C3\30AF”,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol}.ui-toc-dropdown[lang^=ja]{font-family:Source Sans Pro,Helvetica,Arial,Meiryo UI,MS PGothic,”\FF2D\FF33 \FF30\30B4\30B7\30C3\30AF”,sans-serif}.markdown-body[lang=zh-tw]{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Helvetica,Arial,PingFang TC,Microsoft JhengHei,”\5FAE\8EDF\6B63\9ED1″,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol}.ui-toc-dropdown[lang=zh-tw]{font-family:Source Sans Pro,Helvetica,Arial,Microsoft JhengHei UI,”\5FAE\8EDF\6B63\9ED1UI”,sans-serif}.markdown-body[lang=zh-cn]{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Helvetica,Arial,PingFang SC,Microsoft YaHei,”\5FAE\8F6F\96C5\9ED1″,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol}.ui-toc-dropdown[lang=zh-cn]{font-family:Source Sans Pro,Helvetica,Arial,Microsoft YaHei UI,”\5FAE\8F6F\96C5\9ED1UI”,sans-serif}.ui-affix-toc{position:fixed;top:0;max-width:15vw;max-height:70vh;overflow:auto}.back-to-top,.expand-toggle,.go-to-bottom{display:block;padding:4px 10px;margin-top:10px;margin-left:10px;font-size:12px;font-weight:500;color:#999}.back-to-top:focus,.back-to-top:hover,.expand-toggle:focus,.expand-toggle:hover,.go-to-bottom:focus,.go-to-bottom:hover{color:#563d7c;text-decoration:none}.back-to-top,.go-to-bottom{margin-top:0}.ui-user-icon{width:20px;height:20px;display:block;border-radius:3px;margin-top:2px;margin-bottom:2px;margin-right:5px;background-position:50%;background-repeat:no-repeat;background-size:contain}.ui-user-icon.small{width:18px;height:18px;display:inline-block;vertical-align:middle;margin:0 0 .2em}small span{line-height:22px}small .dropdown{display:inline-block}small .dropdown a:focus,small .dropdown a:hover{text-decoration:none}.unselectable{-moz-user-select:none;-khtml-user-select:none;-webkit-user-select:none;-o-user-select:none;user-select:none}.night .navbar{background:#333;border-bottom-color:#333;color:#eee}.night .navbar a{color:#eee}@media print{blockquote,div,img,pre,table{page-break-inside:avoid!important}a[href]:after{font-size:12px!important}}.markdown-body.slides{position:relative;z-index:1;color:#222}.markdown-body.slides:before{content:””;display:block;position:absolute;top:0;left:0;right:0;bottom:0;z-index:-1;background-color:currentColor;box-shadow:0 0 0 50vw}.markdown-body.slides section[data-markdown]{position:relative;margin-bottom:1.5em;background-color:#fff;text-align:center}.markdown-body.slides section[data-markdown] code{text-align:left}.markdown-body.slides section[data-markdown]:before{content:””;display:block;padding-bottom:56.23%}.markdown-body.slides section[data-markdown]>div:first-child{position:absolute;top:50%;left:1em;right:1em;transform:translateY(-50%);max-height:100%;overflow:hidden}.markdown-body.slides section[data-markdown]>ul{display:inline-block}.markdown-body.slides>section>section+section:after{content:””;position:absolute;top:-1.5em;right:1em;height:1.5em;border:3px solid #777}body{font-smoothing:subpixel-antialiased!important;-webkit-font-smoothing:subpixel-antialiased!important;-moz-osx-font-smoothing:auto!important;text-shadow:0 0 1em transparent,1px 1px 1.2px rgba(0,0,0,.004);-webkit-overflow-scrolling:touch;font-family:Source Sans Pro,Helvetica,Arial,sans-serif;letter-spacing:.025em}.focus,:focus{outline:none!important}::-moz-focus-inner{border:0!important}body.modal-open{overflow-y:auto;padding-right:0!important} https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js https://cdnjs.cloudflare.com/ajax/libs/respond.js/1.4.2/respond.min.js https://cdnjs.cloudflare.com/ajax/libs/es5-shim/4.5.9/es5-shim.min.js

論文連結。這是一篇介紹性的 paper,以下是閱讀時的摘要。

1 – Intro

目前在 DL 領域 32 位元浮點數是主要的數值表示方法。

問:浮點數與定點數的差別?
答:定點數會分別設定固定的位元來表示整數與小數點;浮點數則為不固定。

而在實際 model inference 時所使用數值並沒有那麼精準,會使用位元較少的數值表示。主要因為:

  • 可以提高 throughput
  • 降低 memory bandwidth 需求
  • 記憶體用量之後 cache 可以儲存比較多資訊,locality 也會變好

這篇論文主要對 quantization 還有 calibration 介紹其中基礎的數學,對 model 校準的訓練以及在 data set 上實際的校準結果。

Quantization 分為兩大類:

  • Post Training Quantization (PTQ)
  • Quantization Aware Training (QAT)

這篇主要介紹使用 quantization 來加速運算,因此使用 uniform quantization scheme。

3 – Quantization Fundementals

Uniform quantization scheme

Uniform quantization involves 2 steps.

  1. range of real number to be quantized, clamp outliers outside the range
  2. map the real values into representable range

Quantize / Dequantize

  • Quantize: real number to integer representation (fp32 to int8)
  • Dequantize: integer representation to real number (int32 to fp16)

3.1 Range mapping

  • Range of real value: <span class="MathJax" id="MathJax-Element-1-Frame" tabindex="0" data-mathml="[β,α]” role=”presentation” style=”position: relative;”>[𝛽,𝛼][β,α][\beta, \alpha]

  • bit-width of representation: <span class="MathJax" id="MathJax-Element-2-Frame" tabindex="0" data-mathml="b” role=”presentation” style=”position: relative;”>𝑏bb

    • signed integer: <span class="MathJax" id="MathJax-Element-3-Frame" tabindex="0" data-mathml="[−2b−1,2b−1−1]” role=”presentation” style=”position: relative;”>[2𝑏1,2𝑏11][−2b−1,2b−1−1][-2^{b-1}, 2^{b-1} – 1]
    • unsigned integer: <span class="MathJax" id="MathJax-Element-4-Frame" tabindex="0" data-mathml="[0,2b−1]” role=”presentation” style=”position: relative;”>[0,2𝑏1][0,2b−1][0, 2^b – 1]
  • Uniform transformation:

    • affine: <span class="MathJax" id="MathJax-Element-5-Frame" tabindex="0" data-mathml="f(x)=s⋅x+z” role=”presentation” style=”position: relative;”>𝑓(𝑥)=𝑠𝑥+𝑧f(x)=s⋅x+zf(x) = s \cdot x + z, <span class="MathJax" id="MathJax-Element-6-Frame" tabindex="0" data-mathml="s,x,z∈R” role=”presentation” style=”position: relative;”>𝑠,𝑥,𝑧𝑅s,x,z∈Rs, x, z \in R
    • scale: <span class="MathJax" id="MathJax-Element-7-Frame" tabindex="0" data-mathml="f(x)=s⋅x” role=”presentation” style=”position: relative;”>𝑓(𝑥)=𝑠𝑥f(x)=s⋅xf(x) = s \cdot x, <span class="MathJax" id="MathJax-Element-8-Frame" tabindex="0" data-mathml="s,x∈R” role=”presentation” style=”position: relative;”>𝑠,𝑥𝑅s,x∈Rs, x \in R

3.1.1 Affine quantization

Quantize

<span class="MathJax" id="MathJax-Element-9-Frame" tabindex="0" data-mathml="f(x)=s⋅x+z” role=”presentation” style=”position: relative;”>𝑓(𝑥)=𝑠𝑥+𝑧f(x)=s⋅x+zf(x) = s \cdot x + z, <span class="MathJax" id="MathJax-Element-10-Frame" tabindex="0" data-mathml="s,x,z∈R” role=”presentation” style=”position: relative;”>𝑠,𝑥,𝑧𝑅s,x,z∈Rs, x, z \in R

  • <span class="MathJax" id="MathJax-Element-11-Frame" tabindex="0" data-mathml="s=2b−1α−β” role=”presentation” style=”position: relative;”>𝑠=2𝑏1𝛼𝛽s=2b−1α−βs = \frac{2^b – 1}{\alpha – \beta}
  • <span class="MathJax" id="MathJax-Element-12-Frame" tabindex="0" data-mathml="z=−round(β⋅s)−2b−1″ role=”presentation” style=”position: relative;”>𝑧=𝑟𝑜𝑢𝑛𝑑(𝛽𝑠)2𝑏1z=−round(β⋅s)−2b−1z = -round(\beta \cdot s) – 2^{b-1}

可以想像是給定兩項條件:(1) real value range; (2) representable range 之後,把值直接「線性的」投射上去。

Dequantize

Dequantize 為原本 <span class="MathJax" id="MathJax-Element-13-Frame" tabindex="0" data-mathml="f(x)” role=”presentation” style=”position: relative;”>𝑓(𝑥)f(x)f(x) 的反函數:

<span class="MathJax" id="MathJax-Element-14-Frame" tabindex="0" data-mathml="f′(x)=1s(x−z)” role=”presentation” style=”position: relative;”>𝑓(𝑥)=1𝑠(𝑥𝑧)f′(x)=1s(x−z)f'(x) = \frac{1}{s}(x – z)

3.1.2 Scale quantization

Quantize

Scale quantize 是一種 affine quantization 的特例。也就是它以 0 為基準點,設定 real value range 為 <span class="MathJax" id="MathJax-Element-15-Frame" tabindex="0" data-mathml="[−α,α]” role=”presentation” style=”position: relative;”>[𝛼,𝛼][−α,α][-\alpha, \alpha]。

<span class="MathJax" id="MathJax-Element-16-Frame" tabindex="0" data-mathml="f(x)=round(s⋅x)” role=”presentation” style=”position: relative;”>𝑓(𝑥)=𝑟𝑜𝑢𝑛𝑑(𝑠𝑥)f(x)=round(s⋅x)f(x) = round(s \cdot x)

  • <span class="MathJax" id="MathJax-Element-17-Frame" tabindex="0" data-mathml="s=2b−1α” role=”presentation” style=”position: relative;”>𝑠=2𝑏1𝛼s=2b−1αs = \frac{2^{b-1}}{\alpha}
Dequantize

<span class="MathJax" id="MathJax-Element-18-Frame" tabindex="0" data-mathml="s” role=”presentation” style=”position: relative;”>𝑠ss 的倒數乘回去。

<span class="MathJax" id="MathJax-Element-19-Frame" tabindex="0" data-mathml="f′(x)=1sx” role=”presentation” style=”position: relative;”>𝑓(𝑥)=1𝑠𝑥f′(x)=1sxf'(x) = \frac{1}{s}x

小推導

Scale quantization 應該要是 affine quantization 的一個特例。
設定 real value range <span class="MathJax" id="MathJax-Element-20-Frame" tabindex="0" data-mathml="[−α,α]” role=”presentation” style=”position: relative;”>[𝛼,𝛼][−α,α][-\alpha, \alpha] 的話,

<span class="MathJax" id="MathJax-Element-21-Frame" tabindex="0" data-mathml="s=2b−1−0.5α” role=”presentation” style=”position: relative;”>𝑠=2𝑏10.5𝛼s=2b−1−0.5αs = \frac{2^{b-1} – 0.5}{\alpha}
<span class="MathJax" id="MathJax-Element-22-Frame" tabindex="0" data-mathml="z=round(2b−1−0.5)−2b−1″ role=”presentation” style=”position: relative;”>𝑧=𝑟𝑜𝑢𝑛𝑑(2𝑏10.5)2𝑏1z=round(2b−1−0.5)−2b−1z = round(2^{b-1} – 0.5) – 2^{b-1}

(以上的 scheme 為 ceil 或是 round to the nearest 都有可能)

3.2 Tensor quantization granularity

粒度也是 quantization 的重點之一。

最粗糙的就是 tensor 中每個 element 都共用同樣的 quantization parameter。而最細緻的就是每個 element 都擁有自己的 quantization parameter。

常見的為:

  • per column / per row(適用 2-D tensors, like 2D-CNN activations)
  • per channel(適用 3-D tensors, like image)

考量 quantization 效果因素:

  • accuracy
  • computation cost
Computation cost

Linear layer (fully-connected): <span class="MathJax" id="MathJax-Element-23-Frame" tabindex="0" data-mathml="Y=XW” role=”presentation” style=”position: relative;”>𝑌=𝑋𝑊Y=XWY = X W, where

  • <span class="MathJax" id="MathJax-Element-24-Frame" tabindex="0" data-mathml="X=(xik)∈Rm×p” role=”presentation” style=”position: relative;”>𝑋=(𝑥𝑖𝑘)𝑚×𝑝X=(xik)∈Rm×pX = (x_{ik}) \in \mathbb{R}^{m \times p} (input tensor)
  • <span class="MathJax" id="MathJax-Element-25-Frame" tabindex="0" data-mathml="W=(wkj)∈Rp×n” role=”presentation” style=”position: relative;”>𝑊=(𝑤𝑘𝑗)𝑝×𝑛W=(wkj)∈Rp×nW = (w_{kj}) \in \mathbb{R}^{p \times n} (weight tensor)
  • <span class="MathJax" id="MathJax-Element-26-Frame" tabindex="0" data-mathml="Y=(xij)∈Rm×n” role=”presentation” style=”position: relative;”>𝑌=(𝑥𝑖𝑗)𝑚×𝑛Y=(xij)∈Rm×nY = (x_{ij}) \in \mathbb{R}^{m \times n} (output tensor)

The quantized tensor can be expressed as…

  • <span class="MathJax" id="MathJax-Element-27-Frame" tabindex="0" data-mathml="Xq=(xq,ik)∈Zm×p” role=”presentation” style=”position: relative;”>𝑋𝑞=(𝑥𝑞,𝑖𝑘)𝑚×𝑝Xq=(xq,ik)∈Zm×pX_q = (x_{q, ik}) \in \mathbb{Z}^{m \times p} (input tensor)
  • <span class="MathJax" id="MathJax-Element-28-Frame" tabindex="0" data-mathml="Wq=(wq,kj)∈Zp×n” role=”presentation” style=”position: relative;”>𝑊𝑞=(𝑤𝑞,𝑘𝑗)𝑝×𝑛Wq=(wq,kj)∈Zp×nW_q = (w_{q, kj}) \in \mathbb{Z}^{p \times n} (weight tensor)

可以推斷 quantized Y 與原本 Y 的關係:

<span class="MathJax" id="MathJax-Element-29-Frame" tabindex="0" data-mathml="yij=Σk=1pxik⋅wkj” role=”presentation” style=”position: relative;”>𝑦𝑖𝑗=Σ𝑝𝑘=1𝑥𝑖𝑘𝑤𝑘𝑗yij=Σk=1pxik⋅wkjy_{ij} = \Sigma^p_{k=1} x_{ik} \cdot w_{kj}

  • <span class="MathJax" id="MathJax-Element-30-Frame" tabindex="0" data-mathml="=Σk=1pdequantize(xik)⋅dequantize(wkj)” role=”presentation” style=”position: relative;”>=Σ𝑝𝑘=1𝑑𝑒𝑞𝑢𝑎𝑛𝑡𝑖𝑧𝑒(𝑥𝑖𝑘)𝑑𝑒𝑞𝑢𝑎𝑛𝑡𝑖𝑧𝑒(𝑤𝑘𝑗)=Σk=1pdequantize(xik)⋅dequantize(wkj)= \Sigma^p_{k=1} dequantize(x_{ik}) \cdot dequantize(w_{kj})
  • <span class="MathJax" id="MathJax-Element-31-Frame" tabindex="0" data-mathml="=Σk=1p1sx,ikxq,ik⋅1sx,ikwq,kj” role=”presentation” style=”position: relative;”>=Σ𝑝𝑘=11𝑠𝑥,𝑖𝑘𝑥𝑞,𝑖𝑘1𝑠𝑥,𝑖𝑘𝑤𝑞,𝑘𝑗=Σk=1p1sx,ikxq,ik⋅1sx,ikwq,kj= \Sigma^p_{k=1} \frac{1}{s_{x, ik}} x_{q, ik}\cdot \frac{1}{s_{x, ik}} w_{q, kj}

如果把 scaling factor 乘數提取出來,使得 scale <span class="MathJax" id="MathJax-Element-32-Frame" tabindex="0" data-mathml="s” role=”presentation” style=”position: relative;”>𝑠ss independent of <span class="MathJax" id="MathJax-Element-33-Frame" tabindex="0" data-mathml="k” role=”presentation” style=”position: relative;”>𝑘kk,就可以繼續維持矩陣乘法的形式。這時 quantized 的 <span class="MathJax" id="MathJax-Element-34-Frame" tabindex="0" data-mathml="x,w” role=”presentation” style=”position: relative;”>𝑥,𝑤x,wx, w 就可以做矩陣乘法,硬體有辦法高效率的執行這種指令。

提取後:<span class="MathJax" id="MathJax-Element-35-Frame" tabindex="0" data-mathml="1sx,i⋅sw,jΣk=1p(xq,ik⋅wq,kj)” role=”presentation” style=”position: relative;”>1𝑠𝑥,𝑖𝑠𝑤,𝑗Σ𝑝𝑘=1(𝑥𝑞,𝑖𝑘𝑤𝑞,𝑘𝑗)1sx,i⋅sw,jΣk=1p(xq,ik⋅wq,kj)\frac{1}{s_{x, i} \cdot s_{w, j}} \Sigma^p_{k=1} (x_{q, ik} \cdot w_{q, kj})

以上這個推導表示出了:granularity 為 per row / per column 或更粗糙時,可以維持整數的矩陣乘法運算,也代表 inference 時不需要對內部 quantized elememt 做加工。

因此 “quantization per row 或更粗操” 可以保證計算效率。

3.3 Computational cost of affine quantization

Affine quantization 因為計算時需要對內部 activation data 做加工,因此是效率較差的 quantization 方法。不過當然也有更多參數可以控制。

當然也可以推導出原輸出與 quantized 輸出的關係。

<span class="MathJax" id="MathJax-Element-36-Frame" tabindex="0" data-mathml="yij=Σk=1pxik⋅wkj” role=”presentation” style=”position: relative;”>𝑦𝑖𝑗=Σ𝑝𝑘=1𝑥𝑖𝑘𝑤𝑘𝑗yij=Σk=1pxik⋅wkjy_{ij} = \Sigma^p_{k=1} x_{ik} \cdot w_{kj}

  • <span class="MathJax" id="MathJax-Element-37-Frame" tabindex="0" data-mathml="=Σk=1pdequantize(xik)⋅dequantize(wkj)” role=”presentation” style=”position: relative;”>=Σ𝑝𝑘=1𝑑𝑒𝑞𝑢𝑎𝑛𝑡𝑖𝑧𝑒(𝑥𝑖𝑘)𝑑𝑒𝑞𝑢𝑎𝑛𝑡𝑖𝑧𝑒(𝑤𝑘𝑗)=Σk=1pdequantize(xik)⋅dequantize(wkj)= \Sigma^p_{k=1} dequantize(x_{ik}) \cdot dequantize(w_{kj})
  • <span class="MathJax" id="MathJax-Element-38-Frame" tabindex="0" data-mathml="=Σk=1p1sx(xq,ik−zx)⋅1sw,j(wq,kj−zw,j)” role=”presentation” style=”position: relative;”>=Σ𝑝𝑘=11𝑠𝑥(𝑥𝑞,𝑖𝑘𝑧𝑥)1𝑠𝑤,𝑗(𝑤𝑞,𝑘𝑗𝑧𝑤,𝑗)=Σk=1p1sx(xq,ik−zx)⋅1sw,j(wq,kj−zw,j)= \Sigma^p_{k=1} \frac{1}{s_x} (x_{q, ik} – z_x) \cdot \frac{1}{s_{w, j}} (w_{q, kj} – z_{w, j})

可以繼續提取出常數部分。

提取後:<span class="MathJax" id="MathJax-Element-39-Frame" tabindex="0" data-mathml="1sxsw,j{Σk=1pxq,ikwq,kj−Σk=1p(wq,kjzx+zxzw,j)−Σk=1pxq,ikzw,j}” role=”presentation” style=”position: relative;”>1𝑠𝑥𝑠𝑤,𝑗{Σ𝑝𝑘=1𝑥𝑞,𝑖𝑘𝑤𝑞,𝑘𝑗Σ𝑝𝑘=1(𝑤𝑞,𝑘𝑗𝑧𝑥+𝑧𝑥𝑧𝑤,𝑗)Σ𝑝𝑘=1𝑥𝑞,𝑖𝑘𝑧𝑤,𝑗}1sxsw,j{Σk=1pxq,ikwq,kj−Σk=1p(wq,kjzx+zxzw,j)−Σk=1pxq,ikzw,j}\frac{1}{s_x s_{w, j}} \{ \Sigma^p_{k=1} x_{q, ik}w_{q, kj} – \Sigma^p_{k=1}(w_{q, kj}z_x + z_xz_{w, j}) – \Sigma^p_{k=1}x_{q, ik}z_{w, j} \}

可以看到分成三大部分:

  • integer dot product: <span class="MathJax" id="MathJax-Element-40-Frame" tabindex="0" data-mathml="Σk=1pxq,ikwq,kj” role=”presentation” style=”position: relative;”>Σ𝑝𝑘=1𝑥𝑞,𝑖𝑘𝑤𝑞,𝑘𝑗Σk=1pxq,ikwq,kj\Sigma^p_{k=1} x_{q, ik}w_{q, kj}
  • integer weight and zero point (<span class="MathJax" id="MathJax-Element-41-Frame" tabindex="0" data-mathml="z” role=”presentation” style=”position: relative;”>𝑧zz): <span class="MathJax" id="MathJax-Element-42-Frame" tabindex="0" data-mathml="Σk=1p(−wq,kjzx+zxzw,j)” role=”presentation” style=”position: relative;”>Σ𝑝𝑘=1(𝑤𝑞,𝑘𝑗𝑧𝑥+𝑧𝑥𝑧𝑤,𝑗)Σk=1p(−wq,kjzx+zxzw,j)\Sigma^p_{k=1}(-w_{q, kj}z_x + z_xz_{w, j})。其中這裡可以對 weight 做事情,有 bias 的 operator 也可以把 zero point 的 dot product fuse 進去。
  • involves into the input tensor, cannot be computed offline: <span class="MathJax" id="MathJax-Element-43-Frame" tabindex="0" data-mathml="Σk=1pxq,ikzw,j” role=”presentation” style=”position: relative;”>Σ𝑝𝑘=1𝑥𝑞,𝑖𝑘𝑧𝑤,𝑗Σk=1pxq,ikzw,j\Sigma^p_{k=1}x_{q, ik}z_{w, j}。這項 overhead 是因為在 weight tensor 做 affine quantization。如果對 weight 單純做 scale quantization 的話,就不會有這一項,是減少 overhead 的做法。因為有時候這項的 computation cost 甚至會使得硬體帶來的 integer dot product 優化的效能被這項正負相抵消了。

3.4 Calibration

Calibration is the process of choosing the real value range – <span class="MathJax" id="MathJax-Element-44-Frame" tabindex="0" data-mathml="α” role=”presentation” style=”position: relative;”>𝛼α\alpha and <span class="MathJax" id="MathJax-Element-45-Frame" tabindex="0" data-mathml="β” role=”presentation” style=”position: relative;”>𝛽β\beta, for model weights and activations.

三種常見 Calibration 方法:

  • Max:取極值,最淺顯易懂的作法。
  • Entropy:取 KL Divergence 並盡力取減少 information loss。這是理論上的最佳作法
  • Percentile:百分位式,像是只取分布中主要 99.99% 的 element 作為 <span class="MathJax" id="MathJax-Element-46-Frame" tabindex="0" data-mathml="α” role=”presentation” style=”position: relative;”>𝛼α\alpha, <span class="MathJax" id="MathJax-Element-47-Frame" tabindex="0" data-mathml="β” role=”presentation” style=”position: relative;”>𝛽β\beta range。這種做法法可以避免極值離主要分布太遠。

4 – Post training quantization

由上一章節可以看到。可以對 model 中的 weight 做 calibration。可以丟進一些 input data 來得到 calibration 結果。

Model 有分很多種:CNN feed-forward, RNN, Attention-based NN.

4.1 Weight quantization

列出實驗數據。

quantization 方法:max。

  • Per channel 比 per tensor 好。
  • BN folding 不影響 per channel calibration
  • BN folding 會影響 per tensor calibration

4.2 Activation quantization

列出實驗數據。

  • 大部分為 entropy 最佳
  • max 部分從來不是個好作法
  • Percentile 作法偶爾會贏 entropy。
  • mobilenet, efficientnet, bert 有 > 1% 的 accuracy loss

小結論:no single calibration is best for all networks.

5 – Accuracy recovery

當發現 calibration 真的有帶來 accuracy loss 時,可以使用方法來做 accuracy recovery。

5.1 Partial quantization

往往是因為某些 NN 層帶來 accuracy loss。一個繞過去的作法就是讓 CPU 來做這些造成失真的層。(leaving them unquantized)

如果想要測試 partial quantization 的組合,組合會 exponential 增長,所以用 single layer accuracy 的方法來比較的話,可以將所有 layer 排序,並逐層拔掉 quantization 直到有打到理想的 accuracy。(是個挺直白的 greedy heuristic)

而列出哪個 layer 會影響準度,叫做 sensitivity analysis。

5.2 Quantization aware training (QAT)

Insert quantization before training.

The intuition behind is when we train with quantization we may narrow the gradient descent to the optimal. Make model be aware of integer-ness, and find “wide and flat” minima.

常見作法為 fake quantization,又叫 simulated quantization。

fake quantize: <span class="MathJax" id="MathJax-Element-48-Frame" tabindex="0" data-mathml="x^=dquantize(quantize(x,b,s),b,s)” role=”presentation” style=”position: relative;”>𝑥̂ =𝑑𝑞𝑢𝑎𝑛𝑡𝑖𝑧𝑒(𝑞𝑢𝑎𝑛𝑡𝑖𝑧𝑒(𝑥,𝑏,𝑠),𝑏,𝑠)x^=dquantize(quantize(x,b,s),b,s)\hat{x} = dquantize(quantize(x, b, s), b, s)

對無法微分的的地方,使用 Straight Through Estimator (STE)。

STE:

  • 如果在 real value range 內,回傳 1
  • else,回傳 0

甚至有時候 QAT 會帶來更好的 model accuracy 因為 quantization 也有 regularizer 的效果。

5.3 Learning quantization parameters

It is also possible to learn quantization parameters along with the model weight.

PACT learns the range of activation for activation quantization during training.

Initialized with max calibration (意思是:「以 max calibration 參數為初始」嗎??)

  • learning the range (real value range) results in better accuracy

Initialized with best calibration range

  • yields similar result as initialized with max calibration

小結論:learning the range (QAT) doesn’t offer additional benefit if given carefully calibrated range.

不過文章中也有說 PACT 可能在其他地方更有用,這裡只是拿 PACT 來展現 QAT 是在做什麼的。

6 – Workflow

Pretrained Network –> PTQ –> Partial Quantization –> QAT(start from best calibration)

Self reflection

終於更了解 quantization / calibration 了!!!

https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.0/js/bootstrap.min.js https://cdnjs.cloudflare.com/ajax/libs/gist-embed/2.6.0/gist-embed.min.js var markdown = $(“.markdown-body”); //smooth all hash trigger scrolling function smoothHashScroll() { var hashElements = $(“a[href^=’#’]”).toArray(); for (var i = 0; i < hashElements.length; i++) { var element = hashElements[i]; var $element = $(element); var hash = element.hash; if (hash) { $element.on('click', function (e) { // store hash var hash = this.hash; if ($(hash).length = 133) { newbool = true; var affixLeftMargin = (tocAffix.outerWidth() – tocAffix.width()) / 2; var left = markdown.offset().left + markdown.outerWidth() – affixLeftMargin; tocAffix.css(‘left’, left + ‘px’); } else { newbool = false; } if (newbool != enoughForAffixToc) { enoughForAffixToc = newbool; generateScrollspy(); } } $(window).resize(function () { windowResize(); }); $(document).ready(function () { windowResize(); generateScrollspy(); }); //remove hash function removeHash() { window.location.hash = ”; } var backtotop = $(‘.back-to-top’); var gotobottom = $(‘.go-to-bottom’); backtotop.click(function (e) { e.preventDefault(); e.stopPropagation(); if (scrollToTop) scrollToTop(); removeHash(); }); gotobottom.click(function (e) { e.preventDefault(); e.stopPropagation(); if (scrollToBottom) scrollToBottom(); removeHash(); }); var toggle = $(‘.expand-toggle’); var tocExpand = false; checkExpandToggle(); toggle.click(function (e) { e.preventDefault(); e.stopPropagation(); tocExpand = !tocExpand; checkExpandToggle(); }) function checkExpandToggle () { var toc = $(‘.ui-toc-dropdown .toc’); var toggle = $(‘.expand-toggle’); if (!tocExpand) { toc.removeClass(‘expand’); toggle.text(‘Expand all’); } else { toc.addClass(‘expand’); toggle.text(‘Collapse all’); } } function scrollToTop() { $(‘body, html’).stop(true, true).animate({ scrollTop: 0 }, 100, “linear”); } function scrollToBottom() { $(‘body, html’).stop(true, true).animate({ scrollTop: $(document.body)[0].scrollHeight }, 100, “linear”); }

Author: eopXD

Hi 我是 eopXD ,希望人生過得有趣有挑戰。有任何問題都可以寄到 eopxdd[at]gmail.com。

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out /  Change )

Twitter picture

You are commenting using your Twitter account. Log Out /  Change )

Facebook photo

You are commenting using your Facebook account. Log Out /  Change )

Connecting to %s

%d bloggers like this: