From abe5caba6f71e0ce4fb145ebb3a7d36bed73d6e7 Mon Sep 17 00:00:00 2001
From: Varuna Jayasiri <vpjayasiri@gmail.com>
Date: Tue, 26 Jan 2021 16:54:23 +0530
Subject: [PATCH] =?UTF-8?q?=F0=9F=93=9A=20glu=20variants?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/optimizers/noam_lr.png                   | Bin 0 -> 35533 bytes
 docs/optimizers/radam_r_t.png                 | Bin 0 -> 30948 bytes
 docs/transformers/configs.html                | 568 +++++++-----
 docs/transformers/feed_forward.html           | 178 +++-
 .../transformers/glu_variants/experiment.html | 140 +--
 docs/transformers/glu_variants/simple.html    | 868 ++++++++++++++----
 labml_nn/transformers/configs.py              |  49 +-
 labml_nn/transformers/feed_forward.py         |  24 +-
 .../transformers/glu_variants/experiment.py   |   9 +-
 labml_nn/transformers/glu_variants/simple.py  | 106 ++-
 10 files changed, 1390 insertions(+), 552 deletions(-)
 create mode 100644 docs/optimizers/noam_lr.png
 create mode 100644 docs/optimizers/radam_r_t.png

diff --git a/docs/optimizers/noam_lr.png b/docs/optimizers/noam_lr.png
new file mode 100644
index 0000000000000000000000000000000000000000..b8945b93f0d030a5f6b86a339c5cecb0370e44fc
GIT binary patch
literal 35533
zcmdqJWmJ}3yDm(3iP9yVf;7_I4H8PHk|N!m(%o>=AOfODhlF%@cS(1{p6IjQ^}g#{
zdyM`2;}4AczMR*LGmhgp&uOT#q7*s`Aqo@}6uOMGxGEGBj1LqP^b!&b_@wTe+&3sF
zA1E1d5p@^6-FajWw9N;^4`1G9SkM2)c)+&9rO%i6r26KC?57-c6|XO<nw4eZSkCRQ
z+fzbf;cDST2KvdX=RPh@ZYXlq^F18wQyHI_*vnS3#~s{Mv6N3=4cKsfJN7tF$#Bgy
zdnSrR9zdwe{t^DqAEXE`!S6f-{1)(WG@$_$WdN3_g%5N%mMBs&^#3ovv>Qoa)J)~H
z#Kpmpz{L_J_Js~tRv1p%+}@VK9fOF#YCXr|U>wbn_~dnWeRtGJrmd~L=9Iu^zk${u
z`tgORG&0x~&W;kOL}{wPz(BT?aHcUtUZ25C0r$J>GyBbf7dkaq`--U?p>t&)+k!E(
zcpVvPamn-GVDTZT7U@ty@-OJ<o=<Wy^ZJaAj3je@!ea^I6on|i|6BrgU;PD|L4ZZc
zZzyI4hLQ1c5{`GupEbT*pKgD1+|kf#Q)kg{y6wQzci5Z3DIxQ|{R)0Y#EQX8`%~CE
z2L^0*r@nr9TP^5y%TpSVRb|+QfQpKW#O^XyVc2G6ZJiQ~6`Z|o{^8@t>$NzgB?1IA
z{10cpUz-^(ZEkX#_QZ2p&z$bhyFc9D9lFJnDHh?kNJ>gpY)pK4+n37VMd`wNx@<`p
z*A8|hjoUUT0~XVl#MHpxd4g<;6#3ZL<Mh>D!@Y#-*3|bLFVn4mxI6IP9L!p*u`~XJ
z`(k4-YtRa+_Oh3yS-3J&(39{-z^1RXaY?k6KO*M-NPh46-t1ro-{#hqG(C*VT33vK
z`*}bEzuiiEi!bc*)#&KxcCb3y5=1Q6>0rTgb0Fie&-hok>t=>C3@q%ng@?!O%|Y|S
zPeiia`TEJBp~Hv!gNLP*gg?vhez@Z<Rk?>#?y!X;BYbYUT&Sbza=P`Tv~O^5a%9AB
z5~>`_t8=|Ksf{s!-z*agQh@MyTqP7f<Yo(=7rI+MY?jZB4UF;d@UEHd)Ll3FQm~5p
zbnAMN1Pt;O)}YY@IzmE_u9+PS4bk&RJgbeM+UD_C^oE?()YKqk4W_-g*ZbhOpx<w<
zu8J-8cXj#jmvQ-siHVW>=jP@@zRmuye!I0R8p1ZKen<409}b<2a>4zev7+Lo8M?>J
zyM+c<E348G4J1x-;Hcy3IMDpI3;ZGJ9MsVq6-U0Xh#@Z|D4wXFot;5YdQW9T2BBdQ
z;un=kTlBr}s*Y15N%-!sHnUDYc0iGGS>b=9_SxTXUi6e=v3&Q=<9fTW=XJ=M1+3O|
z$P>zjhK8=aXX^KoII#G7gQeQv;+=<AJENA#xozf@7tkODu>b6f(EX(`#n*RfT-G_<
zADo=PdAij*?Ztm)Wo321S+Sq_{@ylI5t>V@+9u<eaB4S6-wTq&i14zV2)zCF=JsT8
z+jL6?ShG&8+wq#rdhpXNs3!M|GJpRzV81kq=~IpF7ung**R4zLBNz}~o<b6;O^ErN
z<XzbVQZfGcNEj3;uLIrb6A}g_WMp3padGjTi7!%-#IE~uuPiMswJ=Gh&2SVUs$QZc
zyHrf;oE6hyIE-8CVbJpYZR+rDwKmY{8s{V*Y;0_Ry-P(9cBHev-m_I_^)b9*X}*n}
z6T!fcH>3QQSsza7d(YQ9AM}*<2zcGPu;(w3Pu1F+j~~g5TS86lVR1ZdQJ;=t=j6;k
z(6}LbZ()HA&2KgJ(%Ra3ofC=E?w@nBGBSe6KiM2)EN7yl>+0?G4^%5u6yfCMP1R$;
z4s|=<)pqnyJtJ%(;<hPmlhNvdX2Sh+bGi+iVQ_6z)jS-4{50EJo@MlWcY3t6HPj4v
zA~iL2k0q7`?@OO2CG@-}UJrK{ckAnGYts3uTX14scbBVCg-Th1z>Vr0cb>0peaApX
zem`C^4h}m!JRA)T4FR1{TnApARj}9%eQ9cHY9NEp`*JzB&r}*KzXw?s1c9Zl7|Q;h
zGVzjg?l;X|0=wu_uZG@x9j_th*Py-cOJ;rX=KJN@q&^6Ns4TEvNf!(5QJ3^coFtEH
zewN{i)6vnP5+=9;JU0lFYynJzS+_2ngb#OwnU62cYxnNvqU!s1U@@c3-riH;_=vei
zfA#>Xt~tx|oS=bgtBaK1C3?S=hL*P3)dxEP8m(EbP%$HUv-FzV&pc>y6pU;p3v0Kw
zv9Uk()%bAfC)UE@`=wDOp+NkgJ4wwa78YQ4EW)^K7tp5ZW2C!aM2o(HXf;Mf7nK&1
znR$PIb1<09DpnkcWnl#-&52AY6&av+P;0+=zCS-(ZM%s3;tjv|efk^{`B=!~P}okp
zQxFJb_=okZ)833iCja!$pT2W2<YP79TaZvWTc9LCLLGyG#U~HIQiTNuizO<Jo~i*Q
zmp@Wd!xJMNAAuzr(wj{ke4d{2Jm3@frma4o!|U-M+vIjaR*MR7S`I@)z1q5>RKZ9*
zf&0oLpB1@I=Z3$-;4iSdv{m*7ZAu`Kx2y2K8)SWasU0TfjQyV%&9(WT6aRVh#bLHU
zCzuLkCZ^lO=xYrYC(6`#MO*iU58mFZ$Lu~XYG-HX;Bd+~7Ge<GW<$GxMEZRbYyr8t
znrblz6%`dfzYY~P&a-FH8Jy~i2DEKuK_nRU8Tj`8i2n@mOdQ6c>-glP8^%JqqRp_n
zZ9r46`=9q)gkcy^AS;Wh)y}#OWlowy5fKoa?M#+C?o5E7f`<}lqWyDj&SPr`QV~&5
zP*7FH`Av@|ZhLF%bZa<QA&qOaPze(i)mWB*fIu#Zd2a|J2990h%bSF>G`Vs-B)I+_
z%gHj56S@SK!yvHR(x%wiK?NVHPqGEdtD(5q*)4&cgP`v3=QmNRJqnI9KK_{z5guOB
zMt>Tapa;Iw`u=?V+aFd0SXjbIz!jeBzj*QD?)DaZ&27*EWu`v}a019g`}_Nfii#jj
z=~bE_H<HHk$aI5*s!^&XlEl#F&y7Z&C&>^8#mSLSpMkjN+6e#h)Q#I}N|*&4wqBzf
z3BOClEzKA}6d-N1&1JkaY_)B^lccX}9=Z_txQ5iIM};kdH$67?EYYyAjrsiaG*9lw
zj~@pOCyJt?qSN;A$7g4O{dx^9BBiv4OXy-q#W6ip%6KbuOJGB`&p?I+7{ekMgT%m8
zLj(7lr@H!B$?MP87YB<43TYrY{Qmu0gTE>ns-K6y8>}pa^RT{Lzxn>|V$u7c@dB4g
zD>^;BFW$(=sOkQC=dVSdflSIwPj4t285x-?6&0j7lP%(3rc>KmT2B6Vk#}$}8$JzH
zRaI0NOfcy;5%X5crgeFq4oh`Lk(uwe&X(ur!=uK;#8i-ZdUzy{#M7$*Z~=#!#H{N#
z!10ZALFH29Vt?LvVu;ITj>n`6%g9uM-=z&;&pbBdF?bl4%fsb9m6q?{Iot#|z9Hg-
zKm}p+fv{|3Ym2E+mXPqe+GhTT)pV;m<MH94&-WmWH?Mpqzr00!aTQ0a1d<rYAeY;P
zStC2t*r5dl8{q`(+0@lvzg``-VLI<li6rSYd(8uc(tLl7>NLMSS_na}{`So{vw}u_
z-wAlY+0&LG>Ohj^-@kiGr|TRUPeT*9Vl1aC*Fkbht~Vb@_kDnlUm4=y=VxV&p?^et
zb-fkYg@sa7j2=JOxVSX*U=L)$S}{m?^BQ7R)Yap4gE9sDM{V&xKTD9tn1y$jFpq{0
zaFI2|E<%Mn1o5@I$~1da2JYqZ;c^=+0^0M*#P0t7@u8s)A3pdDfZRe6gOqy2=e$4U
z?C<ZtELBrm`;#qTzy6@9*C7N1-r29lQgy+N*0Wrf*}~<B090navZt}Heq0aQlIm}C
z1D631#OTP7RWy!tUMMyjmVWGiQMV|Bxt#d+bVKW@D8D3?uRKjJ3-kNzw4UXxc#%~E
zfnES=MI%A9=AfaGtbJn*bud$92_R;l?v&Qm(&6!X@52RvTz(&+#5$8)2AVu?Nco)5
za2*^R;)T$K%kh4ATpd$0d7n?4_jqpgCb5Wdu{&GBz}7JQ7Jw75p7{>kK*KeHZ5Nt5
zol{d_8;a{YCb=cj(UTZBthm!GVX?WT?%szF*bcn?{r$i#cXxNcS5=Xt+JF3*Y%vZ^
zOiGGD@3PzqZ`KbSA*En^Y)oBEE$*H(0rnEu`Ik3UMx-!;S8GqVmRehZBR5_DW;FBk
z@)Epta0#qH6L|0G=~-Z6ZH=Dq=Lch^Q1#O1GxXhj1QGX0uWIpk=Ys_XrPfqFXJ&i6
z`%b#70MmExq|1XzWYYuI*ViSIcK7x|<0_WyaZ6nfm!Q8DpaAoo5~E0nn6p%>ekSHT
zOBH5uv>AXU^_9UVL1aiY8zh-#0Ha}e1O>f@vPC{?meZi30Q^2*=jg9fcwhv@o$iKe
zz|PJtt?=~e(<ucI+5&ADJ>^}dD@{QTDy#;P`u%VYJe>814;8}CpEHa31#4+*U)|gc
zW(j41ObmjiC}e1QnqWbCgN<qigf{oH-_#)0Xleu@+Ty!kc7|q0;Fnom>=Br2O&RUW
z+4y1|RQ$%`Fr2HglQ))oP1-W#>l^g!jhT^&R8U|bJb#5NfmV<r29q%Rr{L@zO%T|4
zJg+$#2oA51+}c1;Je9pX&GT?AtF66?CKoeETW$;1Bo4`Szu1pnd5qoY=%$ac8>?<_
zwmSI2YP~0+PnbA9=kq*3BpH0pY?dXz{JFyg26%$R_RF}APDA_s0`F(Z5NJZWBCuBQ
z5@cEshF&^tkF2kC$Mq3{yzi+mz4%Cb!W88|K=#4zI?M9DPuXW0rZs#FXQzqe<mCN5
zSpgkr4<GJtUEh8E6c$F;A)}R=nwmeTs4|qfDx>xN8qMH*^(WCUQaR=tk^R&f@t?<J
z)q1VRBcE)l?)oCJs*B`d9Kojdels7;$;r{t(J|{wK3WbY11YNYIxmlkj4TT{7?)aj
zBvkKE1mxK^pUa`a)^ZDs{So$wRnH<cqnaR`0zEjO$O*6wjSUUoakkD*O{Mic&l1YZ
z%QG@`C(Cq0d!v8Y%o7q5du;R(G@yk79{M{cTPx@5D1s<A#cMBN3NFg5rW=j`;(Ii5
zL6sD^-<2*$E5IXvSd5nd>#QJt@<gP(++`cl(~bRBt`xEF8z-YTE{8wTl4yr8@z89L
z{2d4o@s#|$yZ~5EThIOojDk5__Q4M^9xGBgR7J!fE;=q$S6H{%6mgFPZ(nhJ$I!kf
ze%C3Vq`7#FQ<`G7MVDA~Z-h)vs+<B>1Sy`|`vW#qbhqeN&TaFbmW#u$u-=gV?D@n(
zk-#M$qNz>;ZSju5nePu0QOEnH*x*xIbsDOv=Q>N7D9FM~O_WC4uJqrz5@3*$s_%0Y
z*Sr#1%6cpbJsSkwEHc+8=V`33u~wcY$oLSZ_RJ6bffM6|4j$yDkf-vvbUoZ$pC6oD
zxmKZuLz4dUotEBdH=^yowVNqWQD*lqMb38OSg2GRsF05$dZ>ia|Fu{PDVjKiF>kad
zH~i{e9flIywGxzYP8B~LtKU@yVRaVE!;B66%?0=^MNSl|_T)O8PBYJ*Y4XMn;A~D-
zL`;z0jHQ3n$EIv|{Ll4}JWZlQnxw{C^B;U&AWo5CFYA?hN>;r_`IDlZ=szFL7^gX=
zM;hb_iVHlxT@{CYW^VjyWlAeV>Oa?nI#fkc#`xeh$r71vl%Lah;cw6@gP?S5$^My8
z)T)5(UkJ+>D)-G{*np2bN$q44jXBRIPJ)NqA+lAue)rV~l3nB@Y%uEE1aNs|2Lufv
z$KPWhLvb()5vl$!paB;TkNveNcBlzR)P0FJ=X=Ho8TsaAJHxq%@dZj*q`0`h_!%fE
zjkibg5AFFJf5Qn$yKe9b2{m`e(Sr0`2UrIHB}NmI(f>NaR=Z$jq_H#ZhHg=6(q`G9
zJ;eQrUnwY0gdSba7m8HSgNMCb<;@$y#{0{o)yK5|qpIp|JxL!#TS`jGFQuh`J^TQ4
zx}Zc}N$J2JQxE_Ja+7lqOlGQW2jv~!85lraS-gLboWctjveEu|45d_EeLdf+SLmet
z$tm?DsdFGMKAes!`TF_-fBxeXNDjq8|2k{jCT=HR;YR9ka?}L>??J&H_A|wBwC{!I
z3jY8Te3&Smx034~Ej<`yf@nuFWv`Rj%`qGdQ|eK1nS3tup3UoT-H^We81TsFvei&i
zP<kugE_J%Pxs6aG-8e6}(vp)yCq_g>)H`gqp8j~a+9WIZrYH6<pHX`iXtdQUn)=IZ
zEv~93-vrNsff<#{i7A5-oqV4nPM9V)?1b>Q#{BLKR?FT@)z0)c+O@5bd^uyI9#1x)
zDLC)Va60X3ZP>8qZzDkv=K#_h8dBCe2N1I9800nsg1Ui<l&Hb9uV#hvDV6U>X|!q_
zwpD?gl5z&9BmlmJwD@`eTHRiqq>O;*wJgKN&Yr?KPT^b+^VhA_Fhq`5yVWp+s8PpR
zFXEL(vcHbfFue*cC2yxU!4{TB;pG}b&uoz8;^Nxa*zof5a&mH-nVIptIq&awJ)zZb
zgbEH01}N;PH4wFra4TWNRi?YsF-cSir>+7}Ngr>ZV{D%>GczBihlYms<>E5y)Vv#}
zkd2uZ3qmuT@CUd}bP*4?>Grs{%5suz2f+I!83XR7>F>O2HkwO+Vwr`k0S?lux|2*H
z?V?C|nzybUu$bbMq|f7ozd)Cw`QgHxEqXtgPN+X=IXXJ>M?inm=*Fo{XKZ0HQ>a9C
zIk~v_0FVv<!A+r|$bry0i!Xgt6cmK6(L~37J@hZ?^;3tUY#-4_WNO$$vx-_t()hl?
z6nXk9f~eZ_#%aUG6goXRIvWD{YBkNawSi5IkXIfP^Tgg62o0yzKU|RNybPgo31yz)
z2zy{m-7Ckl&{%TXPx4`88XhX{EK$upgnR}OfC%teko*E)X{MA=`9Ka(w{KRXgk-4_
zFsM<pd*z(<-d|r|AJRRl8$d~N6ZoFrE~BNVr{~bUu#m}vEXTxb%?o&{FRkG)n1002
z;_3kVM;cU41n;_2x;OTUjaaL`@7fg@;)Hd<c^~gie=91Y1LTN+-OK|B0L8_}^s2G3
zvDtBMM=KKV_5lHb?j;~N-WDPEM8qWJ_q=w{GGNv-%2c@GTf$+~0Lo!n6B@EcE4%IN
zj|gT0e0*W0$)28`#YM8~s&-Rpz$0)`0tW^Lu%>{vKoL^}V8Re41{xX%AZmcd=>>_m
zK}m8}cArw4Y(A;Ss#GSTpALNEzPs!iInz9T@;s0EajLj1dUGH#0~RQv)%sCfd^-@K
z9g3vCN6|LOJM_%Q^@o|Pxya_)n$<!>t?gpdsW36nrHq&<au8R1gaeycOkQ~waG)EJ
zih`p$C*gC_sr{JW<IA&@FB-tUV=4V#$*=^``=jxNkl&I+gfy*XmBS*puVDP%6Y2-)
zzX0LivV&VW_BV3Q2it0E6`9wIMB_tQ)t*b!f&!P~>dKwvaZgkMkomVf_WGdpL^`Ao
zCn?YbqWJNXtRUa%v&;{1^oklGJd*#-U;%9(f9oZK<m!4PL8V>9c#tgbru@a7Wwnb?
zl<7ZXz6IbELHj>^D@qr8gB1la{5OfqEb-#>KKZ_QqRR50FK?r!K#%>n>xF10Sm{O{
z)8oa%m^EYA6k~6Ptd|f_Bl4dU;IYC(W38=)ZYq!H?JErKoVj=&0^D5oKhxN+hEc`n
z80QAUwcZ)2Fl5ptGonK?VW<Etx&IjLb$1WTezPuVv`kKrh23eX{?8Ljl--<BOV(3<
zQ47d_q_GmM8(LA`0iJ81u9JNguWQk|H$6;OV?pUS`;fsZiZ1n^BTw56lI5rv;7H^C
zew|ZfFs65fl~B)QAuvfU8X@zak3yvSTy7asn8vs}qn}-CaqAI;Pssxs_Nl4qzQ%49
zq>)7sRjy7pg991}j7apBug`WsthfWJmhDeJ`0nnttPgH(ZU+mEX$e`(+ra1Fym<rC
z$lh>nXHNp7AyK|md{0-`6c`I&VxVfB3`>zc))6RTLW+O2w|m^3O&+2h#y*Npv9Z_%
zYg=19Zl_zop3w<8LI@mE%}h+jK-31JdlZ?F9-TVtQq+It&({nz0%X{BLl`@AIo#e7
z=$l2JNVhfQ^;Wm53%3H4?F^3&Vm6SDEQ5=WzqPZ2`gIa4r=elwDLohtar<(u`zZ@c
zdFWGGFChD_lgWx00Pgtql}|=S#-iuNJmAhvUUxIy-R}YY1|R5NA$G&V!3pOqHhT+E
zP{3sBi;1lQ8dE5T9V)J-X8$XkKu}N+#CbBW%VnT4Z(;MB18)N|0?U*;!1y-62QxBA
zbU_C3S8wvT`lzft1c(@~^L~14tY{Mczp`F&+CFP^h)}vCIVw8fP+uBWpoLW4KG?mn
zd7>>D0{<8c^9l+?SI=*+>}6T+^m`Lu9L&{<CHeXLLqL@LWd(lt9e=BbD6K;Yeq8GU
z6@wIO#SC6}T@w(E`}+II1-CPq1DM5mMrQ@x&#gKmNw(j|1FZugbb#ODXr<$D+gC40
z?;r@BXv?%%eFl&-p!Tr69r+h5P;Fj3^yA@Q#*)M(3@;MC?RX+}2|*pKT}l5L5*R`i
z-EDHUa*I$mZn2eic=+MnT(&z(uXrWQ7BLy0Q@CDGq*)ZH0FYRSNk~FTU$z5O!+EQN
zG)%8nAnYB_Jp5KtLSooCLASx>iCf)iWn|<4kOWe&Pu-SPSoC{>u`YMN4FdJDFFv~&
zbs7y7YjB9@2T)#Q0jJL5wk5Ss3WC;Yw1`?&3yYEy3&sev=HTR9&M=!8wU=-8{sfd4
z`7A*Jrd@^HteBXu4%;ID^*Q_=12XXI9`Ho&l3H3&5P}Ds|DXaO1qQMVM@#6I97AOJ
zoBL;B15}pi^!(K3HDw+S7$kD8YJOOsv1FiFi{Ix=3v;_19}6|84%wx%n-Ap4#BJ4c
z+b#sqsJQwo30}cnXIChztLNqCJKkM8mN01~*b~ldf-+C_yJ46K?~{IRBfD)6)1geU
zdJ;0SK`R+KIXVA#i?#N&o_H`fpUaz!O0KT14*7>(1ew0e2dE$go8w-YfsRgETG|I*
zQ&Y3xJE)3DV{I^_!`2XTzKT3pS%D#QdIV<m9|34ReE;db0pZ_ty`|MbA86QQPXWsb
zrc3p8z+l(bI9v{wsO1u0DqVJxdAE3@GBPq=76FETwlz#ykcW-AP$9WWlt$gmuQ^$k
zH|(B+VVY~7?Ajj9888->0o`CT&%1To-jR^mq9p+Z(A*<4EYQS;MpJMGt_AT7lAHo%
zx%QAJL(7Jl5Q=xw<n$0vvyX(ZH9vm8@Ui?z>^+<#amX4`%V*dcu;_IYq3U1)QoX0P
zTcb3UA9H1KF+Ee8VzBNmC{HGV65gWgCW*`77Pg0~n&46S8z4FmMhuUB2x-wTPu#zf
zlvElAXzUjygMTS`o*)ybB%#}833_HfZ7nTc!QyautY*z`pj1Gm{q!p<V~+O%g(2X)
zzOG0PF4?Kc$z7!?iE(kcU%m){^uDg#E;dy4S_oRFE#neIF%luK^jg$|y2M0WCUHs0
zR<F~eBSTrW0{Q9h@1HX1)HxvYj`m%Hp!P@G^Ir#+h)%%PJ~UL}{cwM{ZF}~6oPc1Q
zW8!hr{ZX6q&tb~zBx3?4$rE}ebd$Vc1uV)v)d?e`1E)kEbulWAJeE_Z_~vi9J&LW@
z0E}(b56QjPaKhFAwH{#tX_2}M8L}@r6$a3!G0C*PXUh5Rd~okXX%*9XKq%N+WhgIH
zYBh@V%oOq_RsB&U?IlWm+RVwxnerluNjt{_djiC6P=#tZ1B}3E^Di^hdt7WA(xLat
zW6^US7`4OBM1(4FDvh88koltT%Yn*=NZ(tEs9W#E$;A}}za=0hMNl`)qSx@j!GW05
zVhm5;1JXc^zHq<G1{50|?>0gyk*{!WS<&7x#kNg$wkC&!TZTo=JS)iwEJ88Z<W*{b
z+8=|c>im0bd-|KXj428V%GL-VI@yz;^rObY!eSkGjF~`*A)*hkio_?}p*qp1uIXvl
zN7;s`?3U!^_pG7ck@&1ImFZWt*r6V1eAjGw<xST;larGJ0~j+FD{NF<flEsU@2-I8
zkWJ0a%IZr3;`Ey>yVngC+Uqu*P|W#>P<r3Y(D@lSIO6pV_V&=78A)SsA0_tHQ=$o;
z%)o+?B-ru8=UT8b-SoGoOHW;>KMmBb#0Q3;tJC<_VZ9(SK&YWpM{6I>m%|HbJZQSX
zV5^*?#9_z`04%fEuq~(xBuVqZOrg8er}h&G$J#@uY`(8Vc>jtMhiH!}B=N=D5MQ3`
zv`76K2-n!KcIlu_EXXX3tlBm|uUjh)3JI(}a>eUd9Q?$B<*d|~TZl@kNBm5?(33xX
zgGI31-yU4X21??^s=KM40vI-S^B#)4B=#@y?}YNorRh9^r~w&OXQK>A?8h=%K|w*M
zZc~tdL(Z>|Q4=B0fVPt4;pv%1?-=rW<tNwyGg5Di6!Z1ZD?3@g5a*A4`AV11(%t4<
zG^r@_V*guO(6fcsnYOz%?M<{^Y|2>p)zJZeU5fSn=g*%ozCJZ*$fQQ5rt1d>Lf-ct
zD{Ohgxl*WsqUPr2=;-J`vkSRmHU1^6e+6n5(L+yZ<S&nVS+-S8L3M-EZ!DeSnjza^
z0!_+s{za;n%tq!{6Lg3qkjwv7UwI3qmO$JIiUGetr8C(Yq`hD?0+?AeGQoxwVPa6c
z41NY;LET3wE=o$+f5*SaKrX`31CA8G<lB6EjG76r%fjTjLjKz<%S8ltOje)O`lUl4
zk)at<07LG-z0e%$8`)!ktmv4-%fWz&Vk$~dXSg}}f!R<2skd&(`FxcZs0<6FtAEpu
zUa_TE`7IcC)#4czn<&h;W77+~^#@|g&>C@NxN;kChJSx}nZ7pq17W=tO2;wk%~V0Q
z)-ez-T>_t;k<pqY=pBbB=R)FqYUgk{F{YOBThcrIxl19sc3$X5KTcQDQhKLo*vnn6
zhs_Tu3JG{CH#u0x#HFdQrC}1-$8&>V`G}qP1m+O`ma-Vkt3q$_%fA|Lcj)fIdFpRL
zkD7;t9?K#6Nfp^q(a?+#pv_l`j<p!jj&8rTQ@8k2C)+~*DlQx4_5_QC!jkD}N-CdI
zGi<|{tK`6|z~rArM_ndty#6Y=z9f2fK&*Z&tZktW7KM9-U18>_9zXD+22!oJ$OYov
zwrMBbu5eQ>kdGlzVIxm#i=q4T9QXaDvI{)tbkB~h?i6e*TQr8$CZBcH@5-^%eLX2<
z4N!(%{zG>cv<!A(49ysTXu-5W68B2J6Ibt?B`HvXI{7COC(d8ht~gzkRbcI-m1ONJ
z!Rf&lY@S;-S8Jh+<t_$7_z)VuKLtI8Gdel~Hml#ug(kkJdI9+u)6|1Yrbz*h!@&O?
zhF-T93aLUD_uj;pQm1wF-nRNy>lA3G@EzcxBjUdrfki5WGB3&6CoHtSwpD=b&bZ^^
zCx{mtILofXNJj*J9->O3WAK?Hthj>|`OHp^xk;E0C9!$i?)pt4Qnnlad0EVy^`I8g
zmn+Eu6JLK1qi<W5aT^&<J$;Ncy-4JGfA?qYzUB-YyLU)1fa!ypaKpPiqwvUWsVtGr
zOeV^Jny5d^idCxOGh3t#VXT(EM`I;ROcxvce7XITHm11Ngn+!=?$67f;KWCxi=2Ip
zw6+V4FSXOwF}uf;S~E}l*@=1{WfOr_g7fDHLwFzfbFz%6=qp=1-`7e29__PeQ&W~<
z3Fm!%fkgiInKw={NKeknM=dK`P+tYEbZqJ_<Bd(38m}r1ybHowq5Jccf-Xc$G-?$;
zHePJv62_|&)IgS<cbo5)JMdy~HaW$R>@XfTYN=p=njbxOI}R4(Hj+A2y&xC=H>yot
zA~ZI0Uv&sp$sbt;Jk-L4&$g|^PFC5%cZj?SNG7kpW}3^&vsNTK!oTC}r;RFNNk6!D
z#K32|Z6y^!l7B#Z9mS`MR)%A({H^>dyfM?Wd)PZBHT>xGG>igRt4f$5FAV`dvSRB4
z8j>CMpOI+L>-?j|^hw<%aWD5B-z<!^DJ?sb6&PERbuC}fVp&wW3}qTaalfBExE4d&
zt0j}hO@-RJdvIBZ5agWMde755C?Xc({m(;jm;%0VZrJJaxTy%hod*5Tsw1lPR+zxN
zk^at`kyilI&m3qYG=h%%3Kz*De5Hn(eFEzN?A*`W|5|ri*^bR{G%rBpv4I7h0&Je&
zjd;Vp-jD>8vv;wHhIsycm+nrPyU1F)2eBk>YTi`%ZAxcW8wNoQNYTBbA$NbDQNb>j
zw~V?~B?gSUazcF_FUg86Vek`L^`foII@ccM3DVa;M!rPTL(7F~B&lYgNwGZsEWj3-
zw@Fhs7vUh@oBB2oVywkx+u11@vh&ZVl1gpl#uGYcRHz|24aRJ=zlHV-lPRZ1Xdlf~
z%jeNdKRlHf9CGgG3}bp8csiGpkGHOg+-98==ncUyR~8M?{X3GxRR{H{%EaV*R}=Jv
z7-`esy*2jNE!G=Hn=U+r<Qjhh!rU_fbQxdeE0Cx8C#&BKim7$(C+R~t9I7LVUm{%K
zi!%Lu)YNO}XC9s-VVlm92vz#1%k<M<U+Fv7F*e*6I6Fa-ANv`$z&rn+>h`3(xFD4a
zxShTmE$Bbp9&Kw~&`D4IpKPPHw)XV8sHn(n9sI=_r5cK8tJRpz;?j7KCXB00ou@iI
zZT{rcM;n-gYv`m6d7cgM%$*lL{e69Jsw^C@j@PrZvpFp%96*l`HFe+=NU!;>CmX?-
zWL^MV0vz{$E#~{QwvHeloAsx1eq57wc6O$vE&S<^$f8{tpr?d*2)YsCdO|wJ9TY&W
zZEOS{QOBMXqbWb3obcS4mY_>&khG@ldMj6%RqtnD^fl<J01XEJAHJBl81Ux%yW#X#
z_Q~~+*?X^gkxt?Q@45+eB|#=XiBoJ*q@|{|vWau<8YvBqd^Y`y=Q9G}Tqh$i3_w|F
zr0_E|X@jBd;EE}~c-JJYel)x~>~!FYkWB){#8LnI!o&EHRaPX3<S+m81}~VJni~AN
zAGo-<eW?I4h)HR@e_9(Qc)bO*^FCeh1G*WsLc+tTavXZ%=|QWOUm$3s5HU~H)>l=9
z5!za4bXV7gYO@C|ERG|2G8!5heP#*EKZ(xI@9XwZmm#(%VNBVC_26uxD7j3M`b{?{
zRp~QR0+o$dMce^WM$=WDdQw(q8+5k#ztzaMC%zEn^KhVJD#^>6$fDvg2kf&kZEbG@
zP7t7t`P>GG;D+QcUhIKh55UAoNl71WPcTozUz>eUQBiS90c|p6&4#uzG;tA=_C9ML
zP~<2ri@0LN1ACc~x62Me#zEHRiNF?(K!L4^e*=?TVKrR|Di3j%^s<i~oKd5;Q*Tj}
zx*<2#JeBRU{z~kVI=Mh4+b+Z(JBidJ*2@b}&i(lae8q;~C@FJE1>B-7uV!Htd*ouJ
zElF~cGW5mX+&T<rFlE<;$@gd2uE{$GdN*D&38E`SOBS&jKsEpG)nd)JZ{J@1BgyV6
zuTU`o0AFWfYde@8Ao18HB^5=gQ1&7URAl!g=1>)$_P3D+cZ)h|=pOl`^V~4`F=baP
zQ^aK2p=CAfxaHV9Sk04`ZAj(2_;HA$hyQ6WsYpp7$SVNKwD<;4sUfqe>FM(Hk!V2e
z`~x^2DZ>8^aZ*WZK$N$3cS|9VpE^NsLZD^t?N5|PJ*(+vAK1`ev^lIX{lyO|=&g<2
zi-A@gxx6+9D{>#$6$umcM(WU-rCko$p_nH4-wI*qCzRD9NaJK6ot@#fECIg<qN??H
z3C?NI{Jb3IM<BsCc7);dJr)9hc<*Zs8o>1Io<{;t6$=CPApyWP=uJ&bgtvZx*&BC8
zGHRBYAguU6-GjEZ*(yuKon_jETKiyE=|>PeoH)&t3w!C*WnHQ9Dyj%gdU5!MbXmLp
zY|w^wtxUdou=n%qv15Jf$C!OY(Kr(74vDoyB*=9O@D)MW<U23|bg?||{Vx|@AG`&9
z=|o27Qx!V6!=SVU6X?c#O-$LFrV3#dWEFC+>-@&$F3&fRg~+vYBuF*zHe%Rh_ndFJ
z+ovS1j#Rfq$)wUYDxn7hU4uAmCn4mhTh=BG1*bx>oi7~xdb{#etKH4n;2RAa#jE3b
zx5slz(egW=2Q1(|rZa>6uMw4jb9YBSqh>R0{R11z+`C_>cQZf3R7W!<j?yfhptK`*
z+-G;a)=L+W>55Svja?Op@=|gRRD-|c-_Hs*p!osz9R2TDua-SwJ~~J3kVa8qYT<hu
zdo=N6%@x7iv`|Se4o30B!k==H-qH_h`rpUXMKUEIyV-Hu3=UPseG~kI{IeVis|4{6
z_Nm?E4qklj-0#<K!Zw}mZf`KnKb8*0Apux#X&yJfFeBG($-FFNTv0t4R-2)`=;h$v
z@ek2?Oau3?Hh9ZdXIHh_Yma?qe=u2}K-S_tvFIfCI9{RxjDo?{+g{txB>vl<C?m02
zoP1?WpvM&~=p3^P`F*l>Y2^|^qR`H{l4ZaO;qC-T?>|Z9Jt<-T1Cfa1G@=ng`C3ij
zJf~jO^W4M{Wwj7qw3i@MhYd3LoRa(jkGGlF;}UkHrW_)q`I;NbRRmFJg0!yFi}F*s
zBAp*ve7l#um>(xc_;+%wr}$cnS{`ks(+5AuCagSkc+pa1s|Ae6eVPSU<#Clsx=h9z
z7uS&8#Ql3>qPW!F242;+%Dvs998sNDY<(&>8fV-My7ykMJVNh2R=hhF(K#E>rUdPi
zH>aulPdm*tXF_b%<qh4(03^-kJ#1o7S`^4ArlYM?C!+rf(F<A}(OuS=-*2_74@3Pd
zl!8Gn4&v$KP~NZZ;qBb%mtv2595Hqb@_mPdU8%(w5+bHAkYKbf30WjOEoJV<?ro08
zJXClcuHK?JRb$R`Cq^!}3-~u$F|XpaA&tZRF^ouvlDm@(-9-n)zJa|o{e!(<4+H}H
zR6M$gY-b8L$)NY%3FTXHT<*G|>ndW&y^$(KOwByI?32?%W1d4Fr|3i0+9#q#{C^HX
zCzf_#R6X1`K}D;Kg)y$X>FXuJL>gQwU3p=KJ1#VI!*Lj*wr~|ioJw_4FZ=P-2ZrYR
zDC)KKkh%ML)$AK`lD385rbrA|b`e_9ke0La^j4^i#3Xtaa?`>TB}&w1j@-Lt4G!*a
zzUC#<-QAY+!5vZ;PPV(L^X+f04cA=2G7Q8qjc0woL>i2fqpHiG>%dPWrS+DTYvINq
z$EjJ!RKstX`*<?0AU%y9#T^ZQrS5NpWLC6*{DPZN9iqi%l7tTT;oZl`V=$tf-tkr-
zloLCR9Qt#bsgrexwu5Gk@&)X2U_?%z`bWxuRIak>&1VQNtG9S|gxfRST*H{?bkG}6
z#>)@9UZ~WujlbeYA8~HT6sw|i%$8j@SMKCkw|jT~?N->q7}?<)>WolKq+v{U;kWMn
z2A#6|Nr|EbyGDV+mp-+o%XfiTUonPD5C!}2HkH^i;E1N0ewb>Ien$?1x<BFaO4HI*
zrFXQC;fGg>%t4Vbsq*d4Q)RhHQ*p;RV%bWj+1hb4);DYlQ`^I`u=u-dVv58tDSaaJ
zE5bCj?X#v|g1;($UT7?`-_6$+ra$*NaMR>{4OJ2}%IbQ#w<Ak^9h3iyf}Z~92Y*J5
z`Ad?vVm~iYHGzhA_?&j>HiPd0fqoME1ZJ7{XrrA0gXUA1_-R*s<k!d#-0bBw59h&H
zEubK`R!~HflU^*KyY_X{F_4L~E_Q`t@F9{K8DW?;odSQ!(5t2fi#ADI;aTCVJt4wN
zbFHAaqLpP8x+1tee7_x^aosH6W2&Z{`s%~v9cXrD#t}hMyA&XiXfcKJ^HmX7<F?~r
zC@1rM7}QC+9O+c}TW+%zSjPCA0{_g=@Hxj&sYeZ?q=w-{K-yD-Ua8@z`>R1r7?(x=
z=O4iZisAsaD@oFilB?@#EEV!1UYWfLjp`!J+^imI<<Bvd9HbPG{+ZdW(G6-l!>NC_
zD7>VgWW~D2r>hx6lIK2I2!+vavn>S1<IHD$N!1zL5r>%j2G44$gcOF7BA3xb$f`fr
zf0Q^Al136`YTrIj4}$JPXG@X4Dd1|ikd{1ijn+;7;wjg3^t|!T-}%iZUJ@9tBv7aT
z?ilitPM^K9T~H#>m2@NcX^-M?&y!QE`<p_#7cp}f<X+1*?gK4Q8$mVcJV;~1laimP
zIa7x&{1)M-yru$+Q3o5x!vz##=rVaZgh|aVxMQespW@*mzX;E{K+84e(XalzlfsGo
z^~stO(h<D#tUUt4OKu)<KH+TUhURD{bqf7-p&s!CWIc1&)KUmT%VwH)bFBH)v`l=m
zdtsy;6*lB7p~5qafD(@6lVg1i+1qDNmK1G*u`cYY1<Vbg?Ax}J3dt25(;d4+si{;`
zHimT%qm^vE5^2rmUy1LUHp`LfUV9dWOINm&uoaIj9yJ>@DsAHjr7iw^56!~#AxEs8
z77%&@;I0+RM+nlRY2`MtfwB$Z-Mbf2`5PUB%PO>uO0lMSs)ubO?I@KLa&ym)MuulU
zkP1}6&gmVD9yfVQ>hv{c3W(zTGb31mVL#_eZtbG>;!eZ6d)Ldh@sq=%ualiL>(nuC
zpNpD3b3CMURi3t?l_O3%<eln&=a^}u$r}U`jjp_tY5wXiM#yZ<Y$p3lpI6UwUy7-*
zqJQYeq+#LRM1l4g=d`Hp|LC%$SeUVfE3GZ_#L*^<KZ{whP`2xe_==oepkw<OV`Idy
z%Xl)bS0B+eW|qAD39<EHf+3W>kDTA90;5#QO6AaooY<dyU&wuOM~-1eV73l1<T0tL
z#lnNVyN$cE$r4^~^UU6YGRcym;i+2|?#bB)pZjN7QruM3E?)>&@FRb0_JGMPeQYt6
z9%?3jpg>J<Axwwa=IdTU7O*%KV!8JXW~tI+NLYTwQ*yaTuaH|IWfEFwqWZpKM=YyN
zpIsl5cAtQ%p)4*gMcyV1>#f~)0Zm}A&~LMB1K}E%oPC+pjezSQEqULc<Q~v$Qb#qj
zl4dU3OwAt?QH05&=AOEBi)|h0RgS??bp+I9&PG!!??Q`og5=My$b@|2NM0=#;>k0U
z$=C+9;bY6x<a_Dy<id|hedV+9Ah{3OCG|^7c^HKy0-7|%0fs)+nTl@4fx-bgvxa82
z;8>1%<BBWSS7pwT!3QMW#7SON-hTBR+kX7%F8E*Zq-O0+@ms8tSO&-p*<e=KzdVm9
zEsl-VMcSVAzG>@g;HG^Vn!jTEqB)RudbLjK{VdI(Q6s-C24=y>LngZ`yKT+=^M#<3
zj=C@ce0Y*?@B;ILrI8rFo>pNI*?kqDHGuk-VOzoix2gWE*aU)oq4!G!YEVETQg}OM
z4S5h&dq3;9QhaJr(&Repw?g(a!a_?XT1xs~^Cs>qnIoaW%ebY5d!7$gD$q_LPd<J4
zJK+3Y^ek3-^Q&7)T!y1fYcixIf3BY<zBx_}Q_xChjpG+X<TREF3aU_5&1w9)nPfC_
z(jsqES1wK?FCf#@bs5|ExX5mG8QM>;Q25$IaB8M9pT6{&wgCuFx_?!#v)hk>EoC<j
z7W9o}w7n)SS7edSGT}we+-q2Lt(?evW~Ns7t4nPC8U{tmyj0O>&a%+Eng#`djbtVJ
z?y2qpaqllz%$PU-EPg&VSnfHSb*>c{71I4Gx=GAw`dJvYiNdnT8mvM`sq7PN1;<R{
zlyJ5O2~sL4p@QR)ZiW)^P_5mY_Bp9yvGnrU@zp6z-vLY$)$xaEEDL2Ft3hP^mQoFh
z7X<E}OyXt^P8`~5>#?JjNK07{UmVVfwkXQI8}z)i58Lu%3#M&QSh`M(4{p>;qnRLt
zUD9E=dXu5kZ+Q>MshfjbDh`hqWri7=&V-AWKa>fSBf%z*x3ag9Gj%TF>$kCTXy_gJ
zKnY%ymMmDS>W#FgJzahe&}J9mL5I7FyHIpz(B~)GqE+N^vSR#X^;tTJxw9EED^QJt
za|am%-M{4Km5rBl-1SsE5h`M0RGPi+-c;K>UhRP>J6^Q@k825}Zf5^O-e^r$m6ZGi
zI<NQ~wuXw;3SWQLSl@^(Gztn11~qrmlJ@H`o{OMisqmy}CQ8(jXGzjR6JG<NFyygf
zs*d*q7JLgyoIQ2&LfYc?$KQz1IO?d8;uOfOBj%`hb`AF0Rr79szNr0frTm3|zHC2e
z@rJ*V68FSj-MSq+-Exqz<p1S<!{zvAO>kq$2K2HD3q$F$y?p5d2X5M&o}7T|dkdfv
zih?RftB?lt6i{uRpP$c2Pe;HcZ7}JM<sOt)G5YEP3QVBYX2TSU?1%?O#<gV+N?Tdt
zk{5Qwxvxn|^*!e{44L!9RJpIYo{Kj`YlLGUB&fZyRx1;W_InY`Zq9gf`<8Mw<Lv4=
znkjGlwqMj-xJIxm-wJ<w5&7eJ7A?TwyEPJVS!E~XtG+MSYs4Vs*YbL50V<;Q8-0fR
zipejRs~6WaCxuCc7camZPu*gq;dD*Fr8=#kqM(1TiDm=@a2xYKwqa~$c{hDrM`3|=
zsk*c1C&VOo9TS9uaasM6WFnHV)?jiSzD1a+-|sp=yH;FL&X}+ymZ*H^TZ(?+#>DX^
zg?_7ziZ!ni7#{MI{Hz~OO7Q;db`}(9M+++Z0-B<~nE@4QDCYHy_<nAbAg|Z0f_?et
z&!7M8y2HZv@c&y)H1|Nz^%(BD=MVvOo=HeZOq;H(tni!${oKUnrL^RU8Ll=@C-<>+
zA9D~S3x0>F&@n_zDnn`#VV<K~ED$!F!Os#yJ@&G_L!wEkj~@+hEva7&t$OHgRmWym
zDXe^$AQ0&_D1|?BkTI01sjFkN1P#V0o<`t?;+r2<t~(PiHuh84%|T^Rk{&~$tqru9
zr;LCmJV-qnDv$vF)m=wOMARng=Eh5PeX@CM7+!*8;43^-%3s!C7+M>O6`fla$&jf2
zpqpAG%x`cbhO~593Ihl^i8;yT6ZZYSF6ND2g2kWa)O(cebWWS~k3KK{N{Y+aHZ$D)
zEEG}eC6hBtztm#F{Y|V`hIMTMX&fKn1AP_TA-IuvF(~vf#P}J>VE^1F>lc5ZgN{L!
z4^xp5UNYSF4d}S2RL=LXyWg$<#))H3GGdQ+9HB0%n?Y-Pq2w)!6H|Bq`4yi?Ov^Ej
zD)2IM8%qou@wX=phfr(+wvtVRvg6CYl|w3<6E>iBF!RGQwVe0;!m-;yXFq2RdN;TI
zJ&viUa+U%LMys=OLjT)U{<R1#k=Ia!CU+MhCJ$+R^%<g@GQCRz!`r-H0d<R50sCZ{
z%%Nw6E(@<>Q?55oF#<|{eL2_3zeYBYwfm>M44O&mooy3n4f`|rl4?JH`SQgBI}rjQ
zZzOf1v%^kI(%e4$c7A@|C!?i|LTA6!;tN_cT13sv%(lOMZvDb-H5#W7{b|y#EC&9y
zpwJ2uv=y0FlD6k_xAo4t@(90|G?wYONAcKnWf4aS5Bj23aRB3-l_+jCeeIFiE$8ky
zJCBKn<L8Yi`T01pf4g7M)d9MkM5{Spy&{-k0jeoz?>*%LM*i4<nT?Kt0ZPB1XYUXT
zv_p7%KY(_qr%B`yqylf~p7HYWfqSD^N#w&NJaAva-`Y-TIyd?0ly(!S+NE4YH^Dk<
znd42Yr&3%(ohGg7pC<jhO?k<~=gXUi-!%0bvob)7hK7IWs;3MfP3Srt?qZw5Y}lEy
zN%*6f@~$sWRF^c|%T)hfHE>vL@_bum0fZFii;-b0aQUmwr~}1}$lm^hTr8-C3tX=*
z=fRQ!tsdkZpxua+kgzA7jg9TO!{eR3ug0C=qPGTdi2sT)6yP$^As;?>Z>yM*vSPI?
zX4rv7$i%nle47izmv%eE6XPGoUTDO{D<_%tq#b{YqXWv9tk~~_!|Q~@nf=2%+w1lZ
zpXLhJz23_?WE1Hrg~!_!36#rwBD}n@+d_XzgF=pBe$V}AKg$K&aEHTEwt`h-BJa$$
zYIataf-_GAWhQtwV<H>&CUz9^H8~k*jlPUsB0-A10M^@>doRnn^}YUR=DB7Xww-R|
z>GRBGWp_OSG=oGoWDZ}ZxmyQz?IK~I11ta?V2bj$bzyKlnO)Yt(`>^Ja!X=tBefY5
zMD1AWc24d0GW!>b?qsRDfOp5W6Zi?*bSApY=ylo^rm<mEZ6^GLGAkxwaT3PLJQI+^
zWk3qS6S>A1{fZ;^dl_n8Wia>9zEh8kC|M(SSV)VOJ(YJ#b3uMKUN0x;61s9?bfcv6
zE19@oraU&@w%3yTht#xUFHv=*E2ws7Vv9jWf9O8d#W!B4B3t-7(zpyj&4ht*8Bh2<
zI9`GrsxCUH$+LI<J>KIZhwTqW`kVo8X4_O2`CQo&vPL*N6&br>Msr5Y)p2BZj%2E!
z*_}qpxj|broEbT}Lnz0%7RY%68^D6<RP=;Eg3B{d)jLbVxpNpMfw`<av8+_qtnH{h
zd9l%^^Z=AKc<OXs|L$vB*SA(fQ9UKwF6kj{CY8z2Nj8%Xey23_-mlZCx7J-x-3Jlz
z*hXv5k+H`SG-pm203uW|c=LR1(PXrHJW}yBUE;glN?#}pxdC!ZTXHVGMA9I))wl@i
z5$wu7MQCTav?XP}%79B|#uwA8_pcz!FfsdSuw&6ZzKGv5n5{9-y_msu)6VCStFrET
z@Ms2HIh#9o<M*1C%D!q^gn0)p0>Ku!bw3(n8-M8#!Lk@<?KHW0A73ZoG7D`Ir0NC>
zm>_dfgX^=<Wk1R`xg_^q_uc6@^<teK@z=GC{+5?yBPCfzF8jXu8eCS)y|cL?zlwAt
zrAEcsnr-ldy5IT1qFsa;l|vTsk^s7V4Y0_{UkocSzr%ktWHj=>^@4vXic-u9$#jmq
zB+ktEJgfUEobo&ZSMQRtef*7%4(|-+3dMg7o>l7YH|36h1z6<aSvMY53oZ+h1(Um5
zyljnj(6`H2(%!S<ptHm0429R_eJW3>n3+*NR@n(WNKG{j_sFCd&hD*4QpyStb6am;
zeMm-3=u9|oKmkaRGDcRg_6XTT&qRYtn~{aG9DfAQhW-`$rO16z(rs&AjHZ-esH^D2
z)Y3Y-gz-#hn1^H{R>^Zt)?fWA;X{WvfQ+xdRK!Aa5^8E+y?0Hgiecm#&eTMQO?K4#
zq$jo%Q7?!Qn_QaUyvPkb;=EyZ72bs|#SpjeFoVqU{<La%;K-3|uTq2xQS}B#3NMR<
zLJsM*3`0!74R3AAu17w%OzEt|tMGMAj*P4EgQG9E>D4522%N-<V9(p2%t&S@e?&XU
zkuifkx2j)7gl>2c!O||Wr5=eDB6-LY!gZl>jWu<o?|~@NBcL~(lLY3%<yI>v)1o*u
zta>@=wl}Ht$mrzEzB+Ajx%l*e>foiU6)h!*R*XYCb5ddm^=*K)-1xM%(*(kwB7Y7F
z)>th{VuY(kQeVa8F^5=zl&#VChAvLUS)=EzOxr{5@kkegsAED(>XxZz_uX3Mlz>=5
z+oQfE)L1-_RE=0SjfZ3R``RUQMciGF6U|^k`{vz?wK+gpOdt)Qq8Qyo$Y*{0iD<iF
zkwRwJqJI1kT+)00**jY{>eI2A9RTX`+YN7###t`IiHHbR&KHrD+Th>AbX*pgR(s5s
zL0_xsu-Lk<Td1igD+ywAqWrT>c$aBeMqKcs1483j;3)bzQD9t-Ts13)qK`YSdaV+F
z_NKYm=O~3?YTIxeEsP0v4Bavpc43R63)3Y0f^&CG-M%7DiKAB5vD0&69JzZgZCiOb
zl87}KECJ6zBPW3#aJT0QuJ27#QwzbbDD0GIR2d7GHEtk7Tb**GGjUY#+by8X;v7;4
z)gFaPsotRp>+->;jM;VIqgE$&fqMaCH72SueT+)Z?63!vi+QMmsu-(A^B4LgLr#qq
z)00OwTmxfcEmMn`ny9eJx|y8<!B%OutXWL)niMe>D7(*nJdMT=r7*}ays91&1ZjIY
zAu9Dt$L)c0fxq@IkxGQooB2e0vdxKcZb$ac4I#_U-G28yhD;NC*p;@WpNGpcCz8~j
z=5$NtRR6j^Z+L^>viou2fFEx5$L4I8sOti2N^}$GgQr$KbAbLtCDF`xi^1a9SH;Si
zQdzuElO9`bb0e6CtrtrjFox2~!^<!1auKlkH89j2fj*dZZsp>g3-OJJ%cgW6Bc`^a
zCVoz;fw#LJ_>V@MAm!(^*dGF!;+>D}@Zk>YcJxdjPeQ3z`jxmX8Gdcm5IH3=aR=aY
zTSTq6DMWoTK_*X>`Go3zb=Yut$`W%UDeK5bA8hv{yD;l#(moBJUhNyawed|31_hLx
z*%J$3zP$jek&nz_&x2)v(!3~?zO&ki-HWg44YRJ*1xBSk_D!3FPA{7qIIeJ{hP>vr
z1NHAHY`UMAUJBLkxE&2bHPXA*>2_0&-*ndEe|aL<@QjRPp~74Iw)O8mF~zE-iv??I
z7`|S#*}zFaTX>HSO`K779|N7;$g}Mk>ot8df3C^rrb+U1RPRP(!61W>k<1&Ge1L(O
z*=zl7?&_D<VB3jmkGC}%x7MoTyU3mKd66<M_*y3GVlBV*Jy!*g5JEI7@!F@t`PjiC
zl>FkzJ!#2|hP(Q>_2=_86=rT7?5j3yCp*509dEq%c4nyk8GI*NG~AUplDCC*fVjyY
zt|7ilrkgXX{VIIf1~@!vVWO*cW*FwlPuoajCfh-9AvRfJt2mfTa6uZkXVzT^3C87S
z-h%m=jfs<rQ*L=g&}8%0&m2HqDsQ<cNY?FN2*<72$1#z&QwCnWBH#aEGYP*SwV|;~
zyouy?Np?GTw(_9In_NJ{JqGuVxb^tFf##);wn@?`Gtx^+G9XATBz0zrKnb17QK9}u
z5V-6vh+Ec7c5;7X;3j~ckdQ%_WsXJoycZpuww_#A#Nqea6{B1CCFngE-HtAH*=io&
zpv`4hW=%(!^nG*l{5VV-7_EiONjQX;pKd=qAWD~avPO0L#jfp&N-u)ToMsz1ah$jz
z{h=AQtomwpp%tF-0@`ISWRM+Qe>Bvx-p6Bex5QWn@Z=N~y&UW82L2JX`pM}0|5W#u
zVO?$C+VBFwpadmUN>I9#MoLgnx<dg0krJf4l~6%sD<Its(jXnu-QC^Y%{$oJ{a)w#
zpX*%Dc|JaV^tYI6t+~b=bBueAao;%is&1J)bQIWwoo;omTr0*|JMxngA9;2uf95qT
ztzWnjHE|if9~QjZ)0ZZ?Twj?(P#11PiTU%JT6jsDHmUQ=AK8A{ihG3H_&ir<e0^S-
z(x+!%IT8NUx|P%QiswbK=IHKL!-#i3iG&re$ZGG-qwMDfHV<mH&9wytuQa@|bbR#$
zq7A%oyq_t-^U+JoxM%F!?-9-{Y-&tnc}k{-xOX~km!54p$jL<r_<bDs!AbjmA6(XR
zF}=4R#=Ci$gMO8<2a8}eh;-6eu97a=IDPg>5sKb0H<6AwBOCnKx@wON3#7Iz%Mn+F
zVO9A%I@MpTQ6y$=h|RV6N;D_JNkW}WKTplKbFOTf2^EyhO3mIeIP`N-*<<2AaD@Zi
zl_}{2n!3*tw`(h$a9qT`C`;}8#qDElv!C?shJHP_Z2qa(7a=A8#Mq?W2!InN9E|J2
z%A{M|PYDA&?$16x!Wn(1pjhRLc_Uiw`XXip(fx-`Rd<^M>F5%fohVEh+gzms@xLi~
z$oe+McLfwaAAi>GNyK$b+;VqLDc-d414{Wvy<*Lwsd+Pg4r+WdVF65r>>~b53ivdt
zCb=XZmRxF_bE0Q7llpNh;lRET)#tPK(`E9^r0dU?xtC{rj2_shwsP8+Ap8@O9M^X}
zanIvv>JNCw%-IGfooB)d`JXV%Wk2s=lU--)XR9bu3%|BaL`ayu^W;I;@b;Z5L=PQ~
zawq4|J|%aw$mS%@1T_m^o!D-_*MM5`+tjvipOIg_e#Fr-jfaHPKyC=vV*tFPtn*D>
zU9`fX;t{?H7i05-v2OqjPWA<wh!m51tdh>USj9ARyw98-?WqY)ZC5a4|M{-n2<K8T
z!RuTd4gxmJ&k3({loJb|9%X-JLNk*^U4G>%b@iraw~36qrX|D@kfx=a5lj316}64!
zy60!rctbdOl7NuV<FNSZdZZQCOM;(C5i|?R**@VE3pb+$dt2?&=gSnnl}p#6FWtTw
zl)q}<3O4b<IiM`@H5Oe?bPW?LX_xKF*)w?#DWQYC9E{vEyR9qE+zt`d#XBFi;;wwc
zq@5Pqs@Raoolybmt{y4s!28dx9P_B#R^+rEH*jS;p7{k2ewGjKB`DdS4)WU8A_io6
z>YjVT_d0Gd&p$Ur<&Sz_De{;w;MUN`$H=+4&+ztyaD6}fK!5<RgZCi*M}v=C9BIB=
z74uIn&Bkd`c&Mh_WufUgE;@Fl8TsO}+aLCwPL?Ex-nF@Vm9W&cbog_Wy6M`!vq{Kz
z+-_VNo9tiwnLJop%F#3h6$DN`jI<%9Ggv52!)!ko&16;e25gnbJ2fWXwHGw%yKWUF
z9CWWf72%^Q5WSjBaNp>)Q%`f!d69bWF8AIOaG$ls>ndN~G%QwL@*j~-H)G?)UwG>u
z_~WU7-0HvgIhfib!W9vq6L=Hzg;w()jN&PtjOZK2ANWryG?!BePo|>6q+Q-77e1=$
zbc1u38agNzc=-F3Kp&9@SUFiFX@R~$ZCsz3#LXS+lEfzhg`BFxk^Ll>vyKu7;)Sk4
zKrL5G_2?Ih*k9${Sfq`p_2}{FEhSt*YZ&VfLV`t{7@(l8yy_(Uxu8BvGn>gg=fUk3
zt-3a=66U*H>DV^E=JP+G{B&vCY}?u8c_Y-wjOlt+3+gI!Dy>;xbVdX|liBGlW+kzy
z(oc*U^()LgLNoNh)tu4Q_j2Hu`s$eWaBOfevS0kaBbU2cco(G>e_B8QZ3x-zm|OZ^
z5UQv>%CdX@yaCX&v2-;qyI;T{wK9}1!I-gghwKjTc$Vwwx#!B^@!&W-EmZpfu8-ck
z%vWMOVJ~*}!cK;WykZ$LKM7k;mEp{<ZFLmfa6qoD@A!%V>Oug`05XQ+|Lp%Uo}QZO
zDz)E+EWW~F_bp`cOlrTduF|e{UGxr?LtE9+;UP4nUPlF%k-wOsbnCx!u?VoT-f#)e
z%rv#K$_or6xW9SPg(R;#aOfqpBfJFg6Ktt9dNWUsa7MHv6jWPju)d0I#O%@O@mLFp
z(apJkh<F@zmw#?37y=P3>_c22>(To2)E&#CSus{kR{wuH(Equ-s|R=n{*-sg$P~XZ
zGCFSKT?Z1K0RDPnSLn0=8iU6C+3aoXu(X34Jv?I}i~$b`>K-F34{Rtf!<GLMiRk#A
zwBA}9(DiVBU}YfRuMa8(;6x?mb-`VaP}U2xk%tBab#*KaH6V7=YXt_5n0sk?pGTgN
zJ6oL`n9pUvKWNWLSZKcJ&ak({+4v9UmGby_G?$Jt$C<3N$;nCJQrLuc+E2RyPL#{B
ze>W@`J61zbzH`gh5Is=ZEbr_B&d7eI<SXC!@UKDus23g8{plNi$_Bi6_EP3ym~j0@
zT|Kc*j?SldxWZB-0Oc)wulj0cZLQm9JX%a}R0Fgy`Osb3m(CSb44fp1Y6T{Lm{T@e
zH!%ORDy-OM^{ugS%SdlupVt~IpA)n=%PaLeV4{7{o6TOr-WJ<@|K$*tH>_+$dgA<k
zGSmS!=yfp<O8ds=yzl(H*3mzKb+C5uvaq|H$}>ZH(M-NMYet!$k$CI4!i*j$>f*jA
zB*etT`1oit!+?!H`KZ`3Tfaq?Wtc^W*0PRM;qS?cvPt=Te6rZ)^=EHCqHz3a=~X55
zRF=}mKj=TTc%Q6xnp&wGY!64Nd8j>)_kGwzD*S2o0$M!9WiNC&bpR?iDy6SNO;ucy
z6O?WSJc~?%m||2q^;fS>^iq?(c@H^SN*sA}cmD~|Y^F=vUH?WB9a8=A2&wuF)HHOK
zsaF#T;=mWiXxwg;=7Jh5AMTNFJ}v6-f<92#RdH5K)pEYPXh`XQs4>$V6jN2neDw|x
z^j;Op><w(S6Rn3H-WMSkZSGDiG1>-j7rc4z6ED9sWV^M)&*4SiM3JUM{{vqWqcGlC
zA?)j9pbf*NE<0N#cH!hY)q%)<<udgDUT?-&VQ|5a@h>gO`!62<8_e)Q5XC68MM4AE
z#&q;0Y|Ju;U1&p$x>&D!UT}{0u)pqcxT4K>xfo!<Xugcf&6lvVq;KNrq8eF#Tq4)_
z{!NPBQ<2zvWY0u3iDAApzq(EmRA1$gGB#>oj)ubo=oTr4Ix=uJAI7qv>%YtbT2C01
ze`@`D$i+p@`sS)Jo72ID_a$?y^)XU;C!ke~i7{<ExZvaKfOZXMXbV)yP;bPlLHQ=A
z*K$ks97y=Ei8&=THKQ6=nzB~0F<Y@b@k5}PMltc|coaGq{wlw@L$*EnX?c0RC$en|
zaj}nJ?3EWbse&`(oxy2<%Db#PecOF$xP-56-WeX+ew5sf=n4KR)84<g_QZp+;Dt0{
z(}G&X2V@KgSqQmlXg)vUgV!ZoZj{&fT~FjdNcYGKof-9uHqhSv%AdD(L-{HFkq<}6
zZlPouY&BP=)D8KyGijIcTs*m1v$=c!3#XF$qQ_0^Y5%EetPzxlrS;a=cl<62mP)IC
z?Umv@aO*V;9Hsx{>!?jHy>uCPFa9bwlW3)*G#c5m%j9$3b&xYO^*YGVWKnpPk*%q4
zr@DWpY_h3b`PtRYOBXLti6|3?NG3W<xJa+3?Wnww`rY(5w6Zz>=DnKm>S9WLvfFus
zyz{p7-m7;Np<lJr(_TVh6G8B+`rI;Dg!QR=r6%wc)D&Fr*D{niXDj=7%W*5OkX`<U
zj??I!fOXW^y+28q)1!6Qt6%VyO#OoqNUM99I(@I}XH$3tWTo<F5@TuQ+mmeHnwtE0
zEch2Xy8XH8`9rjg+z!Q@Is&TN9#2CsZoxo$!<D!NirW<Dr3d;mz2B^`SRAq%IGcVw
zeJKo84O7(0Frn@!KRKcL0~f0#Tv75a3Z;T2h4Vj59~{4k(gsa}lGcCwCAZM#^r%i*
zPVbKF)@hJGtKWch@ND;7Me#mm0o3VZrwwfv|3;L<d}TO=?~mA7yF=BGwJ%_Bo2A4r
zCcN)9cV)V`kE^eyGvD^SounCg*0RI)$cw^nzi{nZI~L8w7+3y`LA>BAbjyx9P}*AZ
zU^kTpEvr+(#OtX8TEh6V$nw3C#_vb?V%}?>qsL=Qj<cB`qiO}J2T5_KDF3_+*$dAX
zR+ne+({9};dP>>blkbk~`L<FTt<W$>w@JhHk`SUuH6N7nO_F;1`PRURRc@aW(-G6b
zQk%;BX<O2q<z9)6N9g^%ym@Ocl%HfKpG`V;+K)^LU&sQph{sOvkPgc?i;a=d<FJ`U
zM*}3VbqVDmObj+Z_9DY?b^>?1ll8@O^(hT{VnTe18u}H(_GPR~s{9#fC$5gRmsF-t
z&#yaiu@CpxInC>}w9L(wkXy1<f61=Evg92OT37MXetnBg(uD1)$3bdKM^`MdmAfJX
zfpr%t)hI31Q#$b!E63uq^)!Z{Zl#!FleU!pZF4fm@a;n<=?sne@creZaLdsXF%j34
zae3`$MWbWa@GAv}l+-VY2R(fhaK1DAL5z=KrAr*03LV~TyvViQl|@QT<Vq~gh@+JZ
zH$Ia0d9P>t(G!G0XZA2qad}^JjAezE%U%-~edC*!25*pp14(WXDRE^vN2I!fQLTP$
z7+Tmy5O4goO___}byA^QZU~8HTup)LLzL^D27OkXZBJ0N{E;?N_5HeXkpePa%CcqX
zpP&@nyt3M!xITI9OWRssfM)o5_=K_9kXsb)^=$GSDm#gm(_7(9mma4X$na;fp=F_B
zh+`D;yLD}MdpcuH-*gP>+TVUxUWM~cbkgeN3a4{x@xvl)ZPP@f7yUL~t0z^Pm;#?^
zillNr|N5=cjl2|=Sh9D%>qdYZh$n*XoLt@UmPV=0q!Yg~Ej-SHbcnGfRB9aCl$A@d
zidjQBmYFBY!X@N9$M2os-0Q;q$+7Ly`Jz{rtxTzf%p3^qzk^NSuRl!~*)V+qQ;o-a
zO6N8jC+|u6l7Zv<;6Nz$1Y)VJ^5mOF@Ty&JXXR<5N;iI^;PKf?;EIy0{9O`kq^tY8
z!z=QuTIj}DL(d7ER1TQ6uT;nD63HHw>suWOxqQo#;U##(R^phu==cUB&g9_A#(jMn
zHIbLTwzu19Hjxq((Rtstgm4CB9a|>k&_Iko^Eymap|ew)y5PMPtzex=I{eC;dzY{B
z<|(|mJ^jE}YFvXq<U~^2v|0(M@`7$Z3zNO)=R4zXmvuUk|K5o1vyZs-o9SzhgAD%}
zYec*^VI&_VLtm3<?_o<j4jz8xF#j~n7hS`5sqyi%#`uK^isskR_rE^(8@N%b8(jTB
z;PVx-+l5uGp87fZF4@wn^*GTAc`=dwH&Inh#IAo&7Tqsc*4)LfQEU|)T#-WgDc<9$
z@y^tfwIcuZCGNtbxDevN8`fga-+mIyYBWolbh@#BoHh=&Wg%fzy&Hr)O~T$Sjl`|A
zu#~Y#>f&p44$X4cp6arobw1DUi_3Z%-1?G6(0}mI)%fU9n^wgqulgR{rRLi~wqENe
zOEoGm9M6dU*LjlRTf<>I#WofM$h(9;BM#sG<xjtrh#BTVie)+BbOU|k%M%JPusCHl
zSLf!$^~iF`_um;5!U6*W!S2;WRi!P$YG@vg*75kUVRB$fVb)5kA-geuMmzq!d?mH2
ze#-9}?=8JV>#OJ+<n!NQ=H|k0hn0s7Jup=9(2yk!J?Pr8{n}3wim@%mx6lwhd)6#N
z9oa(Pa6<qEi4-ATvgfC8(-q03T+7+$+>b@q@T5vOf4MA#LK2obFvTCVJRgG1G>DWN
zFsyvgw^!ZiU>G#Au6(=pu5#3YbWL}{L<>(F?;Dw1EHj?K6(lvzjz3cnE$e63R+qz%
zD5;dJJlPI{0K2HJf^q8gw=#YSOk`uaEyX#D%Y5X5r+v>-irHE3S+v*kj~6?q9inqc
z`MB<#4sNFEG*#nIS|X3#r*=!va4U1RDrRLw_Kf;7JxQ^2=IjGe?LMDUX@A_4g@(4T
zy50R-3W-;*41Dt9Y!1H}9(2e;Y%HkC_ND%j->8#z?l@CL=@cIfcZa`r+5QyXH=o6V
zoIp8lTeJ$tD?|(LIm!yYjl@6<f1AncZrkl)Nzu|Q%$mpDQ^{YF$M3mDnd^DYW!74q
z^X{)36gyGuqP96*tCTB^>?cQ6RfZKOx!XfMB-#-fp*b?Rz>aSA^4G5){b;6~&WnUf
zt!p}Wi$`Kda=F?~inGMF&8Ula^H+JhbrN-f@$Mz9bP>95$e*{oW9@|B+wqs;<a<#v
zC{=adwLQWd$``T0&^aOtD^L1?QS#WKz|xeLhCB07h4{FbCfk?e(_T}sw$O)~;uVVO
z-tM8%=6@bcfJcN!ee|cptN7kFIsD2xlPZn4d{m{qOZNtLPERcpRn=$~e4o5{G1wX&
z23C~5p&4VpCBgi~P_J};D$;Ej<+=|UM%1;A30k#wk@v_C8F*NNrf>D47+`M;Hkue(
z>%k>py%R;?*_cQ}THZ>(!WNHtAnowi8*=@M9@CULAt|flhPgP?(Q^0<+ab&PU=|+j
zXa=|e>GQFVk>+~Nx*y)tu=Z}`jW@L<?KryzI-11nkzTsvm@}0$KA=#=+;IoLu0J<3
z>N$5dwN`8JO&#{?V3WA7eyop7XY6`6-_9wK=wo4f<?O2CU)4JNiK8mj+d*F?-r>SN
zLyf*spiD1Rvc>8@@R=onMe2i)?VA~WJ1WNzuV85&j)eZnSE1S?;~D4sWaH}?<Fl64
zH2i`Z2a?EWj4?kEI;yI`c%kzXf{m5d+z)%`Hbiu8t!4Qu1CdjxdgHj9&evTU&#<tH
z=`y)l7Nq~u^_V5e<hkUBg)jN_83}PVyuValYi$XUx!cVJLfz#eTHH!jI(nw5uPc(R
zQoY73N<q74tv;ySCtg1^I?j69jgLah>)(tM+PpWRR|<O7Ts#N+)lb&MR-xZB!DCVW
zGSXvK+BdNaL+uIry%DA)Eq)?QR8{JS`#Q;W*uTgbxx3#eigPBI`t3$iCA@95@zk)P
zKVK|Bw`^4#K*ttwF?TU|x$r*ul21GH8Ejs^X9~wPPqCzl@A?TD(W$oAen5USe)hPj
z*fVF>hL@*yq!EWi>fEF@P?4`w+Otx@uhi$lEQx*}yIRFtWXt!RzeTuJ&%W(wry;39
zr^5oH&ocYd?Mi5sf{ql2*+3PbKF7U5$=8oo-`JP}OhoL7(3=izNKU-YXFwzh{Cp+Q
zEVj73OmqMKhDJz8$mPqIGb_#<fXH00Ju)IJtU3Sq@K7rF!TZRqA)`#-5EyQ2`|2t#
zh41uR@`F^ex19COJekvJRP-!g`WGD;!LfXGuaU5F&glVv!6$L?%nCI6G$k}ef>eIz
zj!PAF^wV+X&sfIFg4Gl^Q_%kM{jjUf7>k|-WS^0dk@@-7l9DYWMV3&g5faHfQZJt%
zUk3D7kbn94`T`TdyY57(O!bt)=%2tBF$?8L%h1b1r+m@qE-RZY(Eu%0beb^MB~;X6
zt3_6zN5~khsqr|wdfHl3Qv;ph6+2x(smwnudmCLdYf(Q@VenCioqX=5oSb2QuRg8Y
zO}4x^m%zY>G+2W>9Lr5J7@7hGd66OY{LR;GUQU(N&{s9hC(3|VOfOF+V!!(LbQ4yx
zFKZN<QP^sxD?>eo3-lTVetZC>QMY+*7|+e1)@6IHRi`Pbg8dr@TUs!wp1yuJ{vdEy
zttlvW)8k;K9eDLsS5E`oeJRlDPgiW#1V7Y)k3wafE}25~WvL$m0LAzWU5##IL82(3
z#p6Xn@^>l9lbps!CI*f7uUsub$l~_5eZz?(8zUW=*IrEIJ~sOxxh6`%Kiz`mujN0z
zS)Vyl@?G;wv;O;rfYf=)8#Hb}7-{f!d1|Vv0En&KsdMf;Q3aAHb}p`Y@(j(2a!9VN
zXz1uZ_vt)+iZ;#xrGK?PMBNbpxOKh>HD#fri*W^Jh7i+9pqslO?yOSwqGM!iv1g>B
z68T2-C6!=5Jz|3WD)m}6er0n6B?mEqzeC91!O}Lbrv@Uk8$BUp_I6`(p*ro7^7-*h
z#qoeZQNT^x3qRFa|BPkpMc(n$nu1u;6zg>As~g18%D9yT2IArjl(e*;OkZTbMsNMh
z;guE2`3^XR<j!Ie5_F;OFL*06ad9yvMdoXJ2%kNxR8Yv)dPs@&a$SKx^na<nxYn*a
zxv-F&lte_T(OMJ$jd_~Ml4nFM_XgeHcDTx%s&oiA?Av2#^<QP;ZcDlwHrV}9O0&e3
z?d`iEa$+ml>|$;KhO;+QV!5)p<fSgRmLB9S;TM7XnhvjL+SJI|?|KF0tJhv!D~v?z
z!9Xqikb|<aynGDHhE%{wHTPQ*Hk;PaQj&Krg@L=?blHfoxMp!DX{!x-K;Xl#-Ge)$
zz3z?uIo01{gc=wdu%s~gZ#cfu>8QJ{ujgd5lbiZ&WZo?5GxJT?&Es17DsRd|@Veo|
zXpyKW>O#LWqfbvy&o}*CSH>%%OzkleUS-*jZ6}v=*YFr{cGU!R1yTPF^=&H5^7|*A
zGwAM=?J$E7M|Va)c`eyk$GAVOkGPOnkzYckeTyaI87a94Ie9m(LVem-{K<W?(e59l
z)Q=6tdoA%k->j58lvV`~F}0S~y99+?iX}Ju0Pr~Y2n=&wBiD=oUr^^wm;EiR3fP-^
z5d52yRt18*fO2q5Bh~uyOX$;W7PujDiv?q|(%<ka!A?SbF`_#X77~!eTbE_861?vj
z>Di{6PtR1%#8&5Rbsmb2b59jz9Zilvb(E?Mb(g!+KE~(BfBXnX+$8*On`Uw&B_W~Z
zcxP-(%L+6ThC>ef%#P+|Kza;p+~zup^h*BJQnUsD#(wejMgK_AsW37!0=%a{p-jt-
zl>&GQkg5V(BDk5^>rbZ2?O)};RT|O>328q#QUsg+HR$-`d}=p&25<cN>|8gK0Dk5C
z6dQY86SJWI=}<^l|0M68j{%Y{6!q=cZ|!4(Cr>T!JIqneaYunxFGFc{j_=1LYiw=y
zBYtCoT~7}UMmj5p-xrXLRar;f;S*UZDtwBml>>aIP#>rwBjcx64O|-P)4;zCL|2V|
z<`x!9&}Ztq2vi!qH_h|j4gZi=*VGggIBN{NMypI`1*NWMb;PDEp&UMfcvnsbx13WW
zqg{PmmGklKe;dR_zOAtf+6?vme7h*1Ttg+wUNL^8k-6usO%Q|lIKT5iL>7|wN=Qt|
zNx4)%XTH3_xgzq|daJSVboEHZ_YG&@yFtZQ6S-~@y1nm|nyLfj!_^W|<l-ZJBwCO@
z7w=Ln$Sh@fRLxC1_2PW#Ls0FO$cfmAXKh?)!pe!&a<AM)S)jHKdBnlBE4+jbCZ!jn
z=hMa<y1=@rGwhcp${N*#=}ah3KU_c9tqv>**elNIari0oQ^JVE8c~Jjg1_rR6Gdws
z52rSM<qwy@wq}a78D!6c7{79ovx*rZea1El+hL*N7+G<Iy#0q}kMdv6X`&YGTN-7e
zB*dF#_4`JseH}GNid`w6cCN0hCY=5m-Q?%C>d{|mYsm*czRO#~rj1`IM~C6O85Y(S
zY^~G1?*UDco}wQlW}|t9aLjRx>5mzYhIW@_&W_^W-nZmhifv?<3M%k2&iBpUEn5ux
zE6`2B!SMRU=fth-YGP=c7HA1Um$AXctoSCjIDE{NDW891GKpE2CGYW~(L=Yg9n{D7
zM7z~PSGofBt-2N=%+(|=wD~drlf}>Z#p>9rk#~^8rX(euP3<>-%_4pF+V5a=t0dH(
zBk6g0Dk{n0a?t|Zs5rpovp{xrgqIO#sAIhX!je^zJV+ar8wO70gwt_Q2uYSUH{65}
zGj*6Ark}UkmU&n0(QHq^+jjD*aRL6V#rp+f>^fr*rtcCMffxIB``hiq+e@tBFQsp6
z^;>pQxFv`XH|#XVx5w*ELp+Bu5l6wrL9A=h8urlS_vH;;8f_wk53|)uXRZEVWFy-*
zn*;ly##G50u%Ra_9~E15&ieX)twZEEXV?0IsIU!o(%Dc6v$tjzHaT<VpJlK7dlzz^
zBeb*iuJeN*AJJ>qGcA&k@<6&{Ke54rkarP-JN!jY2hfq-zcj<6cJ9(m*dVRJwwfcS
z%SD<=Q6U-Y@#D*8nW^7e#tp=!{O8EaTt{&8v^8f0=F#+fNtNVnZnzk_ko|irwfdch
z+vSt31$Ol#2|J|4qbfQ9jN(2a?wUD^C8k#uz9_7qHpHlEIPvb=7`#kYizOCWap*m|
z@crwo<%!inhr`kL+{<lF9mdADzbMtQn+kk0quj8W$p2|nBr;^aYQ}Wem7j<dRaF>3
z(yva1yBmb#1}lV8D5XO@;yTfcd@8w%B?<TZL+-DjBKF{o`57IazZcJT;5#eS^`O#a
zt6c?TC5r4Y?=6|p*k!WY&+!E6E!&#58Em~)^T6bu?-knZQ}%|4S|4JBEEnP9R^BKM
zwGSNr-pdy)>zX8po>d=66Xf#!$%x@a8ssPsbNA9Ku?RABMO($TYi-Kkl-aLEuJcDx
z*KkCTt8I(w(AFA8mOTR8v9L(vXM{?Q3Se?lH9q1#9_Qbg{fi<cP*4~t`i(Mn><`AK
z9m}aWj`0#q_2aq+4_PCfXtp2mZYJ#Ps#3uVMO2*)gh$|izhgna7yxF~joYnz)~pvE
zvf2ANGtxqm&6W7I;G6Hm_p2+g`_X<5TAE+4%>_6?^`)wX4Qp$I_43F))_zO6yKYoZ
zqp%(7yIl_nld{wR>R?3W5Ec7b=yrH&2?@X#E!WGJ-7b4iu_e*cYzj?z_F&T7lp*`_
zsF35$GK))vq#&Df&!!wlM5MxY-+HDCd)GwXZ%B8BpnIIMs!E`3?AU8RVMn6C!o+Ym
zL;K%bT53ATbn^w9zIg5W8FipVq3)8I3=<@|JN(qVA~27>muoF;wD;g!r}EO%i-QLY
zMhlbW>t3<{xjmqNw$U<rG`tJ406Vs8;}_w<Vik4Ta|T#_`lgY-EuSRqgDJP)EB$k~
z1Yy&Cp=Sf+#u<9bHB9+C!!uGSBms66TdIvuNrA$JYqHT~06~-p0`N!L%B1AP!ARZ`
zXp7)U2uYYr#2mMb{yOxSo9pm#B1~6FA9pt2htMkNyR_Vr0#!>xJMQLGt3{vX=euqR
z!o%+T&3?b<GxKkRg+5t4&dewCq{>mh$#gH0f0Q60vt1Mx@?)l1qWSTbB73^iu5qrj
zUw*mcj)v=A(b(X$V+%>`jc2x<YFkGy4__x+XLU)emxSdtSs_RK=N?ne)nQ|AAyt}F
zJL3*|io)0Z=&4S?8}F^#KJSf>&Uk*%)dn`AJbtOGt7~6t57RCZ|Ez=LNcAN9G-%%x
zd$735wHC5C{}F0z%UjgkOxLh+&jN@3%8isjLt!Ks;)}N(D6X`2Y6jc}zw%0(gzGI6
zfW7#lq{de@f<M}NmLZ6^Y*0!rM$QfL`(BBj4)-0^n^XIDRcC+B-w;A1bYt3uF(&S1
zKP&tQILl~%WDN3E!v9RO!T{hOuOBVD+pwp&T^4F{V8%nsQuN}*1}9`!F!lXeYK>z+
zd6e<qkCW#md9->8#GG#JA=gR}<PHYUa?w-V%I^@m&A7}oFk}=?tl6}4V@yD;CH-}L
zH6fhbB$lvpK$w!v@z@cAtT$=%UGOZz*3U#1dFN_wmFrfGAs7=nlk#L=BrJ{UZyO)2
zod{A3DhyyE9c~TE?FsJ44-Z&I7!#T&E|X*e4J;1r|G*ppN#fVX0An>x4_3=rF~CNm
zNd}jStl(*2M@C2AfOK=@$$qk;_&A`K41PCySIy=MTys!J-q67^H8A`0D6dPK?Jy>`
z=?Hi!GA+KwB4cK(#apzSGy1G04geKB@I9rxgG9&pr$<U$L18REnL6<^7Kla4_ZP_e
zw+=Z0iilD*#A#GY3%RE-uroU9L)SpKhl;R2-RiLvqj0Mw3c(Xw<ZaQuwZAcU>Lfl6
zF!4O0xTR<+0;`zom!AARNDUC|0KQ#y`Kwx+Jg=<dk;r*|KS#U!2Y@2XI12?n`#j_2
zJ-9(lT&dl6@r}GBA|06N(6SCV(cVQPoY%SL$<7Rjq)|$Td1JO{AI|7K;4KDJ_2kOQ
z4kqI3{byt;e*QqHFCTPY3rcNvc()OY;8c90B2kf9lAqf^<8<EN)5oy9up;cxfBxcI
zV6w!~X|EsZ0mLQ_x9pcu5=`4K68zUuK~1)p0Q3z%Tr~gtfuy}0m^sk08X#Yb+acBB
zdaO}Lo#DC8fiXdj^YL1Jufo)ex<Yi~%5OLqmy16BbGbBCL@W~NqW5TjqvDuF79qzG
z&tOlEv{-vEJf`2>zk0E~KAyh&gg@vGs_GjEpYSWgSd0^o{k+9XGW1x4f&(bVO!E?I
zTVmwEZirj(CG1~IAP5unpTDpyos&fXz7g6#oo1tT794g^Zjr!sNZ0MQZRTw`eR?r`
z9K}BuOXBCbuU=6=>PP%yK8DHE%OaWF)MJMar$z>M64bt=Wnm)G55E;hHX|sEi9I7=
zRstW7@dF7vtL5hAY28LC*RR(1N-4AE=HCk|*H^~4SRS`imW!}(D_?JwElQxA_zrVf
z2~KNMB(RLG%@bnO_X>$o<htI1<$<LyK45hPF=&a;m}h;A_%uSMgK7s*XYu+S^RTY{
zv{3@0r>U9U%>z3et2KOu7yFEX)5XQOY;H`9j6e#v)5MU8b?d?1`|(FcRltaDy*?IE
zw*yL2(D`M`!oUD0^AO{&LYYNkW@gl18#6V&{~h;G%YJ4qB=ox_gl%!X6d;J3G}1Hm
zf}TAr15>#^bZ+l-j2F*Xlb&NtFs)@*30a%X+SSTbY`*&Yn~E(tFkk9K9r*oQNJgf8
zXy_3&wcgrD5mbEunLE)JxfDhGAO8Vq6jfP=goetO7Z(@5DtEH4^0)?6?}<*c#iD*>
z09&k2OiZ}Bxq(87WW=LDqxR<JFkse$s)mPeduW-iO#mDyk9<t@?Aoh&e3Km3@U`1j
zLiyG#?;jerCuZ(nzWBO2RMz}K?+s8SbH_X}i7fVe-&#X63=r7Aw7<#*qIl3pvUMG3
zQMpaY92@U?GlU+8hJ=`}4h{~oi~K4mNIJ&v&GW9VqHnuDJ2|;I<-;;oq+nP;9eTy+
z38GJ=r*PE8lterK(t$a_w4XyC!&}00`fUPjR5nRXV&?9xM8(jw+o;IGU9sgNzM5+j
zv9Xk)L&b%K8%%6^EzzHU*5N8c*4Wz0kb(EKFH>_oO@*b{KOo@g@<U3>59@axSuIS}
z`Yg5qrN04dYe7&zK!@PW%1TSYgLVv1J*f(>mzS51cl<3Z|Af~0B6r<C$10UZF*cj{
zU?%&nh*7KC){$sVJM_YI@tz<m(R6g_pQ58_f=V_14_HEqQ$Lim7#2&~N=iyXOWD6z
z!VAM+eEe?zKnSr!E!0C^0EVyMC>E@z4nQ4f%5<4;D%>H0?B^wGfBWA%1Q(ha`a2x0
zLA}bfwoLG@L`9i}ilaJZH0T?F!-G{q?dvA&xhYJ<m1n~je$bZ;5Cl$F^mMT<q7{JL
z>D*3D4n3*#<&X9bTP4ynmT%KylCz#42TN%Vi+y>yvV04sSKy!N)k<`t`3dwm<`-WX
z#0gSIuNgs;<4LuU`(?v5k9?haoQw$NWG$MXK;oOx+}td|Yg<Ln$h6{I$OX9htD%gF
zX*xF|k|J7?o$#inTS@DXNE}l5+iJpQgvj?<4t^r6#KFM{2nAAP{170QWPb-jcFg&}
zbu-$~@QFB1J|&dhP*?~=sPv7E8M(1OM^4SnT}V+H(*O5ln-bLY*iFd<T~2^h6i@E!
zdg>1=7S)|H1M%_pRly|R&!#^W@hkTzxIEn>6&lDPgy{1wy3Ghn-_4?{_wR>!14Px(
z?UoT4N!IG+?#@u9Gmxdt{$#eE8yo;oo7P=OZ7qG~9{Tl8f-WO7lN^Y6ZFTkZ<QIT%
z_{5iMEEtqBT`6(8pv84-G&ORfB$a9u(}wZv7W&5ZSAu+uv@WiJyy#})%H?Jq+l9{G
z1`h~{D^ts0!6SB(#SYA2`uCMD-5bUH1z{e>M1(f)`PU+eMPB2QOtYG8yMWr@{WsnH
zV{R8)DDw8M27eGXswzqMNRk{9CH<&uY!?H4<4zFm<R|^u8vY|qIOi06O~#K4+o#**
z#ib%++)7GXrJ%inWf_$4ox6qyID!~jehm47rlJ#Muz8l(juc(w5N=KPQ|B#&5uXW%
zJ!|oCNK^06qBve|Y7<S$B`3hV!awSJSFbwE82LzsZ-at7Cb-dS$wT2LGGRXIjyf=r
z_g-g!31i|j;rl8)nIK!g!OjS&ah=@FZWg3+`>*84=4&X7A2cRQXx+FjzdKo6pu(6S
zeWl`gFH%^ei6pWiaeYiCxQd3??h;a|@LaKd)&n7YKRND&Ux{P>mE80s43(QZg;T$M
zgAKMF1+_VoFR&hAHstt&ZlV^J1Bt%ZBdi3(#;5G@;_>7ozu52d0`_lWS#(A|U{8Fy
zcQMz=A^5xMdRHfo#M@<(c2=zmgFH2{&^Pw`D5~%{8pU*fm4U-Es<jS4K_+;_<l1$;
zkc>=AI@p$IiY3EcXgyF5v?{G9KfMk7@!+Fq?xNA|ZH1HP7gjApCBh$c8y;g%b4&)M
zbmY8$&8S@#(UfweC6P<7=!a{&-WO?B7+t=Eu#3z0ScXuzEgQl4JBF5yE2jAPSy)+f
z<UDt58ksS;!s0G#g8%vmnbJ-oD5W*MpHb~ns@a(9YnqCG8#@O{Y3>vJZ9)Qn2AtLI
zd;+kb?;xE0IZeIxHuHE4TjMz<r}O!NtEAYsEFuJn^jp|fcey&@aIJz^dXaZT?wmia
z8^6-ll0HLQ{a)y<{81OyfsjTfAr^w%=&?M6=d5M!`-*Cp)2uE&&SHCrS%MxL!QgUK
zwnTAZGefv#k<Hgp)LD!!^~)nNkBTEv4+{Kc_yc>(j8+8N1Ik@9i7t$V>GP}h*^fv@
zkZaTl7y5?V=p39zOs`>-*w^|EOgqy#U-#AXLoOq3M|l4XXj_|5>uh{1i&PZ!{c3)@
zUs@IF-)l_~Rz57K)$W3hAiLBi{3>Xug=+xw(mEdx;;iIt^~{XBdyJNTx;HC$`b+Ek
zi3j*XS(>RnOOd;7%WXWr0bI&zt>8`PE32mw8fYydu70)n33Z@d5mGY5n(EJhzr2if
zucMCTx}hFSwK8gR(Y5HF_4|RV%)Gs8sY!VaW>I9sA>%5V*=Q>_n>*^kC4XOjUw#Sk
zv>Brpw=Io1!^(NjYG+1o!U}xM|4FT#;60LY&pO0m;@f-t%7}h)_B`yHHvS#%0K(h(
z7ugG4+<o`nYkm~~e!rT`j>PVqF0>0oC>IZ2rot>)SAWe}UVGVWf(*7|q$ljtt~%Y-
ziAP)Fd$Gt_#rQMpm-a_TWu@ZIeq;~k$g>w6t>}+M*b(HDRE<LhKAIWnYEj`w)WWdC
zpY6XQ)2*%v7n_<^CT7~p%*k0h>cO`u`>fg?_yNg?1IQyDS$kbP*RK=AV4b`NNX919
z++MBtDuUc4!&@$r$E~zxA2H`HxzcyJ+5ekuWZoE-o{JGYTm!kx9T>`(vd=OB=6dQZ
z?l_>F|LkO;Oo@hdn@qEhYn{xo1}|Yf`^CW`L=^I`do9tyxB&6E3!|N})Saz6iTJ;4
z9gu$Az3+M&T=g+f^b+DWP?I{3A&m6b{!VtsuN>XKWz2I^t#hUVe2dh(kdn?oFEB#1
ze=o8)pT)1#C}6q%oK~<_>APTV3>9%@(2B<scr?^McYNKG{cr_qpr$msFu*>z?v?&u
zUH#VY%}5aB`IDax8F$gl(jkG}&pZ5tYS)Cql9=a_edaBt*(@X}S8hgoaaDjr1|wVr
ziD#Ch7U`|LcEN9(V=$v)j&rZDgYKar$hA-M?SY<121iM79De0_&H*RS0$NzPRf&Z<
z_f-Umzc0?OLWo+}M`)K3kDM9%X%F_<0z6eF$qi!H_o?C6AHgWb+S;A4qku4!ZlxOv
z0L)cu7NF<_Azji#(EYWtwVfYsZfb(liT`>M$9E{o#m3fMSvdhbdZwMRwV+vIX=!<w
z0X+EEfmQHqh<Cj-7ew&N%cx^ZwKPdWo}-)u*$YDEa$QFp!}A`3JboS9W2yW3?_C;t
z-uwFcV2_34rFU>x<KR%N^+i0v;czzSg2I8A7^2}J&`^9;WZ}8Y3d*id*4EYuk|5wH
z40<`37#M{afI36nGA}nb_uaeEK&}XCuCqgiWd)!M3epdqot@Cz4NTliXQ#(S8KFJr
z=nTq4t~>e=imHXZe5l6D`l7K44-ByEsNb>{p6Y*lVz$r0#x^~ao11GWFA*ZXPnT*)
z@X=FAeRJ&J1GA6(r#XRvy<w*1FBBEcsT-gbx@#n~MAQGMLMy22HLdfNm*-KTi52ED
zdK+;6q`&WfvohcVd%m@~`J@_1o(bK2;Y;xo3aUfH!nUvX|F45j$$jX>2Eo~<<<0&5
zL1)jEe{5j5fmAk+UqXC*&o(;SHfs_Rx#>t$XOy0G9YJPbc|KN<e~Bj~B;?c#)YS&k
zKyBO$daxiFvvnOvEQj+9s{>p)q-12Iq-IN~Wg;cJb&GXgzwUxk_40^3lhM$*a;td|
zmS$&XpBaW97W_KC95!}X)Ya9oC@M!0t@Muz#vz<JN5OPVgnKB6n>rku<?*=HMxcCD
zFR`1LunX$&ufjo)zwrJ9mnkTvS_o3f{>gz^8eyB|)rQ9SPbHbkX+ZZOyDF_ZUFSPZ
zj&q0oxh2g{I3P4z%@aoF)%N`O(NY5>cs@jAdHXPwW57{KAQrDS+>p}%*;UU&Y|PA}
zd-AfgV;bO5;|-U=;6t|TQ$ZbwOEf{A!?4Ta19aiH4GgH7-MYbF+uF)RPwzdK|DpP?
zqrFw)8#iv;x;2n*q;mEk9>fqqzoKj0@9Wn@P<ifu<v;o0K7w>y#*U8kA*}q5xX-<)
z4k*}L!+HG4r)Oq*VBYRr%x)^pRP^HneBg=q^Yfc80iBNjdSH?^bkn@UBaAZKG*j%7
z-EYvSH^kjlzc|>yo<P5ZAc;afmS_(UWFhjhY&%{hf<!ysdCdO|0oS%>i0qP}z&=}?
zgz|rT5wQvajOg8%KR|p4?o#}vq?A#~T05qn)R)1nhzJI~b2)XXN;s4EjtY|2`_>`f
zz(=f*PKj9E+vuGAPBYyjjTefHNefoGa>TtB@>EF`kCil37p$L6V`$0EEHqrdt}5NR
z*cAJkM<j{C$9uMi>FUJG?!W?^9|~5*Im!S2XpIR@%xX5>eHK3=gOUE<A4kDj^Q)?a
zb#!!&GP9mEYyBDJ&qW1y-B1t&-wn2VL>5841s(_GDp(SN{_hVNq?njKK#W<o)x-cw
zpCDe1XCq+<iY8G`TVf=PcmfE5@k#8dkRqIpLrTsM%FQ50vZYb&h3zO*4gC@a8E~fc
z^)}mFuZm`ZK9gp7X?(oLI!N#?4;N%cMvh$}ciMnHj;+;QNHKmE@qvxa4}rG=rPmH1
z)^|ObCLed&ZiW9oOrp&R-;saIOcPV9lc(n#$!lu@=@7_rax%I5_+XB|QdR~E5h%(Z
z0+JXggBFC7!1bn<7WM}Z>_EpxEzjVg<KD{1h<f=lv?2h3NKC=nzb-feGK@~6ryBpb
z^ZpeFvR-C=bMsB8q7Zl80E3OofHO!?`}pYUt<`EeS{%6k-JYirkOdARJrF2#3Zh*0
zPqCNd2=dPbRjmw(Z%Ri2vG<QYYWyBzAVo7{0;k)-Tm7Ff0?v1=UUY|NqR*Z^6BX?m
z-XFCcZ*6X#E~b58VBbYH?U%v8#FWr}A%9ItN!b?8v)!j=3I*fd%f&9IkCf^(T@I-@
zIYD8*4Af?Gl9Fipm0a;Ff!BB<Xdd`-)sI0Jjn`(y4g?i9Ha28H%^<g3V0tL<WgCcT
zyHJ#sm8BSW&PR2I?uwn)0F)9Q5uur^?kOcDMR#3YLn9>%NNr+ZzfoNIA2yFg*BGF&
zfqIU(eZR6Y{@!ws$F|g<5A^==V~LyT5C~u4K;L1ui6CQ{N2w$nV2Hd}P?brc)hsl7
z@2v+kMh8i<R8VJsufprlHs_?dG2qIBbS@X|oSd8kW|2M`h3(iaOtEC#vV$ofF)=aq
zAj-5IIuWsv>m@n)Dvw|(L{sLNO$x7-zf3n7FE5+ZOAM#wS@0JNYaDrk@dcz(6v?V$
z_xB}O%4$p<*M<w$AxWgAr;j~m=Z*O?^XzP1@Vs+mBa;S0i(c)6XlldGH9?z|Tl)`~
znZKS9C9|_NF(pyIe4p1T<u4(6+?S+BkVq9@lnIo!`+xP(w_Z!m>T;%6V`pV#{IU$%
zzR{?yt-PT7H|Df0Wg^!I3QLEXREd%ytRQl?CUBeEEPC!sg)*993>{F(Iv6fh^T+2s
z;Nr^U^GIm0kVm>f^QPj%6c|!Ya;QNF5Jk5MD%u3H2KszoAh%gfDT9+#K^6y}4#JSR
zkJ_MT#D3m!i!uuI$U2Eb$JG-mqp=%-7cFXMkAs!9sl9!E2#SM}ZhPdt7i)(~nj53Q
zS=`RS=Ecdhs8cEU6JC1F=UHetG%%Uz2L19})N}=LSivv9f<TwAmbM_A&%xYqH6%FL
z?0Wt?=eNq*p_?5~Lisn!pnwF#e;$tVSS{eTVjb*(h>Amf7o+zY?4%lOI-o?c=x3(D
zADU=DxCRQ^`%Aq(uBr?&Vwk3;rs6DTC1~Q9p+E9L7!<B&zcgLpq4<V}&JEUQCqOf>
zf;&rBAp}t``DZ5hFtVtnLP<cS&{Xc9$tWbZNl*P}GKRJixWr{;amRCjYh{SJ`xfQR
zAtZja59qSDnU43@_jh)_czfsM<}P={RD<MC;T{NUe<k7hPA)1We2$riF1`<OQ!$7%
zmNx^kSPaC<H3dCaPH#ed7sR5Om~e;P0M~gBv>u<=3d_jISgL|7o?%#!oC%LT$W8BD
zn|F=x`L&z!!c&Q!fL{`Jd`BAC@fB_tKOx4&r*Ptpk&U^AJ9-G(7*Nh|xECCJ>*yYP
zb=)z#0*}KE?YOR?;b5W^@tJmdPXHl0I{F0~_|AKi35ubB`6|x#s+1e;%nxRbrhJB+
za0t&#OiXqx6sA9tyF52-?&^{k#e_AYD`yXlJaP6+=iQg$cIXN4smujeh7=q{J|Zr!
zhPJ7reXWLuq*5m7gGYYpzkW@rkhK43oRV+|Be8V=l%u?lLuS;T3Y{N8^@m6LY@YVT
zw~oM^0Jf*i>H2+8KuWxM-u2X<+H&N@H3D^MoPBh6Kfmj7=zn`Wz%|Q3@KAeMXDd@)
zNbbgdbq0UFiq+ML`2;FWcSiNXjR@PZBA3(Rtm*91Qa*=V3tpC<$zd`&r>#aBQBlz;
z7Y3?WVlI<XDC&ZKu&S>XLs<eGX;~)D=u(Tv5IT30&s6lM(?5C?8riV=IIYLxt;2G9
zfy?=+?cXn79Sf2<kwYN3iN4AgqGH$^dw=bthQ!-UI7ZEXJgk1Aob)n%ts}-goEJ49
zFtC%j3|OB-KSn4L+`9GL^g`&F&2Qg!!j@<RWW&m0idJ8*;o_aWfY_M89QY%F`rWkr
za1NAg3r|_Uk_)!wD_GmbY`NSPXG+S>HV5Nj_w($C$6i**#jxbnULDAGcYNM0{$HwQ
zw2~;ipJaf#yHcp*ltJ6s73@2T^Ze#$hQFb{kc=NFfY?Q^foJS|IZG+@ZMNsBCHG51
z_|9Jlf663!i9|pf+8pz&QsXaiBYN>z*MY$FGB|ST2Q{%v1ia84cJ$`XojYB9*>k_&
z$sdDs8?2+r_4Rgg!%F9BSiJC2Md@!gHa049kGY;Zhu$!kncO=d=drl|j<K6017Nie
z{AVgZRq+7iGc=@Y(q=0KJ(y05uB~LnRJ?nl9RVL0es9mWCt+(3T~10MRs2ev*<-;v
zORo1a9^&*ZrlDK`!(pme5(1V`{ti+TTTWm+b67VQRO^}%HM=Dbok~xHg$+t#z)hWK
zvnk?&sY>`$gc5fS2NNo7s|LX0mW}QH-u>%2=~s~y8=jP@ZW2#w(Y%ZUTgHN1BUs#0
zrO6R`+xP@|cMOHAPw5VW{O`cplpcJ{QsC#;4oX;(%f1!jeiu8iwp<bO&`hi=YzEGu
z(}6D&I0&djSN`2Ch6YP)S<2FOc6JPW7`eGs2bdCwQYUZ}E3B6Zsh*}Or!Qa+fg+H@
z@!PWj8wi>&lA(*Z1wURLpJaOs)HaV?W4_?c*5H+%pUewhgdV55L$V3SBiEB#rggRr
z9m56mpg<|skKr6&<cB|9R9{Ev7YJ6>^7Ng%$ni_3*mt#?l1QPrFfLVGA{peBi#Fic
z0{_N^l<AAwpD%J#!7|Ed@h6>l6<j#IE}Y@_ZsU9$eVt?*9=dikUlu-rmLovXFI~Zj
zuNs9vxGs2N2Ju*EP=dcb58VS=EZWPoE^vUt4&{`sO&JYGZ{DgZX826`kGbHd{XHcg
xJe?-Z<zC-E$YM}+g!9P%G5Gs`>W7VU6c)}Wo|I&cI}pGkVj|K{GlaE1{C|=IFT4N%

literal 0
HcmV?d00001

diff --git a/docs/optimizers/radam_r_t.png b/docs/optimizers/radam_r_t.png
new file mode 100644
index 0000000000000000000000000000000000000000..7b77edb985f7a4d1632f6717c87c32f8937e3ce7
GIT binary patch
literal 30948
zcmagGcT|&I@GklW1QY~Rno9dDNCy>x&_SgL3etN~dMDCLAc}~9f{OGG(tGbAQbg&}
zI}xP!5^8|EiNAB!J?pG>FaNk!eDm(vGqazWd1iKko-4~!l3yo>Ac*pb!eccEB0@nB
z411mkd{R@YR0=^JxlbO;Xm}W{OptkKn4?<}?h_;v^;{vgFk0H7hw_(bNmO|_xH!la
zwJ$$@f0IigyvO^(?XS;?FWf$No;TuLuY$s@k2{K7i}maR>&FcOH=a&jyutqMsj_d9
zWg_y+w$ugZ`BdYn7<CpCuw>6P%Ozp(uk`jK6byp&U_=+ehvb*Q|Dm){5HSRm5uLjV
zLCJLgVZdYg5R4RptVqt^1z+Ko{{R0<$*aFVKa;U|{3j`C#Pr|;8ZCKTRYeEI!cLwL
zTRppR<A!?1)5Xzp+dwMj8kfcUo?B+4H`iR;mk0C3ym#~F=+9kE&c{euL-}(ZF#`P)
znMuB0BPHf!)GP*Q9}&Um3LRbDpxVinhSTHqYXTNl4%s<5T)e!RToJL1j0GhnUwjS5
z$)QdlwH5Qdd-qNchZ>ALXCpp*_z?RsCMKpVpyWCuV{Y-f;Q(b3l^O@_dr@h_^LM3}
z_cvQvSv{~YVPWAHwYj<6_L-cu$M2jxJoevSpkeX+`?)JY>~yQ0U+|uHvXE;~KxMM`
z&cfy|_9VEN7&VLdeo7$A(pV)wvQs0(@vz3UHPoM!a%*o-d%eS^f#g}k^Zi@rSCbPX
zBO>y3il+k@jFQ=o1iHmbo&G!=RonKq>h|281LEP<CXQH7_X9U1MWDMt6Xh<eqqyq%
zM4HN+9yi77wFynW$A`$>5lcBaIrU8CbjQqRaT$}H{rz9bMPD|bSEo}^QOV%N@HgOJ
zMe!MG6&O?lNgO6>j;BtKSzU|0_tqFSS-e+@f%5#y6g1ZhubCVMEKTJJnYYniU!AOv
z%cd`9_1(XbkM%poX`7mwN=QiLtW+Up`lWtqz}5zGhiFv9i){wD4nLz%sMyO;)(s4l
z{~tep*p+l$o@ZdObZZ%z*P?+t_MKL1<JDzs92^SA_7j!K%cRk0&B_=!9Ntq&EroY_
z`0$Mv#cX&*R#eN%8rch3d3p65seP<Jux5tM>D1|oySw{dEc5Q_n7q`&pFb%lk?0KO
zhmRiFx<SzENWWevxtfiQO*Pd@mYYhW-?sDbUo%&#>{!9HjSZ*Kk{3;+xJt?kPV1BP
zwi!zsep|oT1@(!DiLKjH9tHMK*-$#Es;W+*k=BNZ5fSN=4^N-<WU3@Rr39bu_}dN@
zXs$~eHyo|iy8Dtq{Wng+Ng>|k8{Akt-p7?{W_sF2j*gD5tVF}_VaIiPY?$Psf-_lc
zSByaBoc?Hdh22Dr=5hWYq!Rleh`uw{@z&nbz%tC<_h6O*N4107oANU*m4f=&2(L|3
zisI{*I<2eH3>b`^sBtx|bk4elZEI7^h7B65my89<5BTI#ZA9|w_fHK{7QKEAv%e=G
z&^Nvhd~t7isJl4y{rmGcs>UQD$o0$?rh|F98l0VBssrQW<1AT7S{DgN8KBQ&<5JS-
z#15&G%`juX<8q6T(dc`o3sRStM8>YdNUPX^v$;N&m6gD6G}U|W6&#pEm|0r(Mfx2t
z8-ruC;zqU1@|N!E)vS}<?99X~ldTSoP-aGk<nd15dfhe^Z>{aFw!po6DH9()eiRgr
z{Q8xajxM`F4h37g08XM{WMt$ieqO59WwFHPVCzvJ)iG|!c*#~RQN&rU-d{9usqd59
z`eeQ&rS;a|UxE#<wNg({_8Kbf$1EHi)@Em0Vjp0Pk+=bWYRM|(ky~f@=YBF1Zd&rg
zwA0<n2IO|9@Vj^KNJ&YZ=Q<Y0t4j+D3lA+Xo1nZOqF{RJPEH%&o`h$8*LbPt&^sjT
zFrF2Y<9wB#UXIu2?`Qdw?WEI%#YJl)O|F|ar&lYdG9q?}aYpacEc~?CKjO}L05qfS
zPw_i?$OLR2CuG0Wm#NYk%Dfmv$s+DCkfV8UaImyUeEu$`9^hq-QbJQl$5R9Z*|QQ|
zOs)l}dJVv=3j@)`|4h%{MXdpe%r5@li29m}L?`9>Z8o-9@OK8mg8lqCI-L{c>rVoS
z^3XDj0DQ+}d1<!Y7({7JaFnA9*&(N@0KSvA``Z<4VxlW+ds3jjQa@=|Y7MYD`23sJ
zzjcVzxZj<j`U6-t9RcxUQT@TdQy}P>v`q^E_V4^x{*r+2G@tbD5wQP)LjF}kTsZgt
z`5M7Hn1X_WB;P}8fHn&%q5#0*=+2Rmg8^;h)t;0l3jR8$zdtV{3<CQzsen*o-?Z}X
z!79qj4J+(oA|f;;V4)!)M?(!KxG~3ufPjE&*RM+g4A|Y>omMV?f?NVZ);zLlxlV(<
z4lMCIRTg=;lYOweT-d-O;Z^p-fFfIk8ED2`Az#Ot#H0EB<?Z?I!h_w7KTY6FW)c3s
z{!g0{cX(r|X+fs^;drWiwCP_;ithiyxnGm<lUyCM_FD<>YnnSWvTcxHsB=%MwJ@7~
z;W3JgN^e)Du=J<Vz8GpObT=(9{B`zBsOe5s*+lbesnB6Zsh5?jcO~p9nM+2zKCuky
zQyYxeoOYRp%r)t3BV@LV54vXXb4W_>l>wjlCSq#w4XcX4nHK`Z*FLh9zwD&LG8p?H
zjt_ByQ=`MfJzW6FuGc!5o0}h0W;>dXZ)8qNbN;R-^K}+H+`P}?fqf0)>G?2!0jl5;
zt2`lk6D0DO%*J+K-SK?-Lr=SE_WI^PhH8DjhL^jz$r&1V)}*TW<}&}eshH?)af?m=
z>=tSZoP<UB<P2FUjTDnWf@kyJ+DCy65>5;HGsFVB*HxxW#gTi*61ywfQWh)et+3TF
z<Wj551!~jI)|6Tabj#TedQw_{SC2KndJtJ~uIja;jPM>g{MbTrU}&Eqssuls^?Chm
zJzlXFZ_Mrds`=POP7d)SJuXV<$S%=Nz_Vx(i@t+gmLz3jzPt1{i@)1Xr0>NXE1zFp
zeZ?RL=I8C^=KH<9qJxKujdX<?h1d_+=<(?0H_c_KWh=caZ?@iiY5Srmo355kd#V7^
z5%QWVSlJMI-0E26+}M-wJLPRMI<J+3Xib(UWIX4))sboM!k&MP2oft73Q+r#wC3nE
zVS+!{d}wPdfV6K6xS%v1xk5dvf=Himpqt21(+Fw+*_d8je=zW9;D<1-9d`REbOKsI
zK_PiHgn>U%$UfGK6=$tL!N7NsntH0<+ub(d-CrA)C$QbJ;3vI(X^`d0ecC-MO@t?+
ze_~Ii-dKJ7ME=On^-r$g(4fKlYA1oyJ9-pQj~4J%Q=*lX6`v!TF&hDkj^x#yhFkKX
z{p0pn%xh**mxYm1OIySDGmQlhxzl@nwdP!Z4h);{!)6IRf3@r#h27}cySW*)O`+8K
z3pE=rS6aq<4G@k%CwIt6ayjUPg(3;N=zP4S?)ZM@aEJqgh;n?6%>J(Em9HNsAKdzu
zg2raB);VErv~?^>b8(z;2B6gw^K0B~W+TNsf}dJyOV8`myL>rdGFcmo>Ep84CM!Ms
z9HU}!C+YsoXS<>QDrVU#3b_0HJ}@(LpXB`bqIWNbY`^EBJGoO`j35czx7Q3YUh;DL
zrKNFfzViB|(i@ER!guIy2Z_NTRBAAt<#y~HW6OKmIzM&i+}|1snuKzXw2-XqP_OI{
zt!!MaaEA*lbPGvzYn?_dN{#+(eZAK#RbTu~>uDEEiz~u=&hQUxvM8L$$9BL55f-VI
zoh5$B$E2rN=Ii9^R#G)DXGPaN1v?0pp{@CyyieB)Uc2}!D)aT+%Bay?-RBWr3roM3
z>~}Vk*F3}R*P8}iX&6T~T1WYOtZQxW3it-b`22{f({hgPd^lGYTfJ@~Nv8XWEu_eF
zQ~Thd2=_aw^sSeh*=IWbC4!+d*Ie4po0&gVhIQFs=6gDCl;3qN!I|M2-9u;MoAHkk
z)DTKYG23ibmPhz?`njU$cAkCO?aA&^o7(q<>S4(5Q|lAI!}9&)c+4NMX!P0F53dOi
zr-bc)8CsG*u`SFqO_W<XRHc8}lFF=`Dy&3p>iHz^c2r@Uegt9t0FiW!DG4dVpko^%
zk}Aw^)$z0g&)eP?g;~3(OGWRk_;h&2zoUTcNso_m9)$7J&=eH+Z2KH2n3ta<B*b@x
z`ZT&xo9Y`|<R)Fp)t#ETS5ZthCi012B@EfcV_8J8J(5n>Vb8cN(d-a!@mrbaSJ-Kg
zY>ZFK_@F|5bUbl<gn<G~awky#FJlu2CjHN!X_n3vm%od^?bVU&M)A=G(LA9rNKe&V
z#2B8jUz(-FgN2{I9e8og*gQ7W@<?tY-LgqaCtaelt~n|vu=Q>T=L)_X-=r62*fI5?
zVJsfoaz9N+RV%mfc^o*IK;N9IBD5xbvn(3MUK(_?@4X)|>MdR;yOulUX0(K+B@-c*
z=H}p^YrQwI^LKsFXsmrGwtpyX#)DH>l3i}cA<T6m`a9%?MV{Q_jTo>NJk^wm^eI=|
z?5h6NM!v6i?VC36Fy-^VQlYrQDn%US`cMPg)X4-*k-Xtl!jOB#-y{lXRmuEXd+1Cc
zt?Fg#7VCzu35SUdQgyaY{*rEn12GLB>-Kn0UaZE9;9Do$H_UCn5li9W!k;zYGi(HU
z5tPQn_(=vPS=|_6{PxvfvFWOj=(o{X%Sy)Y1t2Kf!`$?sI$wtUpf!E4VY_7kec5iC
z;^p&Y$wA4^p~679ZE6R5hD}y#co##V^$onl3m)p&uLRAgN?Oxm`?w39qfDhvA2JDt
zg%mRk-~_4e(_2E2d)o7&H~Kuk16dV1xQ`gcHYp2NG7?{^2eihrk927t<m~M{MZZmV
zy_^4T#z0r@Hg~7|OZ_i|_r74^q&3N%-KE98T}LytcK!#yrGqS4*P6=rn7;;<CHKr_
zZq0n*2rtkd4lv%|#MF{b%s5TQw&@-JEL{HU{3)p;_Ih)n)@#3kg^SsA;m#<>s@tll
zZeFxBO!?%P8Zq8~KUhF6fk_rO;M}Xup??}11@*KkzS?Kc7FHG*>RhQ~Y54o6>oty9
zudak<Y%nfq+sk4wp>2{}&U}oV@J3=L8V(G@3A}~hzuX}K?x3b&fa|3Dk-)62W5f{(
zlm3wvN>aK1{l|&L%GPz?t6w+8@Nb7|g7Pn{?xOJ@m~%z&C09-|@XRyu`xPE@+*Sb2
zM97AR?j#cVY(|;VVms=+TM*%**I$(tF74JAR~JuTgv=gI=q;tM-|!vGJ23N=2rgF%
z3Cj1h^Zd+=|J2~PQumtmn0CFZdrZd4T+ycGFOx~(b}_9<oX8O;Muc7Y{Y(Z;49&q(
zu{I}${g!gD0Ay9mr8u0=h`+5D)roHm{&8#FE9RwM|Jp)3zogF#$%a(f1<w<Ia}R+$
z<mWT-_Ls6b$ct7x3~8}-6b#J>8dVOuaQprAL|pui6P8X|Kiz2bN6m)PwK2Zw`4+Y3
zI98fRxeoc?A&XtQ(&KQ?)tRWzZ6D^eBi^{F5ab)8xlC(vv>0~~kj!sP-}(#$<1b5M
z{dQJ*s<CilC_(e<uc;@y_bnLwW;1^|EbzV4Iv(EO8Iub3(^;;FD75Ws+SjdyMhPzN
zb6|CqWcbf=1T8kz)@Ztl(=5V_QtaLbBD?E0TwNerqqQuJW5HLwT%J1kC9bDZke<HU
z^?GIhv9D16)Md2a3sNenckjXpX82=ro@97)TZ$9I_FcFcp{}!99lIp<r|~Vw1>nS}
zK{d@^>tc*%zxa+@x=SXuQit_UJU&ddAMX!!s3sY|;1WC^W%%~!cKOR!R7DeL6BcZk
zUj`J_@*`;Gh(3uy3cCQ27j^TpB0znY4G>bN{=-PMC;oZ1pvvH{)iituYiVJ0(<|Qx
zuY#o-LU^Ua|2c<+$|dTJTB+wK?%v*(O{6}y4z(mUWL*G<P;-$<ZH@N()e>{z2gj4k
zb8d#(ZX3kDGV)VR_`^_hUUz{9R+WTUhPJw@0CT&3z}vv^;swp=(AviiMvW=oJI6Pj
zZ?;^69#l8F&|7zhc>COJz}ZbqJ1lojsi5oL1tiY6yJ%x=TzYy5uaRsJKoR-Ou3FQY
zn5&=m?QhQ}9p$XyxT&DzC$#1z9P++IYe7%&a~lsGwB4q)F-5UW;_lo2F(IWbtYlUQ
zHSRm}P}0Pj3e1h=9^3SQ%=F61<N3$AQ4I*sXQ#jKON(i$wmGU|U;I>7HXS>9-!kC$
z;Tq*^)Yq-i)+_Y~`ExLr4+NyJtk_pY8NO2qzApJNej<s@L5TxH5zp|R7_x$=f0@0r
zQM*2J@@u!DT}y&b&V9(l_n1>}W^xCknJ&He<Sd-{c!l$m4C6Y3?xjgxvD~|nepeQ`
z;jEzb9@zQtcUy$b655|tTyt+Fxs!iGIDVpH(>+3IyE%!Z$DXhW{ery|sFw2Rz5_mL
zdPUN1Z%i4Nuy|LU3D8<CBRq|fNvEZLwE6?pT8Ksc8Fvui(NtYQt(yGF&;I$^%Y0#y
z;p30|)0s~%TYl8k(i9r^fs^dNdgyF8c14<MOWi}ApU<Rr>QYoSh?(^2F|B-&PLxAv
zNaWv{Z06!8-xwHbW%Jpqhuh{+++)Z>v&J7LE2^?}L8AhCbrvmmrD%NPH701BDUD6%
z{VZP3;U5`8F$AaDuR0MVFKX>kp_#|u?z*mooJl*DIckAPAAOB#=I4?9P1i-MHxjh!
z9XG-|Cw!Nq<B}h_bm%`9s2B(7kp0irC)<8>mLIQtW|qM*d)S#+Jjs9)TS*1qANeSN
z|4}JsoKH6~aB3bX`BFJox9k?TeTm;E*yukL4ddHPhP>;AM9qKSL=1AoA%$jEZdql5
z!1!9DEOkQSslnnJYYUThNi;WHm|;|e0KH*&&j_LwiT&gLS-&8GMJ~FCTfA~=s05mX
zmTM>;PFfx|)D+p?#?{S*SqaCjtLk46C6_S&ti`IgpbXo+_F!pn)OGr)=$@H-&ieW7
z%XGoy2-5%91FrG(yh(^@>24eNQ1Bf(sc?Qz=SOiJS_1VcKtGIQ0j<fDwtR-;$jz>A
zAPxRRR5xi`4~IUz&LOAwi)aY1b9|h?P_W#l?9)`YSGuCJDGwzJ5oF%$c29whD49MN
zJ4eC8v$9eb1p9{e2MX^aaNM6hPuuzOAF59kQ+5w7txf*H3*itBc_jw;%TN`?zpo#!
zJcXepv7H^rAJb27)_RELtVau&eyviYE_2;+hu``F652p13+_{A_L{iFc?Kh@2lK)w
z#LyzaRFeXG<Rrrlr;NJtq3;EPPMi+>iRvaCYvGU;cMg|M9!05sqZ?IT`S8Rsso%WF
z@3=ps_n|#`0!F8aL~&wXBLn}&!*6tb<H+S=v=RDB<a2BL9h|raG26nkLS#HzFys;L
zIWV&_Jm1kCJv&ajM}|5}&__shvY)FV7cci)I;|}VOzV)(Keo(SKgyhJJ?ieU_&Qho
zpOVGBfPBxK!MF}BXPpHRUp0Q{fgsqkJt~)rb_pWeV~KYEt;eKP+>hiW@krZ~LqN&4
zz_)N|9V^_vsyVT14(shxp6JCO;TTx#S%NA=NRXkag@Hnln=%ROq^EB2ZdY<((3L*G
zK|(n?Kz2R+S!Lff+or~L|9S(LnIg2dN!D{8=&Az7$DOS~2LBr&8Q|bBUG<%)mpBXU
znCL-MDMgf^{-quN>>#r`mP}xsxywMf^uu|CC1hm*kYEgoqK(WYW;%RTc$41|F>X?e
zcUMK7IvqS>g32OG9{T1^&`fQZo-K)|s;vp1>SKZQf(YWa91OsyybmJ2pdj!)8h+((
zP*R8_I~Am-Bg-k3IPyv^e<lZ{%mw;RF<g)#U`<K@k!W;nz<qB;i5A;ibQdvxSC>r}
z<fqsZ{~nvKijhtq4foRrJs*pam9*RHdfhsGNi6LQym*AP>Tc#&GLJ}Nk}94%a!kS<
z0nrp1Qx_QSyeCG{e`0Qp7ryI%;$Bmm&)}@{Lk*8GphHm+3|aMG9GOQqk2krK=fgK}
zhiKV?5DpCK5M?t=dYki?xv%_i%5U(!0&cM*nh@vNJ+gC<A>bT<q`npxs5hu9XtBXx
zX%XX)V)~YSNF{>kO4%h@x{z0YSvL!OQ)_Cu#Wrun`zx1h-2@u#ksxSv-h2aEPq1zx
zj-QMUX_udWtd0bQbbcziF!XNjYuwK%o5{)ft}jCiuD+#j@jvi^x1ch5g6}x@g=oSm
zG9c}_zwc*A;g@~i5aIossHZ`6(umy7eFzGH!!S*EcT;MM+V|<C)3=OJ6$-%J8L<yo
zu*fOnZ;&<}eeeva4v4Wtc1$h}mG{qGGMBp-I15r^_sF_CNn7gP+=?v{u;f*OTH#{l
z3Ut_QS`*pqzkY?B5p9>}ZgF5v7Eh)xCL^qVDMd)&Hxt$zJU>s7U$WdJ`uT=nd0ox`
zT9cXeE>%=SM8x#I(FW}9gLFKt?<n7F#oR*942^CTW$qWeFb(8#R{9BJeu-m_8pPU(
zd5&oWx1F4Qpo@DUC(|v}_d~MX3kK?D+1$(}+Lh?{&Y>zmhGoWx?f6E-f_3bEi(6&y
zx{ei|pE_H@$Xc>tqW;e(v2`I<mL`Yz7aosDq|pTL?^3|RBsW$UH(nP8mwKnrZP&6-
zwOqsWTpcOyu(;9S@~)<Kr=^92hgV~q5;6k_0oK@wI}pbRD9hFfJ&vDf&rj9Yn1acZ
zebtBr)kXeSMSzSkDrjfr%e;QpmkmZA_$9#zvDc0`p1zgX$La9b&liqhvEq=*Cbs01
z6t4DzmYm&>Rof1wBicN3<Xy8QR*nSW&~$N`<VCXm?<-qI>g~2h9>a{h?epPKk4Ajw
zctvqOzqY%3XZT8^PvqqrYoySsKS9aa_rx)Q88Pw=Vv0aK+Bi;HEC`a#FqEFd%?7i9
z(M(6yG5k&EiH_is<_RcS0)*8OQaGDv6p@d|XwU++6s;wWtB}hNH=GOV5xm^wqYHm|
zB;s|Ce`>iFD6c10>R>Kt>E1b8FpOX%RS|KFdRXg+gU9g$+#I*wFNux&c--n@Ar(>D
z(quN(bl&{gt<z!d(?bCX685DW43vI<fX8hGj?m)od#+BtVUSrp4UMuZ+Lh6WZT(8v
zQF|ATKa*}ET(4M;bol4bEcq}NW-&J=QJ1XB%1XG_NU>?F5`X<p&m(Gy?GAy4lSA9Z
zdcL1Lxa=YRd*a~iwiQ=x8V8qp@w(_Lz$O{0RG6f_A1zYWddzRkX(o(}BsqPcgT_A+
zSR8tKdO!)mefFKKtQPWgOGRCmz8cNHt8EQq%@prD{n6Sg28X*W_L_fxL((Wol&w<l
z8<ZlBxuYKXLj|uFKd`KxtHOy{`?Uu4K<(<aht^D(ELRa2lhQT%QpE_NNVDl|pQ5Y>
z#2hEZaYN(*si614C}fvaPm#tf_p!!#f77wycz00W(YmAj(_#N|)TJ*i#Nr1(J2}-`
zMkeSqX>kIJ@~|$RU$mAzEF82T1WA=CCQP~-0^W#e@*+feqx|I;Vgn<i6RbaVqJ(#a
zMf=WTAE3{3iz(!7r&?LOpB}Hd*x9Xc7uJ<HOo+Z<echjF)DS1^feKTW^w}PB1j`tz
z)GFhcUDdW?lzQfZ7FU17Yi$c4VbFCso0i*Ta5lDp6G%m+JC+xabKeAufA9^st&Jaa
z`<-lqRvOu=lw}taB2P_AE7DJ2P3_A!@c#Yer!3effmeK^*}B%ccsf21iFapvA*ol1
z{F<uWy!Z=cykOE?h8K!TOSjckBwPVAhsBz3bLUznA>D9G1C5f8jg3!Vk6u6NUuIy4
z1{M8o=j&z>si4wYeP34*lHs1I=S(Pr7SYn<^_%sO(`l>(hSG<^Yk%X9AAVE5voa$i
z12n;k9;WmR2iT(Noa`X#qDA4O@Eu=zOZUCm1Ab17@&)B-=r8+7SW>$_GFp?N?$1SN
zFMEWzLxb$a;^Ja<Lq>)hXqv9c=zvb6UYV6@XC=FGd@`i1scF4(^9wK*!%R{aF}@tJ
zRUe8jP&Lw&+*pt%cT#*23?~-(_1=VD-66?f2i}lnmf~?$3PNe9aSujA`Z_u;E5pB=
zn{V^**zK;2B#62yDk-I{_GLWNuk$GOVg=l@Jd0=Z__#P9-&$|F5`W!BV{;IV?Qx{&
zH%Z)~?9nC^hpgxK{PPgu#&ARThQ?H7$5f+Wkr}t}>!kMD5*ii$<H7-^NN4qNGALP<
z&=I%;+BXxm?(01%#QE>)>+1R$<wKc7On+R&4FNpn)-L$ACJ7Eyx8%{vc(pV5v-LIL
zK67?ew6!>fW-o)skgIEL+U2h``Nq@0m}$r?`F4~Dk7e$ue*K1EQb%u7GZKQ<01OG<
z$8^dpsTUf5>*gij*bMSk(_RQEt<~|i=OyvTUDrZD_IjEm+xIOPlX8{UKdb1ZhE-8R
z7$hhW02VbW(P1yhRcaMpwshajpQxn8Zr_`4ft9H$vQbpNs3q0T9CA7zQr5ak10~0R
zV%Ue+hc|-A=jEFhqh6cPJ%GWpPTPNIu|0Xu5v2C2ikJj34!fmaK2#++iU?M~S>7T9
z>MC~Xq66*P-b%a6mZN5oW@^Zzm>uqHp39a!6@N3g^mmt}e)QEIUYN`3R*akDr-P&=
z2=GO<t)NOW{2%g&3X$<qJBNY9qa)@XIQgI2Ud^HXjWN3Q8xqIFQY+V>WLY4c9Wm2V
z7qQ*uZ0LuC+U6LcrI6<*uMQZ13S__D8WYSCttXpo;`N3l2Y><%JUkuQK`id8|L*f=
zc%Mc3ydwEV*Rog(YyrIjcbkq|+Hp8xrGR&>=Yes8l7C_1PaCn%EBs(b{SkP*x_yQk
z%6`TUe4}LRr7v!7KxJd6V5wszs-#!^Z1?m4EI^u4<a~vCgF*lP-a3P&D@R0|aA<=J
zEVM1?ie++XWB$ml<N(D~n&|<_9l$M~;k*z=M%x3$+b|sOy738~JEQfRH)kD_Mhp__
z=4%Vdb+>&vxm~@ozo*PQx$#3~qJ+GJn4f(M@<VejgYJ}8pm&KFiO2W|Vj6OJrpdNF
zZitNE&gsI_rft1E6KwEbriU^mBH8bsXDkhi2`&{44PZ%w_??XYqms_NZ4Bs44SpYL
zUrevlFIuSQ1|e-aGM<J>N;n{QH)p2vLO~w@d&w$6E4JJ5dKNn5_dCm&smo0B7oi$}
zV<37jQweLrjG{~(Zgee0FWv<u9p7O*95TGLh^fHUINMW>*D^pZXS|#+W>*|z`J<py
z!c{STU^8*nn-+`tAdP_XbK&}~s!ue%C`kd=JPlfIqM&_d#%Ll{>!FF#o=+^kVp&e7
zO{<Pv{H*;>UBUIqux!PwY~o}@!u|y`e3%?^Ib&LdFlZqJ;QhL_W)o^#8QcGi{Lz^D
zGuH$>Ju>yF#eqGp5%{^o6rh&>rB9NBKprW>7*HU@4_Gib%EJ<E1T>kX+ts;7KK9x5
zZRZ56`$f+vYN>Lq5ktx01n<~IU&<xl2=*ECHZXi;jFlwcxW8vU4RUQ&dNGlFuRpck
zSCuvy+<`Z}AZXJ&p@G@Nn+%~f_Oh~t7EAn{tQl7JElX)2X&-n|LEgMD{W-`={h#N#
z@x3GRSqpC<kB@KGwGYo@S4A;9iUt!~U7M9%nJ#pf+<bQ75_q>21$Z6}lLRgHtvB4-
z%`v3uY9cH9cgNqQ4BS2}N+8Z%wyliT{Q6!DG-WUUlS7NlLpLQi7vA?t^NTEQ)v~v+
z><ZaYSzZpgo2NalCBFjqvbgI^`S!sCV`+(F=39BV>%27Mt<Nh^CqoT>OGpIN$;JKd
zT3DSkBKL~{f($}coVfuL)=Co<MQ>Ppr|W^ihk6CrN!N)Oc$U2<_0Ua@v1U6w6oSlT
z2u2MPs3cxl<?VWq)h(mv>sk#*%{|xpnKyUlh{$HgWeVs&1|abilmsMxvWZ)seoCAk
zMfCI<Jz39RHR6#1nH+9AmudFvnwr69+cPI@&sdXWzp+(Gk!VRQe^$TNkrOjmyjP`1
zKhBW;#c-@0bY+Izm$`W+4aZ?n@+!c^1JZqsuUwcAr8oX9G4H6Ak<u_Kg+l&bCy?Zf
z1UcqRxk;V~p3*}EwxR;&R!Fsxl%A3@y;2dJ6Abt+4onPN%=E?B$H7u#zoRmvA{&g3
z_SLRJ(isHPRMatJ-=9agjdkMVG^TD?Jo<Y6QFvNZ|DF?Wtc&#cfDE*<OYW4v+@jog
z!h#K9<>d;gS!OE`|02PGNlH30V8S4|_|eW~+d+;_vEclZ5q4Rd@F9=N$u1sVyK2__
zguEy*JQ0pM`b^ifTc?XGD?M_{cz$N`pPva9q+f#gc^W*`0qfN1&iFso??-yK*ZF?$
zhTOMLb>idTB+w^*0<;+B6657PdkqVBi!h!rl3|{Xr%ckfdKCAzV#gRpu6Ayzw}7U#
z*|W1M$HbY%WcjzB#<y?pOULdQL`%cmlwIh_{?oLUlxk5hG_GaC%TYkk>KVr-jA6g2
z0XwO`$!i!tu{#$ejJVZ!=M!Di`b>Mbpw}B?K*%Q>5=zA3*(y;?gQkUzRBL5c>iZ?7
zu*R*22vSOh)t^G*`pBSMGbsc))Z;*i0V{U&%CL%qjmoxoCPc=SaK=YCW;dT`IEqw_
zYh)l{KzK)DAI7afb9Pu%{vEDYJQ{RZ5>5Am0PXNXHB<xqGyslzm&IXjX`$)2;O)SY
z%R_kFl3+lg3#ughLphB%5gD&Uw;qt}v$rNS!^YLYG2g$TvYyk`;Vs2?_X30g4VVfT
z+UK?ovx)4JAh`Oy9pP_K;jRHMRkCl4YKI9_xPEA<n^y`{ZB<B+#7&g;F|6X)1YJ^R
z5TDKCY%B+xWGyphl9rerAAYx1hZZ|5{!nXV#t0=d1I++Wl{Rrj<>oZUv^_nF`JeJ*
z{A0s4eH^@S{q>&ii+5))pZfRman@w=4Sx69zTfOxNzYM8hlBUTSxPA;%DFkz)g|~y
z*4HEuR1RXJ53$`!H!=Q?4<h-YCX1bDG*P_HCyp-Z=ek|WNMl3e%z>(EB0xj_J71&T
z2}E}8QIKVng*qXy7xtc9xTnG0X?Z6}lXYVyEX0Bs8b2dYgfZTD4$N*^IT`#m_gf--
z*Dv>Gm=)JkOYgw#!-Cbu)P!bEU=>2$JN%-OpL#>SA()sh^Bg<*#_i=N(~yl>3t$oI
z&~aWRf0w6eA;^Z1#|%&Y3?$p1A(@tLIm><PFuF&y9*FE(4P%?T5n%LhO_t!Bk626s
zYtNl!MdTZD>mR3~=%o{K+uDu8HOB8#0i>XM04pDmdPE6RZ(Lo487-82BC@;k@+Q@P
z%{hl=AV%xzz6R<<)&3K(3ll=NA3oLmy6s}^HCmG~xIUt%>mW{ZYU1aXzVGg3FrpF-
zHVwysmUWcm0E>x1z#|I&nWb=K_wv1f-S$8jgqa|?tX*va*|4fUDnfA<beCzdaR<UJ
zy0x<v<uzzGB`5b=5K8Xfk`{67)Ekx!A;dq(+bOcqI~)ZZ^p>laW(QRpYS7w)DQF1#
z2vRNp9a_}nMSVKFu+a99v_rHn-WAb9#Yx*S7q)b0G{LD1pzb@tDJmvkMUZ)nvreP#
zt!p@vZ)9pE1^e&5Tk<}!)I_qHq65z?q5u*Z@J<U&l+@4_ENI1XVy4FY_^C$ejEp6W
zrXmt_lB<DYpdiBenEa~@L*rW>A~3_B8Ra=JAqEUV{=1J#(K3-|Ci_M(+5WVgF!@Hr
zrXEq<FH4$#scMgMGT1;TnBoE10|g_%1~nzrK-728tI}X9Bg^6kGoJq}RL@N^e4xa;
zE|AObZ_Z%JPGuIh<n3VOiu6|rQiq@_YJz7b8E2AjX!`{d|M(Gq?GbFZewLf+@3U+(
zhPkw}bsmF3oNFexZl!_Q8&D!37J5=9Cnu+M*o1}M!Kj~#)UgYg=``}$lx1XOM1+l=
zRRame%Z<e1G8dgX&5N#V(Y*Z}uZmoaN<DiWpe(M_m{3qq*ev~N3VihFk!5G>K(>13
zj>~3-E0}^RtlvEwQ7bU2zs}0KK0STq>Q#=FTW3Y6yTeBTl40}dmoL1<>MssP;0CnE
zHiYTqL8d@UC?_&B*XumD`;ED2g9{969fpficKPhUD4+zGPs+-=R5A1jQ~*=e138di
z{o`SlI0hvga%P{g(M_bbrnClabkuz2jF$nLGnwd(o=qfKb|>bK+y~P~pt!eLX++{T
zTfrD>3T`u8MREj84!SN6YHCi*kcz1#c^!8Z)I5Z_C6s9l7oaZ=T3SlcRJ2ESzg{}5
zDIe-90COg9`-7iQIVmcVAHxz869x6@`3w|nUw$;KiZb>+8`zxkJFQPjN(u;&o$pG}
zE->I+dDPa{w&kd#sQ6_E?B9)zx=23(8fzYXj%p}B(i-Mdc|A9PbM4=nCWfSGdLy8Q
z{pk=;E!lPExQ|uXpP6RW5dOcWh6<Ag+S&m#5_w_l*`sHin1))b>L>o=Z}qny6=b1>
zC=g-b1kDYuz#OppJ!kd5_j<kdc7A?7uLSR*E)a80AEv*UOC+v&@vH*i#4OhhUKpF5
z%V5{vzFoLJlC;0r0Bq3#ir%NLmm<Ly4{ZY;{?6C0tRZ{Bt6TC7Febrcr%S?j?=FA>
za@FuLL2G$VsZgzoA)npQUQZ^pPVp7@N5MM9uBo&R-P&NG8F6+PVqgDnT13q4!M%GI
zC@A72yxlUD6MB1lt&Ym9`{?-#Bv;w*-P`;u&yu1nFDIv`uP=4*HyjWf`8fS0g8$6%
zYoanw+3bdD^W<R_AHJ?@UUjA!8PxVbf&{*8tS?&&%sqp~E%48mR#wAc9+HY#w8v(+
zy<Mpu0p<b0)K1n&YhT|s7-G!S*jpJXoeg^<SwH@n$j9~FRqU_9a6<Xs=oZ7#W$~_X
z^YA+5tTC{R(&sbIs7gcuxr7s||3Efu`h)z%(L!GuCkD0%1MZiMTYJ6dfvm*6^n60t
zZa4uDqdZUzG%5|BS6u<?Ke8EAhi0rP-MI^5IMn%+aQfAiJW&#sgXB&K+2TXdud;Nq
zv14Q6>0fkDyOwL`o<ur5izfnTK*-iSg8aEKxev}pa#WFIaa3AuD|N%R-Wm+!9v=b{
z8P1N;&3{;D&_V2DKWNkWQRbQqEf!>U$<@W36qSD#(;?#tPLPNurvp45Cl;^OI&1oH
zVvLRPdd{X}k<;B^Vqw|)4}-xPIT$1-l#f1JD&FKpLt(ZLgcE~AAHhz3ZL)WMwAV4Z
zGPZT4aJ{%&H{va$3Jj7C|0m^VE{wB{;qi?028?`TD|dRfc=THdQN{eT0nUcPQcy91
z-Sa9DsNw)|%%s#n&D=JKQnc7r-!VbqiYMf<xF_;)r>(Ed-4#KF%m|vmC6q-i55+*o
zcN`bX$WX)g4T^gkZRKDW4hW^g0^c`m2SK+$Jf$R{Vccm7d67|m?c)xY=qE7rC1)A1
zGv}EPOW)Gub&~_&X94{t0q57nF=#2r{_*{m1AYz+%P*E##<A%B<fW!W=4LTopm%PN
zdsLI{M?H&WoDw;pB7yVdb{b_DbASP)G|`m<ij1n(PF|o*wttG67S<;(nktO5tpL^U
zYk4v?lsbjFpqrgo4%hIXFi)gA8cax>#hEfL08yT(LjEcmEXeS10m({SN)Nz;d44lE
z!=aDnkKdo|=d9y42C*80Y{pUFwxZwQyq;-c#Vzb(+(6<jyB2_T3lMmbH_C5HtCa1w
zz*{2mwpU+&7}-D8JFmY@OI_rg?_UE91I8=?4PdvMOGz^9_a^}|euh?rvH1Dcfk$Pm
zSx$Q@4>8;UXMt@36wDw1`mkV8vSKzBI<0!@sD{c@BUAo_OpW^?oOS3iuhLYWGZUT_
zsDD|oEu`)hZ(~$Z3~>IQVWoM`m1&$I4qo6v0wvE7oKfgFfDG>Rt1Fhbme@@lrR`bJ
zJFHYWv17u0+C;okZ_EUS%R&KTTutV&SP;t#AW0SFZqyKNmY@}_ip>4qCxA#Z29mL#
zxxq%UaULf|rk&EZqAXqP9tUlQOWl%||07+`&jj)R<pka0nD2hq2JB*x)<Puv20J1j
z>s=xa)20MfO3I!i&H>q<5M-yTwXJws2meg8@?{~lI7(R&-4#}o3LP=gg0qyIOAuc&
z2gKGk2|<0iB$j8OKB5M@-1|y**B@4grdQwm4PrGT@TU9JPCu9G1@(=jPDDxIlKdxK
z%AyM$JVsNiNT*tmN>vatGpuJ0RCrIzPhV9t1x@qF!P;S!*){0!O*hcRm4Y)LCiKL|
zptvmJx&qO&`buL8jAdyruGqVjz8oIq6UXqmuVKL!>;njSrn4vNkucM9KgI-_XD}8A
z$+}%5(X^Z|%pOTYpnscDdKv^qaozvv?y@GTNl1%TVa2;WiU^-QkXgRhd(A=L6FDT+
z^or&zbe53-kK2PBMZcX4E@psiCAjnJsQ2PKa3;1xi9$opobuK`wcu%}o%SP}mK^g_
z^Efb4A}1Q&{LIf)+ddf0;{=mFYy*%P<s*3ctqK-ZKiexWAHq)hR=f4Y7Kx71L1t@g
zA{#S(9y~=waN$Z#;{(sAQ$F&IQCnNv^?}jpiI@{pam?ik!b7y+A$t^cHgVld)}g*(
z9i5sk3>2M1d*k9~2DJyQJIK17Jy2<GEb4x>!}j+`_J4vD(apvXOk^FZ`DZr+<^BNY
zbcNV1=fo|sPn>)sCQG1v&qhL6!UHr|on?V1{tw!0d>0WC$JBQ8PbpU>fF#dlTWz&o
z?}_Y%F~ReWlEk1#magW>UD$t1v`>srKVI;xwIq)D{cZquf*+r!#oA`qAU@72af66}
z?of2uZ|sMlSE@$Y?-jX=D3t~4m~gqW7|VBS^`|Avo0)<V87w<j_8fQ!iujmAj3*F7
zpYlG9dY8YW_4BPIA$O|!cm`ZYk=h1-+y&;pIZk_x6~)|<T(?84RX58e=y#IYfBT)3
z>EE2>c=jHkt1Kd>()qWCV<{GoI59y7L13#0O**^oxH@zung)3If1e+wJZ<B23ssO>
zV5w~8@|c$F{%*@RV9zUYm{-JV>*D(T7nH{R+fCR(Hj|;JL<jFm=x8%yS=Y~1si|gj
z@7;8n=vc1Z6ftlEQ8K4DIH#x|A0op^8mj8Wj+YX0^7gE@pBU_&`G`|ir}D^$yU#>h
z*2%kA==o-s9PkTrfP+I-#7hEqD-=OKo6sbSR^(Y5`cm?8c!}Ku<<D~las82o8f(l^
z$VRajoh?0%)ZWxnE(g^uFRd0g8`6bX?c8HbmnvH%8{4Ef;hcp<d9Ej#`~;}*bw%yz
z_6%J(xS%lwbkLHZcjdTssJPn`85Y_km;{3!{N8%NR@uyUe<s;`d|E)pcy04)Ra=UK
z?c#60^rn3Y777)THRmA2#b+VyJUMwb)9zmwLGf?Kf~`7cUtIpwafy7xJ8~V&Dt}|N
z^gvpZJY6<{P;O6Cp5L)xFs4#Tdw=3^4L-PFlSH@cs;{&>Q7L4U==;-qAfRW)=z>wP
zZcKnV?}qZLG~ksT#7urDvSERia3Va2M$HL&Ppu_Ee5EehCIt@upKaO0kDVMY=c~<K
z*bcQrP%3WLr`X*9HlcOLnf2R_&hOz2pP0UU(N;5JJK>uEtO^L3n_+I^7_x)Re5y?<
zTI^y)A5g)E)tg3B3B(7TWH9Kj67om>GpC%_ru}s@L-OlRNJMtzc(2^`#e|KiMOt&l
z`0EKq&r<}}Zwto)2baWTE-d$b&g8^+DSJ&T^WI`Z<*$6JS^8@Z1nh~-rdE<<jegj!
zrB{cp?3^XBw;b>N5=D<L!>7yM<qdFM@*lecLiv5_jn1ab#pPSVq0doKRlJwoZD7G%
z7I-RUJPC9ydHh|*-z1#Mq!q(Slj@Xv1zf6dwdk)T_e$$E!{eqetn98L;=K22by9jJ
zs2$=y0kMQI!gD;E*YPqLB7PKJ^SA_BEY4^dY!#Js;f#0C`OXXEOK!6Id_AFS>gl8}
z=NGNYb%%k(G2g>oU~(OwI}6F)@wR4iHby~wtn34M1b|b<Bb0~R3;!lgJIWN+S40xo
zEjV(l^^YrV4`(EbV-n>6EJ;W8+!VZ#yRT*y`#WO6xZ`+zcf|fQf{JF>i?5tqXcNm&
zrND0$L}5@y2XJ)?>OT2Qw*SD7;!vd$494BLac5e&v(uNeaJ}F=m{kGrQ^ta;M;Y7{
z*scAsMf3XNgt4v>(wV4JzOZ5C$8|$P53UWAexAODoL7i}O3o^QGJ?O@wbMi$P@?8V
zJ6FfHgo(R8yv!BdhU7(pK`{^+0S?c#aM|`jHm&bqvkRGpvvp$(Q6s)~SfQzLze?va
zbse$8m5OYTBKQ!C+tO1P4d}WZSzv(A0O{Ixl#Gl^FHE~ZC<`qetwjJmZ{6RV9DI6m
zR7hzwqa$1?5j9*_*=b%df@2Q-T=6~o-nD=d@%z<h84-BjHW>@HR2*ZRc>0BC<@3so
zwSJ$Q%pOQ>iO^*un6z`Q8V6^R(D&>yrklR-C9#)%5jEB6WcEqc)3qa#cmwHIIySz#
z=8Ux+0c(WZa{r9`sdJuRSpJ<UuZ|+_C2mfLePS*$CitFg_(e59$SOzO0CSe#F=P`n
zCq)<W9X*E20qO4woa4QEZ1Y{S(j8wuS93skE{`WA8-k<*5KFb|o~Z7m%tHC-uw)2K
zNQ8(9hTD33AcarEj9*`YuqQogg16b$%Z#rKe2qOE)#F`fSNCtdUc)a+=dy1TZ~0V9
z`7#yW?ksZ7Kz^Vd|Az>9Oc)W(4#);7Ot~F3wH8IR;{F&IcsxWc`)L<1f_x^E7@RUc
zzc?io`=LlL@9l_@R-a1aY1XM9=Jc9>tT>qot&5FL$A(!L+RD&Kb5h$?0E^$sIOyEN
zW_2&y!ryBCp=5=sga=^iBNj%&QAzci14d!!(NLAOhTazd%jxDF;y08xRDN>Wq#sQ@
zJO+M^h9ye+lMRoGW<HQ-#wyBgpGI^lS=c82g@qQK)!~(nK1vkhhRUkWPmir`=nGf!
z^AumJZ^AJ+pGtJy)4tg|wq2e+dNv{Ah{}AH3C)iU-#)8G0;4G}?^#h<&Tk#E9KQdP
zL=QdB&Qg1uK9{tsr00CkrT4^^xNO4GcR{puF|clE?49lb`|qyV2sm@rGXlTovGjn&
zM1So4%6ak)g^<tFEYJ9Oza19HKS6_YHu|ydOR#ORZs7QgZ{<a%(!TsP5@v6{;sdAJ
z<wAPck)i2C+Wk`U_GV+4{aN89j8V_ng2qOSLH`+c(!4~22)kpsOINu5Yp{0tE>s3j
zSBr@@v~!;s+*SH4#e_pTH_g^ijH9_%*jKjurC49}vTw@0GO{L7#ze4#M!Om6<A)Dk
zI}5)7i2$x4dU<<?m{QKo&F#GX@sW|8icp&}NO_V|7uEaQe&HtZXbiAG-_(qcTWVuW
zK+IA?o@qX6k2%w6HQI6pO*1%}XiXJMkUx3b1SG%Pv;VtVRa69MnWq1fP}y-out)6g
z?}K}wAFtj^r6UrzzE54`?{65-OtLR_G*`#Wf86J~K@6zqDG(~l5VGbGlA9y)1zVM0
z0i5T1HyxbvmR~FFTcvKtJX%8xy)4}d<_20io9hs~gDG1|5O(}m<CohGQ8EhkPX^@X
z<amK=dEjO&-1Nr>t@9hg;T|Y{Ls2@<V4}SFZ|rNo-=;Ewlu<k=?u6(kM0b`04GYfA
z)10h3xcU}!J#!+<dlg-FIvN|2Vp%W~%~=KNgxh3Qb#?gfWUS)3H9ty5|JNjeQaK&B
zme0w-{EqDMx&38F4h)I1GJr)|ESQaImWU<i_4_Cf49O2%vxnY{_N!r#f-p#)X6`jq
z3fUZ7+QYdj3q13wFb=Mxjg9;g68=V747er}7TjVyBo>#Du&v-RsQNV64DM*Ht*wDe
zn(ZAOW;oKSygZ)Mif`X;KLMR+1Py2?8l&sPOoT~1%ICq=l#DA5P0O_v6x`FNtwg9$
zbz_@0$-jd_f$!Ht_b>i-GjF{dSAaL0vvgq_WxIciMxOIg)52pE1vz=5_nL_Q*iwjU
zKh6KX=~ZYve$EwDXtz7^6gD|m`)oYz2?woRx4t+glpEM=fi&RZwJ2Se8+Xm??T5b-
z8@YBjBOQuy-eU%HaYw99A-MOynggqP;GHbkcXcGpmo7=GOn)Cc$#`ho-27htyzmV9
ze(nWvjCR3-_kucVk7SKz&+5tztvd8_(u<Z6aCr1C<RfMTLb}K@^2(_as=7<N$v35E
z86t;$?64IZ8p%QObyJ;o1|yeaAD5CvXHzxy_T@@uJV|-(vOM?<em=LXZ08-R-bv_n
z!C-W<hwa*(GbV;%Z&q6z)79lyg_nT|Q3}%SRrv*T`F2_YsuOp%=%+(7gE(wIWyPOx
zYJ+9%`Tpdn$dOQinwoyD{kQH9ozz$y0w_8`JB&ajhJ-SU#YRWJ4G8E=l&A&wqphv2
z%aEgG*5C#g{a(Qyuhp|NrfK?z-xC<0i)+FRJdnta&YW!xar#DO^Uz_TOK}_2FAvzc
z9Bf!lncPE=&1H`!iI?kg@u@jgehIZN7UQn^ayOKalW{#N1p`1o&P89nejVIgtp`_H
z%dYQkZK*jjiM+a9UNPv)m45|0%#X&x+??Ya#1e>BF0KE!hDf3%uQ;M)F^PHxwihcS
zCo?j;Q3gWZ6zgWTx5_v;X9q4(M^|!9oY&ue`dUX$%b*wk|C(A@nO}lx!=$7V|Aj$B
zQa_ng;qnha2LQ1HR#IO@{k+)Ju38#;h%TOd!r`DY!0S(+NG|TSYGLw{uV8)n?N>!}
zSy@@magutto{H>0s%3m$9J8h^|38)KrM)WFo!)gnZP$%bG@J<agNw6pE3|!b{kquR
zKCWhdMzV7F@YtS@LYb7Q`ue=PK%g7FA4}{*PtSwSIN_#bqn#@Q5;<q3F(bBkh65wZ
z!s4i3_Y4-=5kw1KUEDPgh34w_rBsHj-g5=q$7`VjR2ruRzO!Dd!sR(fw7OTqPg6z3
z-Zy?QjnGFuOnDFfLPNXS+qpS8FGu-4*Hh%DsE6-d`As3qA2pW?ic&x|WF~Dw-d<Gq
z7$3rvc@)mp0h58y7!PeFv2=YunaT)hW<e!o9PryAW4DKhz&zJ@*M9d(=beFt4!l}?
z*jO*e-Lg7vK0bP~=Zy7_cOL$3Pp?)5SvnI|<-l4Y@$Al#S>%HW^PDTl97mt8W8YNC
zK4rh@CC|zJ9bv^~JJGI;J856DHSGqz>L#p~-8oJDs9lXrXMwyw9T)H~$C?XQ;lPY&
z8}`Ha3an~Qlf_O1sW+;tL0WVc=EfDe#N1Zr)+1l9+dE%&u;@{Dz5D^>TkOayw;bp+
zILnwii1KA^wUhbs`Oj|s2xCk=Q0fzW*~EFzg?@`%So~9mJ`5}FutJ>c37u%;F2A5`
z2x2ClB@CPy^7iqwvcVgF2&>dd+P7m8Pj4@CBU$P_d`L*K@S9h`^}P_vfjuw|-Hb>}
z7@C1qMBj8Ev$AaCraQSH0y3ejyr!l2Tm33izDqKN_EeXZUszYg$S|gsbLY4JUvqF4
zd8w%{!S!gsxqm|*?yWmoT51b$ad1RS`T6dxO|W_`q`YW)cd)<T{{c|979G*Q)0LtE
zS$#b~d<3`f1=FIVqrts+d;|EYf?HwY_g^%g!$oAA(=+BEWDW9uk71#8uOf-W8-3ow
z9pW2ZDh@BOVBgwKpHF6f&hI>=w{5-_QXW;^&0ldcjIkdX!S?w+DZUn<*Zt+(L?#gV
zXJcb3IJJS)tW=bg34*p-XC!i8i?a$nuU>z-tu`ZL-0<))5dXRD@$vCMS`ZEU?%KrI
zz<{8rs2-U9`Yq4;Oj+3?#~(K3G4t{AExG4kZr!|T(Vh70GanaM>g8kpa(oCs`n=XP
zaOM2djYn_O5=IP-{z)&z1UMi!(I>EzjsC#tDg+o_z)eU6Cy`z3q3UnwV6JNWb975m
zPCHfN<$XlxCvmG)Mefx9X=<pGaF#XZlbz3hX&qs(y}G|Sllk<IqZ+t!FAf?NgM)*D
zX)^w#y!sU%(C<Hd_%{Cb?c1!%K1bi_Aa*JyxU?}C7{r5Le<<WsH8Pr<I@uMLmv0?w
zNQ)h00E{<iB!M~0Z_trwv>K{4p?)vm?s|UeTYIlWvi%>|5a<58Wqpud;ch(VYlf}p
z59W8CxgS#05y6_my>mJ}z13#nHNL&B^2h-#FY3PLvw{5{>zIv}@Y*rS*-OgiR903N
zOskyqvIQLa4hgTl^*2v{v}K~F9FlVu%<lED>S<{mS6@KNO!Y2RTc`-0tCIZ9x&axN
zOFf32%uNeVS6#-3ZXAL+^PCOi!~eK7$<3`4PgJ}2lT`K_s%7d<`V-6@Tr6%xR=;9*
z3AoRRM_M(K9dC{LRb}0%Okxke^K1!BnQ3Uy@|y7**6eIFQM)hn$Vgllvb#zb<*TBp
z8IhgM$!l-ywaAunI9vsOEraIH)1SY7rF1xsSA9N7rk47<9${f-CVse(25y@3^YH;o
z_xAOr{RS-n!4qjJSb@rp_qsZYe{3ZM^0PcmfJZ8(n1Bud<UQ$`Mx4Bw>bK|FY_{eS
z9P9S8)^T-g_GQdiVtEUse!ul2qdX!#oeSp*dTG=WJD<EB+m2UG`5wUrju!(jUV9ia
zqiy6VE3vh`&3M-Y<@Eq0l3`LO`)Og4YGF)FOkTSy+31hQ{n_edl-D1Ckw;t>DUm_a
zg6g^|%AW7~b5Ph#?1|KVrXqy&^k>^Uke-nj%+Jvi@Ce24)*5A5kMgi&yo_);!<PG%
zNZSxcncUybz+@Ry)mFc6>Rqx8wYm`(8{4MQYWDG(fSmK!s3`ekiyZ2>?D5#Nj*gD;
zaf8EY@Z%7pb)Kbj?*}|n`U#bSuRATt@Y47l@{M)Ah7i;2r>)y(saPfP!d;NFL})ny
zM&|o3RHd--<v~`*JojF)KaSIvS-*S<%zc519V{rjxs>{UwveoIva&q>1TaMM=-#Bd
z+Wza8=Sp!a_}z&Xm?=PxGE_!DA>3qTYqq_N7W_EIr3zbQ3ZWJc_1ZHeFRC86e%4QD
zZz;Y#Qar#gmCO&h=v&K1sQB!%%t!w}jeT`oRBPAv9uV;m79~j7lyv6+3P`7*NGK>0
z(%qv72neVk-6bVRC_^KNC`gM)cS^(1@h&_^kI(aa-|zSNmpbgt?0w(&ifdi#S_>f?
zBpDttTa%QxLI#FQcV>g^P@g}SV5iQTB>iJFpg?!&(xufLIO_eG8jQG`CBM0m92|K(
z_nSX_{8(C25=6r{vpJcdou}(xQBe^Wr>Cpi2PW<?$*IMyA9)%{o#z!46z<=@zm;h>
z*^HZNGuqi$nwc8|?Sigcy+D6|E<V1wzLFKFziIJgCH-_EkY>`+57oX3*T8hSye6xE
z(OMIM_h2gxOD7>f9)7)4#`wXzWBX=#^??fJbj3T%w!L{lV#UH&1!Yr52M$}S=#;gW
zl8}-D(Q+W^3ZV|EhI!`!(K1uqBnpeguFiA@3h!==hG;};sH>ZKu8n;TJ+R&h9$V9|
zCYxd+A|kYTdP?jjLAPcF*tMaaoA8eL7T7i>6THN6Up07Lm+GiMvm+h%+V+==vI<E>
z7Q{-RLFb^$#S$BE@?Db1G3&>1f_NrkE_D>aNv1y@)XaD$iZ}<2hScvI#nv6ZFrlUC
zPjwv0@cWi4q4(S3LkKZoGeSkQSXu2cf*syb%5D)?AHsXr+Q3cR_`q8O*W%}Xcj!{z
z@X+g^2&!0;H;Z!BHjUQkKi!<D`ypWObcc`+rJR+mv#F&H8ACZ%h4ArKdt7T`6{31p
z4*2C)t@}+QYpVS@o=Z_@SobfuClV|D;dl9~pV#_nU~nOx3Q#;OBCjV6o2~y*wpEpr
zF>T36qROFDb?Gie!m83vKezRVOG~ZeZ6dn+!cMiqOneo{e!$~&a6hUMKkY4qNCJb&
z2!}YgKb||+TC$*JVj+47NjQ=B5U+LYg>3+v2N^udarTm(9m5|gP9=FIr)G}Zvc8&X
zwmFA1N&XoP)DEN~`>xia1mBs5o8L-YWNI=pk5}*C&e+0=VA5a48@RHcacRpub0=*Z
z5YN$d=Lbc^PQi&j>__!{!&Cm_EBUHnCQpSVA#SP8Pey$g#l8SeaA;lp(WS8KH;n1R
zN@pJ|mc2!H%$0mFj&18yavZi)qpn*oy~-fL)7C?gq40wkI!+)e`(XtI>i!ja0>Lw4
z60`yz_MD(h>FD2!y3oLy`{r0lnwR+9)uY-?QwJessGg<y>nnMWjnasP|Kq`~7`#Hf
zEO=w_XrHs`OV19pOSyPxI~e=ya3>+^6yev^wU~3mZVZQ=ZQ!hne4%)U(u*YVy6M~>
zftTSYRied75kFOo)x;oq{rcVF$AsVA3$S!U^DNoqo&t6xv?-TnaqdZLbzB4eGv%`n
z-Z-e5Y2Cqj$e`?>{kaPbAi&%+k@X^4g!YONtmn&dU^j|ZKVO5mgFj83I#kVM<ivih
zn!#|TXRUyqzCFtn;7>Z@dk%*!y#KrrVU-NZ)60#4$lB5r=TmZ;WtBE>Aw1#+MX(QQ
zj+>|<Bd6-t;YzjboTNQgI#2Euk@=U;o?XAZM}leY=z00=-rc)jQ)ELR;epch*WTV+
zwAu5RBD2mcAlbKz?{2Rb7ZzqyQ;?H?NkCVDBLk1p`sPlpySuxY3}i)6!G_X%W9>1t
zDEN_#jErctD<(h9WF?B8Hxh4W;3)mj3wi&pzqq#p2*2BaiRmdGS(cxCvdTww#m6X9
zU)M~;bpLICbDN}G2{w<R$P|oAz~#%ALBPTqXzs18tv3se9cI2Vi;3xdzYo$adS#Ap
z^Yi_Kg8B>ZAA9?4aL_<3TF}N2awk1Ky_A#`91h3evDyYYPU)_rGeH6ar??CcT@*TZ
z&JbAW<kQK?$vRcV`hQyk3rwv6-BlEQ8qpgm=5g0mAL<9=Ud|Yvl)4i|Xd)kOLFSez
z`PHtE@ijrd;{4XPtjH#4qSVxbe?0b9{4<sU2_Yhj{%uqTOSsl&zQJGyv}Z#Y^QyF*
z+-Hy)GO@7G<Dua-X)U<dI2HKm)3pYrYuBE*itJ7>#ygKxKg8#A8wmY%j2{9KZjf(k
z1t7b+o4e&}Q1q`lZ|S#oA^Xj2j@L5aOWs-r?N8@pPH)tgPU35D+gzsj^!z(T+KZP|
zd|jjZ5jU7PI9@&auO5jmFt^@>$G*8M;O5#=2sJA96_spkOYP34{|cj|t*tqO3=Iv<
zWWXvbHtmw0mU2(h{ONh=IaTR~?$rDF7jDfl)Iljf1WPxTTOdD7mQYbAKGLB#AI*65
zAZzhUsgxBPHR2(%NjdvWAfGi%ZfpGxk66RPQn(ecRHWZ`bQBsl*7y*cD&^!83z)X2
z4pg|wN=vs%tpR`IK+Ow?KVMTIo>#M(Ue~;SJt{hS4`NKO>9mN*8Kcqp{xX$F&a;e+
ze%w_yqcU6G;ya61nhb8CmweQJqNmt>glO?ooQI6gQ|R&e6fo??rsZfzU^;T-E(?@a
z`b6#oXBrsn>eO?3LEQPCzmMlqcDZ_k>?y<|>if%g$A>`P&1UqYnyzVyqDP#F(>(Ov
zq|&$1tRLt(8Azp^z2e(Cvhh+|UoB*3X9rwxB7G3O*w!cnP*W)(^E>TsCk&V<ekK;%
zYon-v&ikv=Z6i7K&8IgF0*w*^dnNVdS+n|*xxeX>czDGG8ruUp)haUV)xg0_$9Pn-
z?1Ez&TRFjNnHC`-p_`eS>^enHC_hA}z^48BRWn`qlKGcNuIQBVa?u$94Ug}N<KJjI
zRsLS4cc`Np1RHaD9v|a7_x9P?rmSjnB}J=Ij2}*XlCrh@oNq2rB{H(Iye#6UAy)N$
zE3jXils09D%+{KOr^xQ$&}9U=y1LHfQ4mQNTlUY;KTgX`PCihWyHRl0U-0&2M#jnf
zdU_syeoDfpkomXa^}K&CN!u6qYvYQ0Fjt=^*;d4IO2+mpEj#E#eLKGZ&*qQ?C10n2
zxLc?m)_T76+1p^R8;A8t<Ug~06x&0xOd%sF0I>0Ueu}^UUt-xHTH^$Voh`^qT+pJY
zs4kEydwpqnS=equ@xg<fn3&!SwUl_Tz0JP98FTda-rnBs?z*}<y;6JAY78o`YEPI#
zN35Wz$llR0Jpob&-4J@Qk0A_s0|OGj{J-0}8liJmlctTMz)Yixjwh?7A3swbcEtU5
zgS*}S!)#h!sxl}|aF(8=i+&MOn~BV{AK1x1_&bq0*#m;0P8jVS4#Ai!ZbmxJ6Np^Q
zUznMh`Ot2lJa%9_TMjhMRwkMn8ymB!4<fLr`|+ET=zX~=P%Z;rTKnD=-GBU>b>0B^
zQ1R0(V2Cg`7pXISf2S{6J?x}R<NeMjn&zd}o34w-HR3JtG!cgLudMQ%YW8!yxGhbN
zsiI%&JuEG%RQ*}bsoEhrwm2_t7e}*J;`)LkZ=p0d2hVGzC$gY{1?(-r@8xfo3MOK{
zK#}sM)#(Yk2)&y=wUH>AtN(g`(Q2#t`NC?Khk_jqcc_moW-fe~OUwq8)<kVPG|FJ1
zYSt`ydxX1}+OHzS^&_M4sQMS=#lJ4n1=@o(v+zY9s6gYs&QU)y=9dp-(ey*Db|XjZ
z71foOhO=!#*9<z!2M1dnf&*I)iTI{dM2zlHBSHVhGIwci`$*9Z39PNQt+!IuJER@b
zWZ2jB3?Zvcy58!g)_<+J*OPhK2w$K{APU_#Us@-Aj+0CKVTT_Q{perMXo_KF?L>Kc
zSS64S;Np0-noKE<y^;0~iSVUVQ%l3r#ASMkk2R4xiT8PU=HniF#1r-zS{V;lg(K0C
z|8W-<v%h0b%CEX!=QqLxAAM5=DH;abA5ix6r0Mc?cPW|6NvRC4^zRmYL7mub@;hLN
zI*cvXr5j4x-kSXj+hsxVGu}YImya0=@s(j^$u2(?#tlfAJ`Le@QXLP|;Y^A8R^F_F
z4Sb~a>%YOKD1Hva7iYJ(Xh|yVVp!&Mix-g?zKusWTT_=6SaWtlO&qq{3q&IP+@WmH
zNp;0zq|nCV8YVpTcO-U~W~hMM0kdCKA@VRVYi6HNKMM2mkugigv&gdP(2~1RVXJqe
zhB*=nzJ6GC;UwF?NBkr_X`(Xquvz+9vl(pir9dpgek7oP>Cw3}ctlvtnciy7{e|(G
zdsw?e?9GC;<13y^GY$(veoM<1Lv}53Wvd2*KNcFj0?VZN%WDZ1g*a?PraCGJ>qJ@x
zHfmUR(r#X7<mqJ-tNg^lJ|n|#sX=i2_!nK2#TO|#C9bH$){-cO>b$*f^YxcB=xz_|
zTXUAp1dF>_Wf2ee`kGm{L}s@(Nbf{DWfd5lC?ByFwLJFMZR_9HyBxNTQjfxw`E?a9
ztnFrZy*DSU>yJKHY%^VE`2<U6&%$URx6<Dt;Oo%t+`8+Qt(+t7iZeS*?}55~IkLp?
z2Be7m{QS^YhfjDE6uPLKBG4#PNTuJsd&iruk|4gfw<o0yVuFxTn_E~|7#s6DFAf#n
zZ!vQpJb#B$-YnoV55^_fGq|F{jX^ng=`RftMr)M&gG%_-XY(@ddfn~us%SBzzfZrp
z{rRnA*9)uBbLWth+AkFNAR-F+puWC-ettfr%h0k3LY~Q7AdIJ_rBzv3>F@6k<ioU_
z3hQCKK|Vmbz{>bOethA=1#DSaF{IXPY_GSRy2uJu)ipqYXr~*UpoS0*9-(wPA|KF6
z++&A|r93<TNLDrDGA^)W6E}Dwdh$A{0riNBD5`|`_7`}13Em$Lj*b%%*I6hCUazfS
z*x1-i!!PFq15+2W$|6-&)mD%|yyBM?pVyIRPz8d0ayLB4f3BMl;iA{k9Y4Gc`yNvp
zzcXA<6b92cY(k_S$!h5)3sXjL>7B@Qv70CkCwMJL>qSr>g`&Yz%ME+^a-4Mqw|f1D
z5Choputafjaj8Iv;=H}RKOR@k)_TovnN!;1)86&*ql}*CP89d}R9lKv*J-QrJNNH1
z=7<v=vSy=or)9#l47hGcdR0;DA`5xFqc4WeP8Cl*SniM7Hr)D>>AF1sBQTh4wZff?
z?bWwIE`*alSvM&%wJO<lE14>9FC;MS)hi9A$G*NqYyigaGJZQ%LoMR?pe0@`PrpK5
zW30Nm+PEn?Dl&3W55lPQ4{2S@Y-~`#`2k&>olvYY8`L#8GNe;pJSn#2MH@xUJb!#=
z&ukO_)X^usNtP!Im%|x?846No#Gmg@GT5N4nMbz7`eLaQ&ze0!#lC#WXZDqeNIRNc
zqf$9exbw@GrwgTyb3Ls|SEi*i%)zno3e7{Lgj>jEgvXtA>h9PdRd8bW7jYKK)ZJ>%
zSEafeSe7W`v>d=6CvUqbdU4FMrF(C1;i<3-j(<J|)tbon;YzD$&qPrXud9-_ws!P!
zb4M}Qw-Ze<Gs|`bGEYw>bzirF2Ozmll7>{eE@WZ7oji%Ds!z|>%`G1HX=wLWn>Mf%
zX4S1xAU1IJSqM>NW%}rG4mE~}Vm*9yrQft8eW{V<WtPYGS`XtLQ+1Y?5Iw~Ug)Wiz
zK++Hu69c<NzfX<h*H#&_@Mgw@+P$cwEAKZL!AmVo6l+*aEkn--%-|F{Q%;Dl3Qjw)
zFtZk46Qx0DxQ%m9J(Eh}<m1Dwt);(?7Ii5APP2^6;kOVDzHe#@Vj!cWd<SU6hYugf
z7?nv*^XEX(rn{S>o(~@nZ)bNm^u-GhZQWj*6ESH$BX2TX$3XDwl+1k7X2x`^+$hGX
zXMR->wn4e?4o_Yaa4oha5({Dc&L)=YV%AxJ*?9a#bV(I?MgU(W+|SSN?c28yuQd5G
zn{Ud9h?D`SyWIHx{d<5Y6JtH0gvcD((UHA3vCvo2@$Fl9Wieze&CSga)k$@oH~eWP
zgfQC==C8o&CtRFABanh!jkEf!M?}&|@iL4)pi-!fRTpc19hQ;3EX9QWsYW(378J#r
z7<7{<lNnf;>}oF;8GFXb^BrB3b3c$hE2>K$bV6HaF7#z(`-^X$w|BZa<DGo+;47au
z&c*N>C#9wX=@OfmnwZYhFCYRoDgB)+fxuBAf`+D~&kQ@BQlI)IiG$fmpD5JmWg|4y
ze_R!LsnQ)`y{^Mh5<#BnWl>9DZ5)fu=3-`L6-lsN@Y7T4_sqrK8B-{9f3@c!&LmEz
zO@`uC{%w-G4=>I=>UBqW*!dKMYSR`@-g@h(J^JicMxX7(mRojKdU>t3uXCJy={Az^
z1fSRJ*M<46hpk_?_-(DUpaHlvRBIj>h!>K%I+NZt`_21SSmo}!RWhe%WUN?3%bnfQ
z>X4zjUEi9_dPcdnU01?RWiK_g%`Ux>00Bb7@ati2c;I2#Tl{DMN{{9<KX?8-i11c!
zj1b$|t>`e|U3BO$54T18;5oOxlN>o(fI|n(;ScSMyr3IqW^j!qhc6mVXdxOE78k<I
zfoR^DATlvJSj$dE%#USvCZ)8rG^kt+0&a;-5zdU!FEsD&V!{meEu~|xT-RATQWK*#
z!Y(9V61x8bonO(<I-k^bFMM#n*YoV<rK9QPNaDXMEQGn(9~&AG6Ju^{yax4)Cn8y)
zrPF+6?vvaEi*Mp;KhRA-*WF^Dt-hG=u2vb3jm8-)6mn?kH;UL#FQFgh^?$Cd$VP^e
zB1`akB7I&&L-#y3IOJ99Dp1XPdwRI{LntXI`t!n~qghNl%qin2lu)AXE!H-&Q_;(;
z7&K*ZW1V;M3$~=sYpjoaQkuy2h|rV~58X@<iPwMMlDibqhi37{a5k{kb#&u>{rw3U
zx%2SQcE!zVNJyW6dSAW~Embw67g@1@d)I<!<SUz^lF0}A25Sc8;Xb`GcoKSCb_7LS
z#A)kDxc!j>b#5RrK&JwklV<xMi6F077~Vz-KEri$Hr8w`fxUbksr2~Sk*)sEx0?Tw
zjTzLT90L<ktoj4b1i{d7+10%<tC*6>)C7U;E;w@^56b$04v0z}i}Y!CL$ZJDxMy5c
z2rn{e8v#Y|kzl`P@Ir~@NRyDg)dZF7w*yKe?N0jU^ZK-;2ANr3_D7mRzG;d@bS>pA
zCT=LwS{%k~-I)kLb&e;W1{2E4%4%RR2nGkNe{@t7BsY@0GLt`FHSVvWQc(u8rI9+#
zw*Y+hWram(H|DDzY>1Su5Ou_B{?@MHU}V28vcKv0_T%A<M6A7n$#QGMB)LP>_++Wb
zmYGbhGw4l`f6?ViEY-C)GBuSeynDw1N?N1!Azf2l#0SS2_VNuqzsW1z-@lu<ziG5_
za?0DW<cHtFQ*{ANG$e-#%_?pl!u#Hpreox7E0SkYH@x9>U7RDSCAPK^Z-mD#XuKHC
z2JcTVoVond3)ek!*!fU+JzMLvfF-#Wvj;OKs}xuJ43%?CU7*64FqbY7*<)GEHhX<~
zwgd@Q*ED}{roE83@0jk)mPVaNjNfIU2@heRs#7h@cu3rdaiY4ham#+(sx*iGE(2Be
zs8AJv-gDiBlbzYxrUA64PJPe0$rDW*@NtS34|~Zi7pficjyWTRX3BE;t^>rn$Y6<9
z=lA0m20e%1t)$(obYB&Aj$aNc@t?eiV4C?pAk7U8G~hkK4NG^7LzB+m{!<Z_;&?S*
zH+^v&-bPvCLRNHpv)y#r_jHt)_prRoynqP1eU@m&W+Cpr^2NSf9F`Q3*ya6i9~oHY
zK#I~0ZHnO6Kw1R3QnHwLRM;`a!MSE!hFqCKp~bOe%2s<dtfC`ZdP`>zB+l?)dn_fN
zeyE-a`T{Gk%j)P#KVA;|N*-!~SUu^MK<%t#CS*sidDIasE)iU3lVRrH-SetoO<tES
zzM}C?Mpv=%#4$Xi6(78esvdhOw59ZxIawJRj(~&lB|ZHO`kFxU)hsWW@TVsQ%i5Fi
zmb`jJYVtbYYnLG6A?K<r%JZSmqC{{DEAFd~wj!%@!Nk~tBM3bs+*9i{RsSJE!u6>(
zZ`&;>D!jU_tfE3ysqOW1-xiP#bxvw)ARHvOZsg}))@(8TOkl0drXtPzWUTjCI_o)~
z7T37qIF(O1;{~ShxJJM1-)DY|SKvkvp7S&)eg2rHYG|o;ZTsF_E#Js)-A`vBQaD0#
z|5ei@``}k4AoZsj{&8JIoSqD-{G6e5-PUH4F5>2O!{V9bI5`v5mfpO&i{1j2{xDbO
zMh+E1;{S*D)2FSB#B;`Nc2fMPMBAchIkS7t;4`x@_d9cET8;PTeB3!fW3wCYd>pAZ
zs{caK`fCkZ&^!rv-+2m_4<~t%3i!I+&eXg|nA{HA_M_~A2<xMex_e(;xoDB<(f5B|
z^;g^#R5`mDU2(E{CRNcSp5W*U=RPBQ_RU=HdO2}d!Y}A#!5DS*tdp>IoV34u4Y*t9
zwajkv>eGh&)~q!eM-)g`p7h-k@)?}RR?Av}Yp-Ic2yg>gx)XoCk^6D6(x~$3kxQ22
z%aZJvF47Z0E+hf+@?D26u13yTX?SXr7FV?0EU|Sa=R!W6<k6Nm{WF$%hnl4skw=wZ
zbR;f5S@4v2s6u%}($#U*)_3vKK*+b)8o6fV_SkQK0E^A=$SV;UaCA9cEY+rfQsmmf
zrrx94>Dta&w8?8Bn5P^bkwg#~-#^}ayr;V`!D919Twepjl@g@VT~+kjF?spxr<E@(
z*T)(}kC#D8mv@9=9Z#aR`j0oLBEk>5qvfSoA9L#pTM5!!dX7HT9L^BL9gh+=JeOse
zEE)eoNQrs0Ng6vkIzE2<`1PwjsHF^p8tl9+2zSi(yv>e^x(WCoBw>4}-X;7K2gTdo
zB_epE@+d@@@uk9G$?H{DqE&EQC4@$sF+%H6b~kMJrPFG-K-25?L4HHzQv)U&C?Nd&
z-#&l-TvKz7hew0g7<${tTUH0h#>Q&>j@{DH@m%kBO6C%-|L5q6&^aa};?8r4!oy5d
z)%5ru&f7DJPS3|EuqeOVGOj0x1@Q$^a`^5~j4L3@+bNJwD<~*v_oU|}J;nbZ^3B%n
z`e4=4$I~>suJ8Z3r3z8xRn%<4Se|%cIPvW|=cOZhOq$`%+l*^<%l->PKC<2;b!Emd
z>f;HMHuUU;NvFAPW#PVl9T^$9q!bM#yd!ME!NEt59;LY=$h*!bATSxC%y{V%P>4L<
z<5i&7=}@|4vS?YNq3RzGXS@e(QSO?f@2iDv&yElDHhORP=QYk}Z2P#1ejr5_Pd%~q
zF>GJSHu^u4A@w^TR-+Wde^9|Q-3epX_|Wf)L2^Gk4=#~c*Q*EHs7NFu!F}xXOGjWg
zu;S${CTtvy^cuUN>+5_7k--0Y9AA)$DZ`@o9f$%w@%Qg8wlV<mp2?<;-KPqNz{a-h
zZ9c^lyuf$0V)-6BMx*(O>{o3a$M^L=l!+5Ij3=oAfegy4NV;@<6&Mfzw#nXsPRZ9z
z(R>=2t8TA64p}RS3vK*Gu$X!J{fyAT2CKM8ug0-QWrd5X6_a_>#ev#B-vM$<2r2c!
zGEJ00sdY4hS);euD$nDjR+Z<juAZJxC&++nAQ9o;xGOxb{xY!pbNk>;d6vX=xAuBu
z^m=o(Uc+hhS}Xaa;!Y>3tY5VvA&Q42J~RU3&rD9Fp`l@AW!01QIg}A3aPHo<X4q8v
zCrKHdpC5pJQV@9pfqsI+he2J)1@PQXeQAhLAVh;BdCWsR6%lvytWeICERE6_8gdCx
z8T6@)2vk^gv&KiN&m2G$^z`&V&j74|_w-R>;?dJ{v$L0!V?S|3goT02@w4sv@+AlS
zETlk9&CQgRdMdx}e!@*w6_NG*GKsgbpz@(#Jyx!~!%F{*mtW8*w)F@y7Wl_er?0#(
zNMQXX$y68FE`3t=BRwbQ!RY4;y-}56l?$(hpW8ATz++3u{9#aTZ|kj*SLcg}Pv8mK
z2Vu*OhNZD|^|#e@Hxtxt?OtO*s)zpg>7ioBN@)~r)t&=8X7s3knR%#2B5m<7iL@1C
z@f9#A-&{*Y5ey0E&k*q<A5cmFn{*Dv+vo|1z#^5@7w2=;W-4~P_9+#odyU?qEelXc
z5aA!@P}I5iHqp?S!7Uq9Y{bQRPi&*W!g_+&<94T^N9U<JY!9}>a@`+R>ez-GLa-QQ
za4Tz4yhZFJ*0T9kpIf&Wrrl|y;aQ9j!H6Y--2Bt&ZCik7mAkpV0<K|Tq|#3?&bqAr
zftUPXo4E>f-FyaVgsDHjzK6hi8qyqnKWShi7aKC}tZN*Ru*2k0Z*`=RgZ+dPKEjLl
zEA`<580+Ru>NF0I{y6ib0Y0_9*W+WIz3y9kRx2e^7uf~yFfswZSF7b`I*5wp>C&A;
zYxDs#zrBARZ6_Na$Dg1P{tee*h#;H4vlRfSh!u!75l_x~U5_m%n_}Ooue179*7x;<
z_9*l<AC-V$BjjLQmth7BbYN>}bUef-^!4)#ENi@X|Gr9`@LNb`VfO*(zJ$Y3gtYSg
zlU#0i`R%Wdr)vjJ6sgwFQaWju+6e4-8kEF{dcE5t2^PgiIPK~%<U|DbvVZ~raDHfL
zDCF_R=^MT~0RBL+k%7zb1B5s)LPG)nNl8xa-%g_cxp|jKhKT6#v4NOSsSk7Gw#zTq
zSg~am<;62s^-naH;vu&u15j-_UESSL23nb#fS2{O;OiG##zC{O`sNJ|y}yw;VNAq#
zTU4x#T%SISG>XQlZynXKUeyZDFYiPU5`{2Un4Hj1BPHh1=^AJd-NSF%{-M5J*2Ko%
z-rm}}^;C_^{_Z*`rm3uq@>}!(&A}Yh($dmg(aHWu?sNIHc%%0O*7IyO5jtY)w=bt1
zc=-Hj;jKvop%GWhRaInXSa}#<8TPiR5L_HHISirkp`@asiY7BB83uMMOy99@PnA1y
z;sofYK?n{*NKOQ`2_3QyT3)c7Rxr3&mmKev+_8C|mO$6=E)OD+(s>*o3u9lvppY9^
zuxjO)0OJF9w$u;FZJRs9?m^{voR9JD1nzQGf9JU>K{YBw;(XWfW0J?9(mODqe*?F&
zlERd<(0MA^?Z^;K+G873c~*#VOo1lf@_J}kWkT*Tw)YpI1G^J)D~w5{h(!;t=gl<r
zw^oDt+ZS@8c<#f@i42y7%_^AVqoB|@bHzU7r&k_Cf>*}zHd>1LdbQ|E<fnV@HxEDE
z{<ypb(GfjttTR4Rcg)OY=H`@c@xj<8V&cH3B=aZofg@#YXE$2khqXz8DMn!((6KIL
zKcxz@QYKmw4Cbz{V*a_kc$E+FP!*bzhCO}UK7LvORgUIXGNMHg&D#?JcU+vEZ``^S
z6B*h1K=AVAr&dLUg)xFQ$()yXdF@*hCCNJh$`vFo6;HKvWmC=_!*d=a_f$$_v0D_C
zo))=he)TnjC^(yc;fu|~)<Ya%1+{5YzPrpH_Uc!=bQGHIL=XM8HI*p$DK~aZh)u$w
zF-n&0rcq0w67jKUXOO_q{5P}{z^p4z;B9<m(b4q?6Iq{`yC+$9QE-L&&n$hJbck*g
zi-?zTy^`qejdL4ldzfLr1ovtEH=MG9DESkC^(y;bdy13Rp@=N&dFV@4`fXa_9ylvC
zgw^O+#nz7#ie4yW=Fx9cF^H^9si54str^l5stIJnunTal#h=%X_cLSk#31F#6SbA@
z1JM|rKR@-M13<&JR{|9quXko1qItI8CMya|7WF>o&>qjbV0|~8gG(>}`QZ}&Jh{m_
zrI_(-w==igKOg@uR;F_XA`P*mhcW@OQ>dSzZ8|FFvZk+v?&^wW4E6p6<8gmC=+gB8
z=rL85Nl{U;yu94d$S5v07I*{PJUr4yOqg3mCN|k8_pui|ej+rVc`=<ct>#V-jA>~&
z_m9w5Ac(5$!B&jM35$rV0CWjKmi)F$HF-5)5XmzCT*Z&s>{U_j^_1*`5jn@sHc&)Y
zX9Or5z}2%Ez;uFvA^(9_A0^-<EqwQG9^~h6`la@Ayb=tMg+`UmiN=Fc#TkDlZKg{a
z_hVZ-s4ARsUb>gX#NwpR^73k$*x1-43FUu0ew6fkxr^Ohh#IUHKRrJW5AhrT8k!V4
zUy>mmGzDXs<J`=**QbM~`^$=9I1%<%;8&%*?!}j9AqpG)0BtTvbuI4)<XfsH3ly(+
z05}U2v2VJK(v485gW5$ufcHQhYrmw<7R_(rx-=r)cIpHG2Hr2!uSeYUB(D~jp9tJW
zkfF0QBWF#XAJ75|Tg1tVXQJN&Y?crZLd`P)e1_v%Kn+}l(LAQBJlFbHY3}xMxLXXj
z&7-%o7ZH$i5k3fo)cO=-_=616v8IES6^?x-Fk5Zz+v{GZK@Z0He>}z*k~_*S2XWwm
zG)2xH459*IjJB!!fXlpX0FNjDLM^oiwuhX^Co?m%xLClV6S!n-moIN*KfR<FIW#fx
zF(e8e6xvkC$jDj}&?<HYJUoR!xBZ<Ve@5xndoRO=>aX-Dds7M^KA3KkRc$5~mQn*|
zKECVi>G%l7@&G*&6&1~S{aQ&;k;$Z{re@<etP69RCJBoydL3ad5EZ*<aG*h?kJ)Y1
z?EL)RRR@PbdpMkIf9HIw^wp(ehk~d%5zNx<zmwbrR8DU1s}bm3p%$`DeIpE@d%ep^
zSg5b~zsyID_^<VWg0TvRK_v(xHuv}MavAtSD>3f(!OvC!3~MnHTe>4d3m!O)tHKrM
zx+5?G9o3n?k0_4hwg<dF=oi2B`|KTpk$aj$b{EkM>&J5lULHZgpg!Rg^~{mcQL>XK
zZEbCd;EyEnOP_!%7n7ybN_evw`Ra~?9F}St7Q3;9`w@{Oqj`=cEW=)1gXG}iPAJ<j
zzm#A4dW=QQCQ{|avuML3x8ZBh{lEVrLtYI4c+oVh<vN@v8k)W5QepQ-oZXI`C%Wy~
zaqwU44cs>6VvkP{ypc~!o=f*2TvGP2cMmt>)<b9LS0vfveA~r{H1h`RZwWVWxH7l(
z3z^-%ipsgNev$qOq0)c+2+1wsL!3{NT;Y<=)AH*~suiw052lY-pYff6e`{MY^`}VS
zW}4olh2BS~C@4zG$~wI2=u`>+`z9`j;q_sjUpbh4bqc`)TU?DCyd2~(1~xmwiMSko
z^Xk>BprD}4%xNI=)YjJ4^#`e3yY>lAFci=doJam*U|>k9fB+g8NOL`Jd0>wg7gxFB
zvRWO0!VZKJKC1zkvgQun-^Im6O55K4AkE1r2g%k0@<l5<8d%M<gH`)>En@36#2im)
z_*h^n;uOo}%dx@^W>6)Ihgp7YDYEzP-aVoREQIqiPffn%P<wR25x$|We$JkNfS_Wz
zffb@QsLKo#nTEEqD?l;PV{<$rCc<s*Z5Pm4HXOs0U%)V=wme|FcsAxGyYksNIeAH4
z0XoQr<7iF{2|mmgQ?2+fYg^|{L%r3z-l2kq>`8^Z8hT{X-TSM&(D$;uy{`Zne#)Z3
zsbYP=4QfNA#Ka;l%VQvG%j-CMqjGm|cN=m-I<)5wm?g*A?~)5WxF21_985frhrb#W
z-jW0A%ar!lo}R7u-h}R5ka-F_&N|P2&xc`k@)2DIUU60&U|9Og)<qV|$*8I2?pgJh
z;vfs!%uP*A1xonD_Cgg9E(-koWP!GghljVpeiII1VqPA{T&G@%?L8QC2i74?K1)rQ
zxj9~JdnAx!ZMsAD$ZgF$WAbV^Vx`xoVR)c>nawvPUzB2@K2U8BqM(~MZ?>`D#hnd`
zFh)97BPJ>V2_MzAv$Zu*SBR2tT=h+P4p8=J8C0JV+?PMEuG%|n7Ny^742MY9z3VA8
z&tnfom&S&M<mdxpeWSUwO@Z6GWhHcu1eZ3)3Z<|QcDc-CzIwd-12WyWCT)*6B4C>2
zu4#s|(AzC|*=ZOmt$QH@`~cKmot|L_#xO*JK*G&0EF8r@)tOB}z}ZB5HAv=`KnVdS
zMev#iFQW?}2E4@`r`j`24_g}=o_pQynb0a?Kh<hgx#>%}xAlVqRFsk`Qd1vi$&;Dr
z#B<)M0k+NKT)><=0{hCGR7{>{KYn~@OO8Xo{90oXgHmI6cefOWQn1(doPbUwsPaxM
zpy#f3d^+IZFUMF_GJi385)nbQn?;<>EfY%8lMj>qfVd|;Jrlg=sFWZcNBJ0$!I?>;
z6g#vafCTI|=(GU)#f#Dw9~?q&Asv^48mMS!Sc<^aw#MEtx|WhoT@4~h>76kGRzwvC
zuN8!yOzuDw4)&0u=QE6nldv<;*;udI7lXhuDQ+*dJ+Sp@iCG3M&{B?p029ywhqQrO
zEMt=_bQ=n*MOBK#P-|bP+|F%{Pe@R~4Wsu`yT1TM&4=B<z+eU%O^)k~0kAUBeLo?s
zLI5OVw_G}z^y1ps6@YQL8S?n=CqOR)_MDuO0_DcT$LJL$W1w(T<ZuNMU_Uan6~%oo
z0L;T;q)N)J2R}b-fWQ9w3&MO$OH0r)N_Qfb*QSDklu1WAY1)yh03??odbvH;Vf5oa
zt9MkgOOXd-bEGslHZBg<>seUXg`!h)C)kXPt*lhaESdE5S~-|^XO&pmmO)2Kju>>2
zq;VIf2dmI?JJfS!%9f}ccJyE7(7<*8O_>c=iYeEeG&#)ab@Cf9uQxll*B1hnYm)tm
z2OTyR@3hI?kWYKD5G`bv{#2zV`J~#G&dr;q`iW_|qOXjgBEDnh?!H6T*)n}xx!j=2
zQ*5_=x!CoU*g=I_yVBPc4yQ2sRc&O&?piMtV_|ci6A)NWJ}mzn%2=_Ks$z1>%ga+C
zoim-;LGLWNNr3DlDLDek0JD@^A8%+!`B=71vVh@TA=lnEPX@Nd$xD~@%Ohv(01b?c
zJQdv)z<$Mb{H1nHZ7rmtooNbG!`v5KSz%4XC*H&Kz+`nasAY3kl1#&NKYjs$V@Hot
z=9(QpDxnm`?c<mn!hlvTWA<)bn%lWf3d?S(si`R;Q4KriOlj8py1I+(?3Ms4ZDF(C
zXS9Af59A6Vp{N19BE@a%hYuf`g((XUA1|Wm%x(g<qD0l3)m#yXS-|#m02ct-IN#>~
zv=R_XWc&tb$eaYkB_$<QG-d5hIBb$w9nerSiW&{Ath~k>D28vri&IG2?z3xY+ZzJm
zuFvHsWT8^cT;t=od4eM72-WCH7p`NEZ#TemwM{mxsm4hU;3_~At-!+XE_Wp0w>j({
zjHjK7Gc`BQ4*9Va)Bthbg9i_${L;{7tMelr+ClAduc)gt^qYW_rUC5&TXNHRWNZdy
zjzLxe&7}YcfbV@u`@i_MgU|tkI&<(j33A~2ZzDfFJ-)>M=??$@F4;qW(3zw6Z|4Mm
PiXd{*%2#uxZa?}Twh?c8

literal 0
HcmV?d00001

diff --git a/docs/transformers/configs.html b/docs/transformers/configs.html
index c9097bfc..b8e20f21 100644
--- a/docs/transformers/configs.html
+++ b/docs/transformers/configs.html
@@ -86,11 +86,15 @@
             </div>
         </div>
     <div class='section' id='section-1'>
-            <div class='docs'>
+        <div class='docs doc-strings'>
                 <div class='section-link'>
                     <a href='#section-1'>#</a>
                 </div>
-                
+                <p><a id="FFN"></p>
+<h2>FFN Configurations</h2>
+<p></a></p>
+<p>Creates a Position-wise FeedForward Network defined in
+<a href="feed_forward.html"><code>feed_forward.py</code></a>.</p>
             </div>
             <div class='code'>
                 <div class="highlight"><pre><span class="lineno">21</span><span class="k">class</span> <span class="nc">FeedForwardConfigs</span><span class="p">(</span><span class="n">BaseConfigs</span><span class="p">):</span></pre></div>
@@ -104,7 +108,7 @@
                 <p>Position-wise feedforward layer</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">23</span>    <span class="n">ffn</span><span class="p">:</span> <span class="n">FeedForward</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">31</span>    <span class="n">ffn</span><span class="p">:</span> <span class="n">FeedForward</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-3'>
@@ -115,7 +119,7 @@
                 <p>Number of features in the embedding</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">25</span>    <span class="n">d_model</span><span class="p">:</span> <span class="nb">int</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">33</span>    <span class="n">d_model</span><span class="p">:</span> <span class="nb">int</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-4'>
@@ -126,7 +130,7 @@
                 <p>Number of features in in the hidden layer</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">27</span>    <span class="n">d_ff</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2048</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">35</span>    <span class="n">d_ff</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2048</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-5'>
@@ -137,7 +141,7 @@
                 <p>Dropout probability</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">29</span>    <span class="n">dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">37</span>    <span class="n">dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-6'>
@@ -148,7 +152,7 @@
                 <p>Activation in position-wise feedforward layer</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">31</span>    <span class="n">activation</span><span class="p">:</span> <span class="n">nn</span><span class="o">.</span><span class="n">Module</span> <span class="o">=</span> <span class="s1">&#39;ReLU&#39;</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">39</span>    <span class="n">activation</span><span class="p">:</span> <span class="n">nn</span><span class="o">.</span><span class="n">Module</span> <span class="o">=</span> <span class="s1">&#39;ReLU&#39;</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-7'>
@@ -159,7 +163,7 @@
                 <p>Whether the FFN layer should be gated</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">33</span>    <span class="n">is_gated</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">41</span>    <span class="n">is_gated</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-8'>
@@ -170,7 +174,7 @@
                 <p>Whether the first fully connected layer should have a learnable bias</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">35</span>    <span class="n">bias1</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">43</span>    <span class="n">bias1</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-9'>
@@ -181,7 +185,7 @@
                 <p>Whether the second fully connected layer should have a learnable bias</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">37</span>    <span class="n">bias2</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">45</span>    <span class="n">bias2</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-10'>
@@ -192,7 +196,7 @@
                 <p>Whether the fully connected layer for the gate should have a learnable bias</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">39</span>    <span class="n">bias_gate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">47</span>    <span class="n">bias_gate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-11'>
@@ -203,7 +207,7 @@
                 <p>Predefined GLU variants</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">41</span>    <span class="n">glu_variant</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;none&#39;</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">49</span>    <span class="n">glu_variant</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;none&#39;</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-12'>
@@ -211,11 +215,14 @@
                 <div class='section-link'>
                     <a href='#section-12'>#</a>
                 </div>
-                <p>ReLU activation</p>
+                <h3>ReLU activation</h3>
+<p>
+<script type="math/tex; mode=display">\max(0, x)</script>
+</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">44</span><span class="nd">@option</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span> <span class="s1">&#39;ReLU&#39;</span><span class="p">)</span>
-<span class="lineno">45</span><span class="k">def</span> <span class="nf">_ffn_activation_relu</span><span class="p">():</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">52</span><span class="nd">@option</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span> <span class="s1">&#39;ReLU&#39;</span><span class="p">)</span>
+<span class="lineno">53</span><span class="k">def</span> <span class="nf">_ffn_activation_relu</span><span class="p">():</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-13'>
@@ -226,7 +233,7 @@
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">49</span>    <span class="k">return</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">()</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">59</span>    <span class="k">return</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">()</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-14'>
@@ -234,11 +241,14 @@
                 <div class='section-link'>
                     <a href='#section-14'>#</a>
                 </div>
-                <p>GELU activation</p>
+                <h3>GELU activation</h3>
+<p>
+<script type="math/tex; mode=display">x \Phi(x)</script> where $\Phi(x) = P(X \le x), X \sim \mathcal{N}(0,1)$</p>
+<p>It was introduced in paper <a href="https://arxiv.org/abs/1606.08415">Gaussian Error Linear Units</a>.</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">52</span><span class="nd">@option</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span> <span class="s1">&#39;GELU&#39;</span><span class="p">)</span>
-<span class="lineno">53</span><span class="k">def</span> <span class="nf">_ffn_activation_gelu</span><span class="p">():</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">62</span><span class="nd">@option</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span> <span class="s1">&#39;GELU&#39;</span><span class="p">)</span>
+<span class="lineno">63</span><span class="k">def</span> <span class="nf">_ffn_activation_gelu</span><span class="p">():</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-15'>
@@ -249,7 +259,7 @@
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">57</span>    <span class="k">return</span> <span class="n">nn</span><span class="o">.</span><span class="n">GELU</span><span class="p">()</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">71</span>    <span class="k">return</span> <span class="n">nn</span><span class="o">.</span><span class="n">GELU</span><span class="p">()</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-16'>
@@ -257,11 +267,11 @@
                 <div class='section-link'>
                     <a href='#section-16'>#</a>
                 </div>
-                <p>Create feedforward layer</p>
+                <p>Initialize a <a href="feed_forward.html">feed forward network</a></p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">60</span><span class="nd">@option</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">ffn</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
-<span class="lineno">61</span><span class="k">def</span> <span class="nf">_feed_forward</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">FeedForwardConfigs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">74</span><span class="nd">@option</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">ffn</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
+<span class="lineno">75</span><span class="k">def</span> <span class="nf">_feed_forward</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">FeedForwardConfigs</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-17'>
@@ -272,53 +282,129 @@
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">65</span>    <span class="k">return</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span>
-<span class="lineno">66</span>                       <span class="n">dropout</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span>
-<span class="lineno">67</span>                       <span class="n">activation</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span>
-<span class="lineno">68</span>                       <span class="n">is_gated</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">is_gated</span><span class="p">,</span>
-<span class="lineno">69</span>                       <span class="n">bias1</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">bias1</span><span class="p">,</span>
-<span class="lineno">70</span>                       <span class="n">bias2</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">bias2</span><span class="p">,</span>
-<span class="lineno">71</span>                       <span class="n">bias_gate</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">bias_gate</span><span class="p">)</span>
-<span class="lineno">72</span>
-<span class="lineno">73</span>
-<span class="lineno">74</span><span class="n">aggregate</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">glu_variant</span><span class="p">,</span> <span class="s1">&#39;GLU&#39;</span><span class="p">,</span>
-<span class="lineno">75</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">is_gated</span><span class="p">,</span> <span class="kc">True</span><span class="p">),</span>
-<span class="lineno">76</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias1</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">77</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias2</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">78</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias_gate</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">79</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sigmoid</span><span class="p">()))</span>
-<span class="lineno">80</span>
-<span class="lineno">81</span><span class="n">aggregate</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">glu_variant</span><span class="p">,</span> <span class="s1">&#39;Bilinear&#39;</span><span class="p">,</span>
-<span class="lineno">82</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">is_gated</span><span class="p">,</span> <span class="kc">True</span><span class="p">),</span>
-<span class="lineno">83</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias1</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">84</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias2</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">85</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias_gate</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">86</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">()))</span>
-<span class="lineno">87</span><span class="n">aggregate</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">glu_variant</span><span class="p">,</span> <span class="s1">&#39;ReGLU&#39;</span><span class="p">,</span>
-<span class="lineno">88</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">is_gated</span><span class="p">,</span> <span class="kc">True</span><span class="p">),</span>
-<span class="lineno">89</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias1</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">90</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias2</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">91</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias_gate</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">92</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">()))</span>
-<span class="lineno">93</span><span class="n">aggregate</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">glu_variant</span><span class="p">,</span> <span class="s1">&#39;GEGLU&#39;</span><span class="p">,</span>
-<span class="lineno">94</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">is_gated</span><span class="p">,</span> <span class="kc">True</span><span class="p">),</span>
-<span class="lineno">95</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias1</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">96</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias2</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">97</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias_gate</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">98</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">GELU</span><span class="p">()))</span>
-<span class="lineno">99</span><span class="n">aggregate</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">glu_variant</span><span class="p">,</span> <span class="s1">&#39;SwiGLU&#39;</span><span class="p">,</span>
-<span class="lineno">100</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">is_gated</span><span class="p">,</span> <span class="kc">True</span><span class="p">),</span>
-<span class="lineno">101</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias1</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">102</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias2</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">103</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias_gate</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
-<span class="lineno">104</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">SiLU</span><span class="p">()))</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">79</span>    <span class="k">return</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span>
+<span class="lineno">80</span>                       <span class="n">dropout</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span>
+<span class="lineno">81</span>                       <span class="n">activation</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span>
+<span class="lineno">82</span>                       <span class="n">is_gated</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">is_gated</span><span class="p">,</span>
+<span class="lineno">83</span>                       <span class="n">bias1</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">bias1</span><span class="p">,</span>
+<span class="lineno">84</span>                       <span class="n">bias2</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">bias2</span><span class="p">,</span>
+<span class="lineno">85</span>                       <span class="n">bias_gate</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">bias_gate</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-18'>
-        <div class='docs doc-strings'>
+            <div class='docs'>
                 <div class='section-link'>
                     <a href='#section-18'>#</a>
                 </div>
+                <h2>GLU Variants</h2>
+<p>These are variants with gated hidden layers for the FFN
+as introduced in paper <a href="https://arxiv.org/abs/2002.05202">GLU Variants Improve Transformer</a>.
+We have omitted the bias terms as specified in the paper.</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-19'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-19'>#</a>
+                </div>
+                <h3>FFN with Gated Linear Units</h3>
+<p>
+<script type="math/tex; mode=display">FFN_{GLU}(x)(x, W_1, V, W_2) = (\sigma(x W_1) \otimes x V) W_2</script>
+</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">95</span><span class="n">aggregate</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">glu_variant</span><span class="p">,</span> <span class="s1">&#39;GLU&#39;</span><span class="p">,</span>
+<span class="lineno">96</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">is_gated</span><span class="p">,</span> <span class="kc">True</span><span class="p">),</span>
+<span class="lineno">97</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias1</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">98</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias2</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">99</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias_gate</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">100</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sigmoid</span><span class="p">()))</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-20'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-20'>#</a>
+                </div>
+                <h3>FFN with Bilinear hidden layer</h3>
+<p>
+<script type="math/tex; mode=display">FFN_{Bilinear}(x)(x, W_1, V, W_2) = (x W_1 \otimes x V) W_2</script>
+</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">105</span><span class="n">aggregate</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">glu_variant</span><span class="p">,</span> <span class="s1">&#39;Bilinear&#39;</span><span class="p">,</span>
+<span class="lineno">106</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">is_gated</span><span class="p">,</span> <span class="kc">True</span><span class="p">),</span>
+<span class="lineno">107</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias1</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">108</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias2</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">109</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias_gate</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">110</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">()))</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-21'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-21'>#</a>
+                </div>
+                <h3>FFN with ReLU gate</h3>
+<p>
+<script type="math/tex; mode=display">FFN_{ReGLU}(x)(x, W_1, V, W_2) = (\max(0, x W_1) \otimes x V) W_2</script>
+</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">115</span><span class="n">aggregate</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">glu_variant</span><span class="p">,</span> <span class="s1">&#39;ReGLU&#39;</span><span class="p">,</span>
+<span class="lineno">116</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">is_gated</span><span class="p">,</span> <span class="kc">True</span><span class="p">),</span>
+<span class="lineno">117</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias1</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">118</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias2</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">119</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias_gate</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">120</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">()))</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-22'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-22'>#</a>
+                </div>
+                <h3>FFN with GELU gate</h3>
+<p>
+<script type="math/tex; mode=display">FFN_{GEGLU}(x)(x, W_1, V, W_2) = (\text{GELU}(x W_1) \otimes x V) W_2</script>
+</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">125</span><span class="n">aggregate</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">glu_variant</span><span class="p">,</span> <span class="s1">&#39;GEGLU&#39;</span><span class="p">,</span>
+<span class="lineno">126</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">is_gated</span><span class="p">,</span> <span class="kc">True</span><span class="p">),</span>
+<span class="lineno">127</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias1</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">128</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias2</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">129</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias_gate</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">130</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">GELU</span><span class="p">()))</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-23'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-23'>#</a>
+                </div>
+                <h3>FFN with Swish gate</h3>
+<p>
+<script type="math/tex; mode=display">FFN_{SwiGLU}(x)(x, W_1, V, W_2) = (\text{Swish}_1(x W_1) \otimes x V) W_2</script>
+where $\text{Swish}_\beta(x) = x \sigma(\beta x)$</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">136</span><span class="n">aggregate</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">glu_variant</span><span class="p">,</span> <span class="s1">&#39;SwiGLU&#39;</span><span class="p">,</span>
+<span class="lineno">137</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">is_gated</span><span class="p">,</span> <span class="kc">True</span><span class="p">),</span>
+<span class="lineno">138</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias1</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">139</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias2</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">140</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">bias_gate</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+<span class="lineno">141</span>          <span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">activation</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">SiLU</span><span class="p">()))</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-24'>
+        <div class='docs doc-strings'>
+                <div class='section-link'>
+                    <a href='#section-24'>#</a>
+                </div>
                 <p><a id="TransformerConfigs"></p>
 <h2>Transformer Configurations</h2>
 <p></a></p>
@@ -328,73 +414,7 @@ These are lazy loaded and therefore only the necessary modules
 are calculated.</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">107</span><span class="k">class</span> <span class="nc">TransformerConfigs</span><span class="p">(</span><span class="n">BaseConfigs</span><span class="p">):</span></pre></div>
-            </div>
-        </div>
-    <div class='section' id='section-19'>
-            <div class='docs'>
-                <div class='section-link'>
-                    <a href='#section-19'>#</a>
-                </div>
-                <p>Number of attention heads</p>
-            </div>
-            <div class='code'>
-                <div class="highlight"><pre><span class="lineno">119</span>    <span class="n">n_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span></pre></div>
-            </div>
-        </div>
-    <div class='section' id='section-20'>
-            <div class='docs'>
-                <div class='section-link'>
-                    <a href='#section-20'>#</a>
-                </div>
-                <p>Transformer embedding size</p>
-            </div>
-            <div class='code'>
-                <div class="highlight"><pre><span class="lineno">121</span>    <span class="n">d_model</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">512</span></pre></div>
-            </div>
-        </div>
-    <div class='section' id='section-21'>
-            <div class='docs'>
-                <div class='section-link'>
-                    <a href='#section-21'>#</a>
-                </div>
-                <p>Number of layers</p>
-            </div>
-            <div class='code'>
-                <div class="highlight"><pre><span class="lineno">123</span>    <span class="n">n_layers</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">6</span></pre></div>
-            </div>
-        </div>
-    <div class='section' id='section-22'>
-            <div class='docs'>
-                <div class='section-link'>
-                    <a href='#section-22'>#</a>
-                </div>
-                <p>Dropout probability</p>
-            </div>
-            <div class='code'>
-                <div class="highlight"><pre><span class="lineno">125</span>    <span class="n">dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span></pre></div>
-            </div>
-        </div>
-    <div class='section' id='section-23'>
-            <div class='docs'>
-                <div class='section-link'>
-                    <a href='#section-23'>#</a>
-                </div>
-                <p>Number of tokens in the source vocabulary (for token embeddings)</p>
-            </div>
-            <div class='code'>
-                <div class="highlight"><pre><span class="lineno">127</span>    <span class="n">n_src_vocab</span><span class="p">:</span> <span class="nb">int</span></pre></div>
-            </div>
-        </div>
-    <div class='section' id='section-24'>
-            <div class='docs'>
-                <div class='section-link'>
-                    <a href='#section-24'>#</a>
-                </div>
-                <p>Number of tokens in the target vocabulary (to generate logits for prediction)</p>
-            </div>
-            <div class='code'>
-                <div class="highlight"><pre><span class="lineno">129</span>    <span class="n">n_tgt_vocab</span><span class="p">:</span> <span class="nb">int</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">144</span><span class="k">class</span> <span class="nc">TransformerConfigs</span><span class="p">(</span><span class="n">BaseConfigs</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-25'>
@@ -402,10 +422,10 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-25'>#</a>
                 </div>
-                <p>The encoder self attention</p>
+                <p>Number of attention heads</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">132</span>    <span class="n">encoder_attn</span><span class="p">:</span> <span class="n">MultiHeadAttention</span> <span class="o">=</span> <span class="s1">&#39;mha&#39;</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">156</span>    <span class="n">n_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-26'>
@@ -413,10 +433,10 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-26'>#</a>
                 </div>
-                <p>The decoder self attention</p>
+                <p>Transformer embedding size</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">134</span>    <span class="n">decoder_attn</span><span class="p">:</span> <span class="n">MultiHeadAttention</span> <span class="o">=</span> <span class="s1">&#39;mha&#39;</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">158</span>    <span class="n">d_model</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">512</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-27'>
@@ -424,10 +444,10 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-27'>#</a>
                 </div>
-                <p>The decoder memory attention</p>
+                <p>Number of layers</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">136</span>    <span class="n">decoder_mem_attn</span><span class="p">:</span> <span class="n">MultiHeadAttention</span> <span class="o">=</span> <span class="s1">&#39;mha&#39;</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">160</span>    <span class="n">n_layers</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">6</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-28'>
@@ -435,10 +455,10 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-28'>#</a>
                 </div>
-                <p>Configurable Feedforward Layer</p>
+                <p>Dropout probability</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">139</span>    <span class="n">ffn</span><span class="p">:</span> <span class="n">FeedForwardConfigs</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">162</span>    <span class="n">dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-29'>
@@ -446,10 +466,10 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-29'>#</a>
                 </div>
-                <p>Encoder layer</p>
+                <p>Number of tokens in the source vocabulary (for token embeddings)</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">142</span>    <span class="n">encoder_layer</span><span class="p">:</span> <span class="n">TransformerLayer</span> <span class="o">=</span> <span class="s1">&#39;default&#39;</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">164</span>    <span class="n">n_src_vocab</span><span class="p">:</span> <span class="nb">int</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-30'>
@@ -457,10 +477,10 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-30'>#</a>
                 </div>
-                <p>Decoder layer</p>
+                <p>Number of tokens in the target vocabulary (to generate logits for prediction)</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">144</span>    <span class="n">decoder_layer</span><span class="p">:</span> <span class="n">TransformerLayer</span> <span class="o">=</span> <span class="s1">&#39;default&#39;</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">166</span>    <span class="n">n_tgt_vocab</span><span class="p">:</span> <span class="nb">int</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-31'>
@@ -468,10 +488,10 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-31'>#</a>
                 </div>
-                <p>Encoder consisting of multiple encoder layers</p>
+                <p>The encoder self attention</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">147</span>    <span class="n">encoder</span><span class="p">:</span> <span class="n">Encoder</span> <span class="o">=</span> <span class="s1">&#39;default&#39;</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">169</span>    <span class="n">encoder_attn</span><span class="p">:</span> <span class="n">MultiHeadAttention</span> <span class="o">=</span> <span class="s1">&#39;mha&#39;</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-32'>
@@ -479,10 +499,10 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-32'>#</a>
                 </div>
-                <p>Encoder consisting of multiple decoder layers</p>
+                <p>The decoder self attention</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">149</span>    <span class="n">decoder</span><span class="p">:</span> <span class="n">Decoder</span> <span class="o">=</span> <span class="s1">&#39;default&#39;</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">171</span>    <span class="n">decoder_attn</span><span class="p">:</span> <span class="n">MultiHeadAttention</span> <span class="o">=</span> <span class="s1">&#39;mha&#39;</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-33'>
@@ -490,10 +510,10 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-33'>#</a>
                 </div>
-                <p>Embedding layer for source</p>
+                <p>The decoder memory attention</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">152</span>    <span class="n">src_embed</span><span class="p">:</span> <span class="n">Module</span> <span class="o">=</span> <span class="s1">&#39;fixed_pos&#39;</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">173</span>    <span class="n">decoder_mem_attn</span><span class="p">:</span> <span class="n">MultiHeadAttention</span> <span class="o">=</span> <span class="s1">&#39;mha&#39;</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-34'>
@@ -501,10 +521,10 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-34'>#</a>
                 </div>
-                <p>Embedding layer for target (for decoder)</p>
+                <p>Configurable Feedforward Layer</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">154</span>    <span class="n">tgt_embed</span><span class="p">:</span> <span class="n">Module</span> <span class="o">=</span> <span class="s1">&#39;fixed_pos&#39;</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">176</span>    <span class="n">ffn</span><span class="p">:</span> <span class="n">FeedForwardConfigs</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-35'>
@@ -512,10 +532,10 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-35'>#</a>
                 </div>
-                <p>Logit generator for prediction</p>
+                <p>Encoder layer</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">157</span>    <span class="n">generator</span><span class="p">:</span> <span class="n">Generator</span> <span class="o">=</span> <span class="s1">&#39;default&#39;</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">179</span>    <span class="n">encoder_layer</span><span class="p">:</span> <span class="n">TransformerLayer</span> <span class="o">=</span> <span class="s1">&#39;default&#39;</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-36'>
@@ -523,10 +543,10 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-36'>#</a>
                 </div>
-                <p>Encoder-decoder</p>
+                <p>Decoder layer</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">160</span>    <span class="n">encoder_decoder</span><span class="p">:</span> <span class="n">EncoderDecoder</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">181</span>    <span class="n">decoder_layer</span><span class="p">:</span> <span class="n">TransformerLayer</span> <span class="o">=</span> <span class="s1">&#39;default&#39;</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-37'>
@@ -534,16 +554,10 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-37'>#</a>
                 </div>
-                <h3>Multi-head Attention</h3>
+                <p>Encoder consisting of multiple encoder layers</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">164</span><span class="k">def</span> <span class="nf">_mha</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span>
-<span class="lineno">165</span>    <span class="k">return</span> <span class="n">MultiHeadAttention</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">n_heads</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
-<span class="lineno">166</span>
-<span class="lineno">167</span>
-<span class="lineno">168</span><span class="n">calculate</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">encoder_attn</span><span class="p">,</span> <span class="s1">&#39;mha&#39;</span><span class="p">,</span> <span class="n">_mha</span><span class="p">)</span>
-<span class="lineno">169</span><span class="n">calculate</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">decoder_attn</span><span class="p">,</span> <span class="s1">&#39;mha&#39;</span><span class="p">,</span> <span class="n">_mha</span><span class="p">)</span>
-<span class="lineno">170</span><span class="n">calculate</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">decoder_mem_attn</span><span class="p">,</span> <span class="s1">&#39;mha&#39;</span><span class="p">,</span> <span class="n">_mha</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">184</span>    <span class="n">encoder</span><span class="p">:</span> <span class="n">Encoder</span> <span class="o">=</span> <span class="s1">&#39;default&#39;</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-38'>
@@ -551,29 +565,21 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-38'>#</a>
                 </div>
-                <h3>Relative Multi-head Attention</h3>
+                <p>Encoder consisting of multiple decoder layers</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">174</span><span class="k">def</span> <span class="nf">_relative_mha</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span>
-<span class="lineno">175</span>    <span class="kn">from</span> <span class="nn">.relative_mha</span> <span class="kn">import</span> <span class="n">RelativeMultiHeadAttention</span>
-<span class="lineno">176</span>    <span class="k">return</span> <span class="n">RelativeMultiHeadAttention</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">n_heads</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
-<span class="lineno">177</span>
-<span class="lineno">178</span>
-<span class="lineno">179</span><span class="n">calculate</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">encoder_attn</span><span class="p">,</span> <span class="s1">&#39;relative&#39;</span><span class="p">,</span> <span class="n">_relative_mha</span><span class="p">)</span>
-<span class="lineno">180</span><span class="n">calculate</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">decoder_attn</span><span class="p">,</span> <span class="s1">&#39;relative&#39;</span><span class="p">,</span> <span class="n">_relative_mha</span><span class="p">)</span>
-<span class="lineno">181</span><span class="n">calculate</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">decoder_mem_attn</span><span class="p">,</span> <span class="s1">&#39;relative&#39;</span><span class="p">,</span> <span class="n">_relative_mha</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">186</span>    <span class="n">decoder</span><span class="p">:</span> <span class="n">Decoder</span> <span class="o">=</span> <span class="s1">&#39;default&#39;</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-39'>
-        <div class='docs doc-strings'>
+            <div class='docs'>
                 <div class='section-link'>
                     <a href='#section-39'>#</a>
                 </div>
-                <p>Create feedforward layer configurations</p>
+                <p>Embedding layer for source</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">184</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">ffn</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
-<span class="lineno">185</span><span class="k">def</span> <span class="nf">_feed_forward</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">189</span>    <span class="n">src_embed</span><span class="p">:</span> <span class="n">Module</span> <span class="o">=</span> <span class="s1">&#39;fixed_pos&#39;</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-40'>
@@ -581,25 +587,21 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-40'>#</a>
                 </div>
-                
+                <p>Embedding layer for target (for decoder)</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">189</span>    <span class="n">conf</span> <span class="o">=</span> <span class="n">FeedForwardConfigs</span><span class="p">()</span>
-<span class="lineno">190</span>    <span class="n">conf</span><span class="o">.</span><span class="n">set_default</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">func</span><span class="o">=</span><span class="k">lambda</span><span class="p">:</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
-<span class="lineno">191</span>    <span class="n">conf</span><span class="o">.</span><span class="n">set_default</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">func</span><span class="o">=</span><span class="k">lambda</span><span class="p">:</span> <span class="n">c</span><span class="o">.</span><span class="n">dropout</span><span class="p">)</span>
-<span class="lineno">192</span>    <span class="k">return</span> <span class="n">conf</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">191</span>    <span class="n">tgt_embed</span><span class="p">:</span> <span class="n">Module</span> <span class="o">=</span> <span class="s1">&#39;fixed_pos&#39;</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-41'>
-        <div class='docs doc-strings'>
+            <div class='docs'>
                 <div class='section-link'>
                     <a href='#section-41'>#</a>
                 </div>
-                <p>Encoder layer</p>
+                <p>Logit generator for prediction</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">195</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">encoder_layer</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
-<span class="lineno">196</span><span class="k">def</span> <span class="nf">_encoder_layer</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">194</span>    <span class="n">generator</span><span class="p">:</span> <span class="n">Generator</span> <span class="o">=</span> <span class="s1">&#39;default&#39;</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-42'>
@@ -607,24 +609,27 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-42'>#</a>
                 </div>
-                
+                <p>Encoder-decoder</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">200</span>    <span class="k">return</span> <span class="n">TransformerLayer</span><span class="p">(</span><span class="n">d_model</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">self_attn</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">encoder_attn</span><span class="p">,</span>
-<span class="lineno">201</span>                            <span class="n">src_attn</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">feed_forward</span><span class="o">=</span><span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">ffn</span><span class="o">.</span><span class="n">ffn</span><span class="p">),</span>
-<span class="lineno">202</span>                            <span class="n">dropout_prob</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">dropout</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">197</span>    <span class="n">encoder_decoder</span><span class="p">:</span> <span class="n">EncoderDecoder</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-43'>
-        <div class='docs doc-strings'>
+            <div class='docs'>
                 <div class='section-link'>
                     <a href='#section-43'>#</a>
                 </div>
-                <p>Decoder layer</p>
+                <h3>Multi-head Attention</h3>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">205</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">decoder_layer</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
-<span class="lineno">206</span><span class="k">def</span> <span class="nf">_decoder_layer</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">201</span><span class="k">def</span> <span class="nf">_mha</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span>
+<span class="lineno">202</span>    <span class="k">return</span> <span class="n">MultiHeadAttention</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">n_heads</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
+<span class="lineno">203</span>
+<span class="lineno">204</span>
+<span class="lineno">205</span><span class="n">calculate</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">encoder_attn</span><span class="p">,</span> <span class="s1">&#39;mha&#39;</span><span class="p">,</span> <span class="n">_mha</span><span class="p">)</span>
+<span class="lineno">206</span><span class="n">calculate</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">decoder_attn</span><span class="p">,</span> <span class="s1">&#39;mha&#39;</span><span class="p">,</span> <span class="n">_mha</span><span class="p">)</span>
+<span class="lineno">207</span><span class="n">calculate</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">decoder_mem_attn</span><span class="p">,</span> <span class="s1">&#39;mha&#39;</span><span class="p">,</span> <span class="n">_mha</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-44'>
@@ -632,12 +637,17 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-44'>#</a>
                 </div>
-                
+                <h3>Relative Multi-head Attention</h3>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">210</span>    <span class="k">return</span> <span class="n">TransformerLayer</span><span class="p">(</span><span class="n">d_model</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">self_attn</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">decoder_attn</span><span class="p">,</span>
-<span class="lineno">211</span>                            <span class="n">src_attn</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">decoder_mem_attn</span><span class="p">,</span> <span class="n">feed_forward</span><span class="o">=</span><span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">ffn</span><span class="o">.</span><span class="n">ffn</span><span class="p">),</span>
-<span class="lineno">212</span>                            <span class="n">dropout_prob</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">dropout</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">211</span><span class="k">def</span> <span class="nf">_relative_mha</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span>
+<span class="lineno">212</span>    <span class="kn">from</span> <span class="nn">.relative_mha</span> <span class="kn">import</span> <span class="n">RelativeMultiHeadAttention</span>
+<span class="lineno">213</span>    <span class="k">return</span> <span class="n">RelativeMultiHeadAttention</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">n_heads</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
+<span class="lineno">214</span>
+<span class="lineno">215</span>
+<span class="lineno">216</span><span class="n">calculate</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">encoder_attn</span><span class="p">,</span> <span class="s1">&#39;relative&#39;</span><span class="p">,</span> <span class="n">_relative_mha</span><span class="p">)</span>
+<span class="lineno">217</span><span class="n">calculate</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">decoder_attn</span><span class="p">,</span> <span class="s1">&#39;relative&#39;</span><span class="p">,</span> <span class="n">_relative_mha</span><span class="p">)</span>
+<span class="lineno">218</span><span class="n">calculate</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">decoder_mem_attn</span><span class="p">,</span> <span class="s1">&#39;relative&#39;</span><span class="p">,</span> <span class="n">_relative_mha</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-45'>
@@ -645,11 +655,11 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-45'>#</a>
                 </div>
-                <p>Encoder</p>
+                <p>Create feedforward layer configurations</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">215</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">encoder</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
-<span class="lineno">216</span><span class="k">def</span> <span class="nf">_encoder</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">221</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">ffn</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
+<span class="lineno">222</span><span class="k">def</span> <span class="nf">_feed_forward</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-46'>
@@ -660,7 +670,10 @@ are calculated.</p>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">220</span>    <span class="k">return</span> <span class="n">Encoder</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">encoder_layer</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">n_layers</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">226</span>    <span class="n">conf</span> <span class="o">=</span> <span class="n">FeedForwardConfigs</span><span class="p">()</span>
+<span class="lineno">227</span>    <span class="n">conf</span><span class="o">.</span><span class="n">set_default</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">func</span><span class="o">=</span><span class="k">lambda</span><span class="p">:</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
+<span class="lineno">228</span>    <span class="n">conf</span><span class="o">.</span><span class="n">set_default</span><span class="p">(</span><span class="n">FeedForwardConfigs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">func</span><span class="o">=</span><span class="k">lambda</span><span class="p">:</span> <span class="n">c</span><span class="o">.</span><span class="n">dropout</span><span class="p">)</span>
+<span class="lineno">229</span>    <span class="k">return</span> <span class="n">conf</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-47'>
@@ -668,11 +681,11 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-47'>#</a>
                 </div>
-                <p>Decoder</p>
+                <p>Encoder layer</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">223</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">decoder</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
-<span class="lineno">224</span><span class="k">def</span> <span class="nf">_decoder</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">232</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">encoder_layer</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
+<span class="lineno">233</span><span class="k">def</span> <span class="nf">_encoder_layer</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-48'>
@@ -683,7 +696,9 @@ are calculated.</p>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">228</span>    <span class="k">return</span> <span class="n">Decoder</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">decoder_layer</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">n_layers</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">237</span>    <span class="k">return</span> <span class="n">TransformerLayer</span><span class="p">(</span><span class="n">d_model</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">self_attn</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">encoder_attn</span><span class="p">,</span>
+<span class="lineno">238</span>                            <span class="n">src_attn</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">feed_forward</span><span class="o">=</span><span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">ffn</span><span class="o">.</span><span class="n">ffn</span><span class="p">),</span>
+<span class="lineno">239</span>                            <span class="n">dropout_prob</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">dropout</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-49'>
@@ -691,11 +706,11 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-49'>#</a>
                 </div>
-                <p>Logit generator</p>
+                <p>Decoder layer</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">231</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">generator</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
-<span class="lineno">232</span><span class="k">def</span> <span class="nf">_generator</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">242</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">decoder_layer</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
+<span class="lineno">243</span><span class="k">def</span> <span class="nf">_decoder_layer</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-50'>
@@ -706,7 +721,9 @@ are calculated.</p>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">236</span>    <span class="k">return</span> <span class="n">Generator</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">n_tgt_vocab</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">247</span>    <span class="k">return</span> <span class="n">TransformerLayer</span><span class="p">(</span><span class="n">d_model</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">self_attn</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">decoder_attn</span><span class="p">,</span>
+<span class="lineno">248</span>                            <span class="n">src_attn</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">decoder_mem_attn</span><span class="p">,</span> <span class="n">feed_forward</span><span class="o">=</span><span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">ffn</span><span class="o">.</span><span class="n">ffn</span><span class="p">),</span>
+<span class="lineno">249</span>                            <span class="n">dropout_prob</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">dropout</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-51'>
@@ -714,12 +731,11 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-51'>#</a>
                 </div>
-                <h2>Positional Embeddings</h2>
-<p>Source embedding with fixed positional encodings</p>
+                <p>Encoder</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">240</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">src_embed</span><span class="p">,</span> <span class="s1">&#39;fixed_pos&#39;</span><span class="p">)</span>
-<span class="lineno">241</span><span class="k">def</span> <span class="nf">_src_embed_with_positional</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">252</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">encoder</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
+<span class="lineno">253</span><span class="k">def</span> <span class="nf">_encoder</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-52'>
@@ -730,7 +746,7 @@ are calculated.</p>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">245</span>    <span class="k">return</span> <span class="n">EmbeddingsWithPositionalEncoding</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">n_src_vocab</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">257</span>    <span class="k">return</span> <span class="n">Encoder</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">encoder_layer</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">n_layers</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-53'>
@@ -738,11 +754,11 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-53'>#</a>
                 </div>
-                <p>Target embedding with fixed positional encodings</p>
+                <p>Decoder</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">248</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">tgt_embed</span><span class="p">,</span> <span class="s1">&#39;fixed_pos&#39;</span><span class="p">)</span>
-<span class="lineno">249</span><span class="k">def</span> <span class="nf">_tgt_embed_with_positional</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">260</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">decoder</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
+<span class="lineno">261</span><span class="k">def</span> <span class="nf">_decoder</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-54'>
@@ -753,7 +769,7 @@ are calculated.</p>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">253</span>    <span class="k">return</span> <span class="n">EmbeddingsWithPositionalEncoding</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">n_tgt_vocab</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">265</span>    <span class="k">return</span> <span class="n">Decoder</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">decoder_layer</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">n_layers</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-55'>
@@ -761,12 +777,11 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-55'>#</a>
                 </div>
-                <h2>Learned Positional Embeddings</h2>
-<p>Source embedding with learned positional encodings</p>
+                <p>Logit generator</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">257</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">src_embed</span><span class="p">,</span> <span class="s1">&#39;learned_pos&#39;</span><span class="p">)</span>
-<span class="lineno">258</span><span class="k">def</span> <span class="nf">_src_embed_with_learned_positional</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">268</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">generator</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
+<span class="lineno">269</span><span class="k">def</span> <span class="nf">_generator</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-56'>
@@ -777,7 +792,7 @@ are calculated.</p>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">262</span>    <span class="k">return</span> <span class="n">EmbeddingsWithLearnedPositionalEncoding</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">n_src_vocab</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">273</span>    <span class="k">return</span> <span class="n">Generator</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">n_tgt_vocab</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-57'>
@@ -785,11 +800,12 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-57'>#</a>
                 </div>
-                <p>Target embedding with learned positional encodings</p>
+                <h3>Fixed Positional Embeddings</h3>
+<p>Source embedding with fixed positional encodings</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">265</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">tgt_embed</span><span class="p">,</span> <span class="s1">&#39;learned_pos&#39;</span><span class="p">)</span>
-<span class="lineno">266</span><span class="k">def</span> <span class="nf">_tgt_embed_with_learned_positional</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">277</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">src_embed</span><span class="p">,</span> <span class="s1">&#39;fixed_pos&#39;</span><span class="p">)</span>
+<span class="lineno">278</span><span class="k">def</span> <span class="nf">_src_embed_with_positional</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-58'>
@@ -800,7 +816,7 @@ are calculated.</p>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">270</span>    <span class="k">return</span> <span class="n">EmbeddingsWithLearnedPositionalEncoding</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">n_tgt_vocab</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">282</span>    <span class="k">return</span> <span class="n">EmbeddingsWithPositionalEncoding</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">n_src_vocab</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-59'>
@@ -808,12 +824,11 @@ are calculated.</p>
                 <div class='section-link'>
                     <a href='#section-59'>#</a>
                 </div>
-                <h2>No Positional Embeddings</h2>
-<p>Source embedding without positional encodings</p>
+                <p>Target embedding with fixed positional encodings</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">274</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">src_embed</span><span class="p">,</span> <span class="s1">&#39;no_pos&#39;</span><span class="p">)</span>
-<span class="lineno">275</span><span class="k">def</span> <span class="nf">_src_embed_without_positional</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">285</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">tgt_embed</span><span class="p">,</span> <span class="s1">&#39;fixed_pos&#39;</span><span class="p">)</span>
+<span class="lineno">286</span><span class="k">def</span> <span class="nf">_tgt_embed_with_positional</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-60'>
@@ -824,25 +839,96 @@ are calculated.</p>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">279</span>    <span class="k">return</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">n_src_vocab</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">290</span>    <span class="k">return</span> <span class="n">EmbeddingsWithPositionalEncoding</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">n_tgt_vocab</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-61'>
-            <div class='docs'>
+        <div class='docs doc-strings'>
                 <div class='section-link'>
                     <a href='#section-61'>#</a>
                 </div>
+                <h3>Learned Positional Embeddings</h3>
+<p>Source embedding with learned positional encodings</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">294</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">src_embed</span><span class="p">,</span> <span class="s1">&#39;learned_pos&#39;</span><span class="p">)</span>
+<span class="lineno">295</span><span class="k">def</span> <span class="nf">_src_embed_with_learned_positional</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-62'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-62'>#</a>
+                </div>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">282</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">tgt_embed</span><span class="p">,</span> <span class="s1">&#39;no_pos&#39;</span><span class="p">)</span>
-<span class="lineno">283</span><span class="k">def</span> <span class="nf">_tgt_embed_without_positional</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span>
-<span class="lineno">284</span>    <span class="k">return</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">n_tgt_vocab</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
-<span class="lineno">285</span>
-<span class="lineno">286</span>
-<span class="lineno">287</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">encoder_decoder</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
-<span class="lineno">288</span><span class="k">def</span> <span class="nf">_encoder_decoder</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span>
-<span class="lineno">289</span>    <span class="k">return</span> <span class="n">EncoderDecoder</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">encoder</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">decoder</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">src_embed</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">tgt_embed</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">generator</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">299</span>    <span class="k">return</span> <span class="n">EmbeddingsWithLearnedPositionalEncoding</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">n_src_vocab</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-63'>
+        <div class='docs doc-strings'>
+                <div class='section-link'>
+                    <a href='#section-63'>#</a>
+                </div>
+                <p>Target embedding with learned positional encodings</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">302</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">tgt_embed</span><span class="p">,</span> <span class="s1">&#39;learned_pos&#39;</span><span class="p">)</span>
+<span class="lineno">303</span><span class="k">def</span> <span class="nf">_tgt_embed_with_learned_positional</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-64'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-64'>#</a>
+                </div>
+                
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">307</span>    <span class="k">return</span> <span class="n">EmbeddingsWithLearnedPositionalEncoding</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">n_tgt_vocab</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-65'>
+        <div class='docs doc-strings'>
+                <div class='section-link'>
+                    <a href='#section-65'>#</a>
+                </div>
+                <h3>No Positional Embeddings</h3>
+<p>Source embedding without positional encodings</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">311</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">src_embed</span><span class="p">,</span> <span class="s1">&#39;no_pos&#39;</span><span class="p">)</span>
+<span class="lineno">312</span><span class="k">def</span> <span class="nf">_src_embed_without_positional</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-66'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-66'>#</a>
+                </div>
+                
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">316</span>    <span class="k">return</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">n_src_vocab</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-67'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-67'>#</a>
+                </div>
+                
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">319</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">tgt_embed</span><span class="p">,</span> <span class="s1">&#39;no_pos&#39;</span><span class="p">)</span>
+<span class="lineno">320</span><span class="k">def</span> <span class="nf">_tgt_embed_without_positional</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span>
+<span class="lineno">321</span>    <span class="k">return</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">n_tgt_vocab</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
+<span class="lineno">322</span>
+<span class="lineno">323</span>
+<span class="lineno">324</span><span class="nd">@option</span><span class="p">(</span><span class="n">TransformerConfigs</span><span class="o">.</span><span class="n">encoder_decoder</span><span class="p">,</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
+<span class="lineno">325</span><span class="k">def</span> <span class="nf">_encoder_decoder</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span>
+<span class="lineno">326</span>    <span class="k">return</span> <span class="n">EncoderDecoder</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">encoder</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">decoder</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">src_embed</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">tgt_embed</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">generator</span><span class="p">)</span></pre></div>
             </div>
         </div>
     </div>
diff --git a/docs/transformers/feed_forward.html b/docs/transformers/feed_forward.html
index 053e69b2..e263236f 100644
--- a/docs/transformers/feed_forward.html
+++ b/docs/transformers/feed_forward.html
@@ -84,12 +84,20 @@ where $W_1$, $W_2$, $b_1$ and $b_2$ are learnable parameters.</p>
 <p>Sometimes the
 GELU (Gaussian Error Linear Unit) activation is also used instead of ReLU.
 <script type="math/tex; mode=display">x \Phi(x)</script> where $\Phi(x) = P(X \le x), X \sim \mathcal{N}(0,1)$</p>
+<h3>Gated Linear Units</h3>
+<p>This is a generic implementation that supports different variants including
+<a href="https://arxiv.org/abs/2002.05202">Gated Linear Units</a> (GLU).
+We have also implemented experiments on these:</p>
+<ul>
+<li><a href="glu_variants/experiment.html">experiment that uses <code>labml.configs</code></a></li>
+<li><a href="glu_variants/simple.html">simpler version from scratch</a></li>
+</ul>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">26</span><span></span><span class="kn">import</span> <span class="nn">torch</span>
-<span class="lineno">27</span><span class="kn">from</span> <span class="nn">torch</span> <span class="kn">import</span> <span class="n">nn</span> <span class="k">as</span> <span class="n">nn</span>
-<span class="lineno">28</span>
-<span class="lineno">29</span><span class="kn">from</span> <span class="nn">labml_helpers.module</span> <span class="kn">import</span> <span class="n">Module</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">35</span><span></span><span class="kn">import</span> <span class="nn">torch</span>
+<span class="lineno">36</span><span class="kn">from</span> <span class="nn">torch</span> <span class="kn">import</span> <span class="n">nn</span> <span class="k">as</span> <span class="n">nn</span>
+<span class="lineno">37</span>
+<span class="lineno">38</span><span class="kn">from</span> <span class="nn">labml_helpers.module</span> <span class="kn">import</span> <span class="n">Module</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-1'>
@@ -97,10 +105,10 @@ GELU (Gaussian Error Linear Unit) activation is also used instead of ReLU.
                 <div class='section-link'>
                     <a href='#section-1'>#</a>
                 </div>
-                <h2>Position-wise feed-forward network (FFN) module</h2>
+                <h2>FFN module</h2>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">32</span><span class="k">class</span> <span class="nc">FeedForward</span><span class="p">(</span><span class="n">Module</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">41</span><span class="k">class</span> <span class="nc">FeedForward</span><span class="p">(</span><span class="n">Module</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-2'>
@@ -119,13 +127,13 @@ GELU (Gaussian Error Linear Unit) activation is also used instead of ReLU.
 </ul>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">37</span>    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">d_model</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">d_ff</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-<span class="lineno">38</span>                 <span class="n">dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-<span class="lineno">39</span>                 <span class="n">activation</span><span class="o">=</span><span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">(),</span>
-<span class="lineno">40</span>                 <span class="n">is_gated</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-<span class="lineno">41</span>                 <span class="n">bias1</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-<span class="lineno">42</span>                 <span class="n">bias2</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-<span class="lineno">43</span>                 <span class="n">bias_gate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">46</span>    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">d_model</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">d_ff</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+<span class="lineno">47</span>                 <span class="n">dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+<span class="lineno">48</span>                 <span class="n">activation</span><span class="o">=</span><span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">(),</span>
+<span class="lineno">49</span>                 <span class="n">is_gated</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+<span class="lineno">50</span>                 <span class="n">bias1</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+<span class="lineno">51</span>                 <span class="n">bias2</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+<span class="lineno">52</span>                 <span class="n">bias_gate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-3'>
@@ -136,14 +144,7 @@ GELU (Gaussian Error Linear Unit) activation is also used instead of ReLU.
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">53</span>        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-<span class="lineno">54</span>        <span class="bp">self</span><span class="o">.</span><span class="n">layer1</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">d_ff</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="n">bias1</span><span class="p">)</span>
-<span class="lineno">55</span>        <span class="bp">self</span><span class="o">.</span><span class="n">layer2</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">d_model</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="n">bias2</span><span class="p">)</span>
-<span class="lineno">56</span>        <span class="bp">self</span><span class="o">.</span><span class="n">dropout</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Dropout</span><span class="p">(</span><span class="n">dropout</span><span class="p">)</span>
-<span class="lineno">57</span>        <span class="bp">self</span><span class="o">.</span><span class="n">activation</span> <span class="o">=</span> <span class="n">activation</span>
-<span class="lineno">58</span>        <span class="bp">self</span><span class="o">.</span><span class="n">is_gated</span> <span class="o">=</span> <span class="n">is_gated</span>
-<span class="lineno">59</span>        <span class="k">if</span> <span class="n">is_gated</span><span class="p">:</span>
-<span class="lineno">60</span>            <span class="bp">self</span><span class="o">.</span><span class="n">linear_v</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">d_ff</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="n">bias_gate</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">62</span>        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-4'>
@@ -151,17 +152,136 @@ GELU (Gaussian Error Linear Unit) activation is also used instead of ReLU.
                 <div class='section-link'>
                     <a href='#section-4'>#</a>
                 </div>
+                <p>Layer one parameterized by weight $W_1$ and bias $b_1$</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">64</span>        <span class="bp">self</span><span class="o">.</span><span class="n">layer1</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">d_ff</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="n">bias1</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-5'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-5'>#</a>
+                </div>
+                <p>Layer one parameterized by weight $W_1$ and bias $b_1$</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">66</span>        <span class="bp">self</span><span class="o">.</span><span class="n">layer2</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">d_model</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="n">bias2</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-6'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-6'>#</a>
+                </div>
+                <p>Hidden layer dropout</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">68</span>        <span class="bp">self</span><span class="o">.</span><span class="n">dropout</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Dropout</span><span class="p">(</span><span class="n">dropout</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-7'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-7'>#</a>
+                </div>
+                <p>Activation function $f$</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">70</span>        <span class="bp">self</span><span class="o">.</span><span class="n">activation</span> <span class="o">=</span> <span class="n">activation</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-8'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-8'>#</a>
+                </div>
+                <p>Whether there is a gate</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">72</span>        <span class="bp">self</span><span class="o">.</span><span class="n">is_gated</span> <span class="o">=</span> <span class="n">is_gated</span>
+<span class="lineno">73</span>        <span class="k">if</span> <span class="n">is_gated</span><span class="p">:</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-9'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-9'>#</a>
+                </div>
+                <p>If there is a gate the linear layer to transform inputs to
+be multiplied by the gate, parameterized by weight $V$ and bias $c$</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">76</span>            <span class="bp">self</span><span class="o">.</span><span class="n">linear_v</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">d_ff</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="n">bias_gate</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-10'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-10'>#</a>
+                </div>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">62</span>    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span>
-<span class="lineno">63</span>        <span class="n">g</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">activation</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">layer1</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
-<span class="lineno">64</span>        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_gated</span><span class="p">:</span>
-<span class="lineno">65</span>            <span class="n">x</span> <span class="o">=</span> <span class="n">g</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">linear_v</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
-<span class="lineno">66</span>        <span class="k">else</span><span class="p">:</span>
-<span class="lineno">67</span>            <span class="n">x</span> <span class="o">=</span> <span class="n">g</span>
-<span class="lineno">68</span>        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dropout</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
-<span class="lineno">69</span>        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">layer2</span><span class="p">(</span><span class="n">x</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">78</span>    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-11'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-11'>#</a>
+                </div>
+                <p>$f(x W_1 + b_1)$</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">80</span>        <span class="n">g</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">activation</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">layer1</span><span class="p">(</span><span class="n">x</span><span class="p">))</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-12'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-12'>#</a>
+                </div>
+                <p>If gated, $f(x W_1 + b_1) \otimes (x V + b) $</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">82</span>        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_gated</span><span class="p">:</span>
+<span class="lineno">83</span>            <span class="n">x</span> <span class="o">=</span> <span class="n">g</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">linear_v</span><span class="p">(</span><span class="n">x</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-13'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-13'>#</a>
+                </div>
+                <p>Otherwise</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">85</span>        <span class="k">else</span><span class="p">:</span>
+<span class="lineno">86</span>            <span class="n">x</span> <span class="o">=</span> <span class="n">g</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-14'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-14'>#</a>
+                </div>
+                <p>Apply dropout</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">88</span>        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dropout</span><span class="p">(</span><span class="n">x</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-15'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-15'>#</a>
+                </div>
+                <p>$(f(x W_1 + b_1) \otimes (x V + b)) W_2 + b_2$ or $f(x W_1 + b_1) W_2 + b_2$
+depending on whether it is gated</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">91</span>        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">layer2</span><span class="p">(</span><span class="n">x</span><span class="p">)</span></pre></div>
             </div>
         </div>
     </div>
diff --git a/docs/transformers/glu_variants/experiment.html b/docs/transformers/glu_variants/experiment.html
index 8ab55ca6..e73e4151 100644
--- a/docs/transformers/glu_variants/experiment.html
+++ b/docs/transformers/glu_variants/experiment.html
@@ -71,19 +71,21 @@
                 <div class='section-link'>
                     <a href='#section-0'>#</a>
                 </div>
-                <h1>Train Autoregressive Transformer</h1>
-<p>This trains a simple <a href="../../">transformer</a> model for auto-regression.</p>
+                <h1>Gated Linear Units and Variants</h1>
+<p>This trains a simple <a href="../../">transformer</a> model for auto-regression.
+We try different variants for the <a href="../feed_forward">position-wise feedforward network</a>.
+The reusable &amp; configurable are defined in <a href="configs.html"><code>configs.py</code></a>.</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">14</span><span></span><span class="kn">import</span> <span class="nn">torch</span>
-<span class="lineno">15</span><span class="kn">from</span> <span class="nn">labml</span> <span class="kn">import</span> <span class="n">experiment</span>
-<span class="lineno">16</span><span class="kn">from</span> <span class="nn">labml.configs</span> <span class="kn">import</span> <span class="n">option</span>
-<span class="lineno">17</span><span class="kn">from</span> <span class="nn">labml.utils.pytorch</span> <span class="kn">import</span> <span class="n">get_modules</span>
-<span class="lineno">18</span><span class="kn">from</span> <span class="nn">labml_helpers.module</span> <span class="kn">import</span> <span class="n">Module</span>
-<span class="lineno">19</span>
-<span class="lineno">20</span><span class="kn">from</span> <span class="nn">labml_nn.experiments.nlp_autoregression</span> <span class="kn">import</span> <span class="n">NLPAutoRegressionConfigs</span>
-<span class="lineno">21</span><span class="kn">from</span> <span class="nn">labml_nn.transformers</span> <span class="kn">import</span> <span class="n">Encoder</span><span class="p">,</span> <span class="n">Generator</span><span class="p">,</span> <span class="n">TransformerConfigs</span>
-<span class="lineno">22</span><span class="kn">from</span> <span class="nn">labml_nn.transformers.utils</span> <span class="kn">import</span> <span class="n">subsequent_mask</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">16</span><span></span><span class="kn">import</span> <span class="nn">torch</span>
+<span class="lineno">17</span><span class="kn">from</span> <span class="nn">labml</span> <span class="kn">import</span> <span class="n">experiment</span>
+<span class="lineno">18</span><span class="kn">from</span> <span class="nn">labml.configs</span> <span class="kn">import</span> <span class="n">option</span>
+<span class="lineno">19</span><span class="kn">from</span> <span class="nn">labml.utils.pytorch</span> <span class="kn">import</span> <span class="n">get_modules</span>
+<span class="lineno">20</span><span class="kn">from</span> <span class="nn">labml_helpers.module</span> <span class="kn">import</span> <span class="n">Module</span>
+<span class="lineno">21</span>
+<span class="lineno">22</span><span class="kn">from</span> <span class="nn">labml_nn.experiments.nlp_autoregression</span> <span class="kn">import</span> <span class="n">NLPAutoRegressionConfigs</span>
+<span class="lineno">23</span><span class="kn">from</span> <span class="nn">labml_nn.transformers</span> <span class="kn">import</span> <span class="n">Encoder</span><span class="p">,</span> <span class="n">Generator</span><span class="p">,</span> <span class="n">TransformerConfigs</span>
+<span class="lineno">24</span><span class="kn">from</span> <span class="nn">labml_nn.transformers.utils</span> <span class="kn">import</span> <span class="n">subsequent_mask</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-1'>
@@ -94,7 +96,7 @@
                 <h2>Auto regressive model</h2>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">25</span><span class="k">class</span> <span class="nc">AutoregressiveModel</span><span class="p">(</span><span class="n">Module</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">27</span><span class="k">class</span> <span class="nc">AutoregressiveModel</span><span class="p">(</span><span class="n">Module</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-2'>
@@ -105,8 +107,8 @@
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">30</span>    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">src_embed</span><span class="p">:</span> <span class="n">Module</span><span class="p">,</span> <span class="n">encoder</span><span class="p">:</span> <span class="n">Encoder</span><span class="p">,</span> <span class="n">generator</span><span class="p">:</span> <span class="n">Generator</span><span class="p">):</span>
-<span class="lineno">31</span>        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">32</span>    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">src_embed</span><span class="p">:</span> <span class="n">Module</span><span class="p">,</span> <span class="n">encoder</span><span class="p">:</span> <span class="n">Encoder</span><span class="p">,</span> <span class="n">generator</span><span class="p">:</span> <span class="n">Generator</span><span class="p">):</span>
+<span class="lineno">33</span>        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-3'>
@@ -117,7 +119,7 @@
                 <p>Token embedding module</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">33</span>        <span class="bp">self</span><span class="o">.</span><span class="n">src_embed</span> <span class="o">=</span> <span class="n">src_embed</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">35</span>        <span class="bp">self</span><span class="o">.</span><span class="n">src_embed</span> <span class="o">=</span> <span class="n">src_embed</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-4'>
@@ -128,7 +130,7 @@
                 <p>Transformer based encoder</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">35</span>        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">encoder</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">37</span>        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">encoder</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-5'>
@@ -140,7 +142,7 @@
 this give logits  of the the next token</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">38</span>        <span class="bp">self</span><span class="o">.</span><span class="n">generator</span> <span class="o">=</span> <span class="n">generator</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">40</span>        <span class="bp">self</span><span class="o">.</span><span class="n">generator</span> <span class="o">=</span> <span class="n">generator</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-6'>
@@ -151,7 +153,7 @@ this give logits  of the the next token</p>
                 <p>This will be initialized on the first call</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">40</span>        <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span> <span class="o">=</span> <span class="kc">None</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">42</span>        <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span> <span class="o">=</span> <span class="kc">None</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-7'>
@@ -162,7 +164,7 @@ this give logits  of the the next token</p>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">42</span>    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">src</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">44</span>    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">src</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-8'>
@@ -173,8 +175,8 @@ this give logits  of the the next token</p>
                 <p>Create subsequent mask, so that the transformer can only pay attention to past tokens.</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">44</span>        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">src</span><span class="p">):</span>
-<span class="lineno">45</span>            <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span> <span class="o">=</span> <span class="n">subsequent_mask</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">src</span><span class="p">))</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">src</span><span class="o">.</span><span class="n">device</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">46</span>        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">src</span><span class="p">):</span>
+<span class="lineno">47</span>            <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span> <span class="o">=</span> <span class="n">subsequent_mask</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">src</span><span class="p">))</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">src</span><span class="o">.</span><span class="n">device</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-9'>
@@ -185,7 +187,7 @@ this give logits  of the the next token</p>
                 <p>Embed the tokens (<code>src</code>) and run it through the the transformer</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">47</span>        <span class="n">res</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">src_embed</span><span class="p">(</span><span class="n">src</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">49</span>        <span class="n">res</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">src_embed</span><span class="p">(</span><span class="n">src</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-10'>
@@ -196,7 +198,7 @@ this give logits  of the the next token</p>
                 <p>Generate logits of the next token</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">49</span>        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">generator</span><span class="p">(</span><span class="n">res</span><span class="p">),</span> <span class="kc">None</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">51</span>        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">generator</span><span class="p">(</span><span class="n">res</span><span class="p">),</span> <span class="kc">None</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-11'>
@@ -208,7 +210,7 @@ this give logits  of the the next token</p>
 <p>The default configs can and will be over-ridden when we start the experiment</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">52</span><span class="k">class</span> <span class="nc">Configs</span><span class="p">(</span><span class="n">NLPAutoRegressionConfigs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">54</span><span class="k">class</span> <span class="nc">Configs</span><span class="p">(</span><span class="n">NLPAutoRegressionConfigs</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-12'>
@@ -219,8 +221,8 @@ this give logits  of the the next token</p>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">59</span>    <span class="n">transformer</span><span class="p">:</span> <span class="n">TransformerConfigs</span>
-<span class="lineno">60</span>    <span class="n">model</span><span class="p">:</span> <span class="n">AutoregressiveModel</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">61</span>    <span class="n">transformer</span><span class="p">:</span> <span class="n">TransformerConfigs</span>
+<span class="lineno">62</span>    <span class="n">model</span><span class="p">:</span> <span class="n">AutoregressiveModel</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-13'>
@@ -231,8 +233,8 @@ this give logits  of the the next token</p>
                 <p>Initialize the auto-regressive model</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">63</span><span class="nd">@option</span><span class="p">(</span><span class="n">Configs</span><span class="o">.</span><span class="n">model</span><span class="p">)</span>
-<span class="lineno">64</span><span class="k">def</span> <span class="nf">autoregressive_model</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">Configs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">65</span><span class="nd">@option</span><span class="p">(</span><span class="n">Configs</span><span class="o">.</span><span class="n">model</span><span class="p">)</span>
+<span class="lineno">66</span><span class="k">def</span> <span class="nf">autoregressive_model</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">Configs</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-14'>
@@ -243,8 +245,8 @@ this give logits  of the the next token</p>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">68</span>    <span class="n">m</span> <span class="o">=</span> <span class="n">AutoregressiveModel</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">transformer</span><span class="o">.</span><span class="n">src_embed</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">transformer</span><span class="o">.</span><span class="n">encoder</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">transformer</span><span class="o">.</span><span class="n">generator</span><span class="p">)</span>
-<span class="lineno">69</span>    <span class="k">return</span> <span class="n">m</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">device</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">70</span>    <span class="n">m</span> <span class="o">=</span> <span class="n">AutoregressiveModel</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">transformer</span><span class="o">.</span><span class="n">src_embed</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">transformer</span><span class="o">.</span><span class="n">encoder</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">transformer</span><span class="o">.</span><span class="n">generator</span><span class="p">)</span>
+<span class="lineno">71</span>    <span class="k">return</span> <span class="n">m</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">device</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-15'>
@@ -252,11 +254,11 @@ this give logits  of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-15'>#</a>
                 </div>
-                <p>Initialize the configurable transformer encoder for our autoregressive model</p>
+                <p>Initialize the <a href="../configs.html">configurable transformer</a> encoder for our autoregressive model.</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">72</span><span class="nd">@option</span><span class="p">(</span><span class="n">Configs</span><span class="o">.</span><span class="n">transformer</span><span class="p">)</span>
-<span class="lineno">73</span><span class="k">def</span> <span class="nf">transformer_c</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">Configs</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">74</span><span class="nd">@option</span><span class="p">(</span><span class="n">Configs</span><span class="o">.</span><span class="n">transformer</span><span class="p">)</span>
+<span class="lineno">75</span><span class="k">def</span> <span class="nf">transformer_c</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">Configs</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-16'>
@@ -267,11 +269,11 @@ this give logits  of the the next token</p>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">77</span>    <span class="n">tc</span> <span class="o">=</span> <span class="n">TransformerConfigs</span><span class="p">()</span>
-<span class="lineno">78</span>    <span class="n">tc</span><span class="o">.</span><span class="n">n_src_vocab</span> <span class="o">=</span> <span class="n">c</span><span class="o">.</span><span class="n">n_tokens</span>
-<span class="lineno">79</span>    <span class="n">tc</span><span class="o">.</span><span class="n">n_tgt_vocab</span> <span class="o">=</span> <span class="n">c</span><span class="o">.</span><span class="n">n_tokens</span>
-<span class="lineno">80</span>
-<span class="lineno">81</span>    <span class="k">return</span> <span class="n">tc</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">79</span>    <span class="n">tc</span> <span class="o">=</span> <span class="n">TransformerConfigs</span><span class="p">()</span>
+<span class="lineno">80</span>    <span class="n">tc</span><span class="o">.</span><span class="n">n_src_vocab</span> <span class="o">=</span> <span class="n">c</span><span class="o">.</span><span class="n">n_tokens</span>
+<span class="lineno">81</span>    <span class="n">tc</span><span class="o">.</span><span class="n">n_tgt_vocab</span> <span class="o">=</span> <span class="n">c</span><span class="o">.</span><span class="n">n_tokens</span>
+<span class="lineno">82</span>
+<span class="lineno">83</span>    <span class="k">return</span> <span class="n">tc</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-17'>
@@ -282,7 +284,7 @@ this give logits  of the the next token</p>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">84</span><span class="k">def</span> <span class="nf">main</span><span class="p">():</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">86</span><span class="k">def</span> <span class="nf">main</span><span class="p">():</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-18'>
@@ -293,7 +295,7 @@ this give logits  of the the next token</p>
                 <p>Create experiment</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">86</span>    <span class="n">experiment</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">&quot;glu_variants&quot;</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">88</span>    <span class="n">experiment</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">&quot;glu_variants&quot;</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-19'>
@@ -304,7 +306,7 @@ this give logits  of the the next token</p>
                 <p>Create configs</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">88</span>    <span class="n">conf</span> <span class="o">=</span> <span class="n">Configs</span><span class="p">()</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">90</span>    <span class="n">conf</span> <span class="o">=</span> <span class="n">Configs</span><span class="p">()</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-20'>
@@ -315,7 +317,7 @@ this give logits  of the the next token</p>
                 <p>Load configurations</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">90</span>    <span class="n">experiment</span><span class="o">.</span><span class="n">configs</span><span class="p">(</span><span class="n">conf</span><span class="p">,</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">92</span>    <span class="n">experiment</span><span class="o">.</span><span class="n">configs</span><span class="p">(</span><span class="n">conf</span><span class="p">,</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-21'>
@@ -326,19 +328,19 @@ this give logits  of the the next token</p>
                 <p>A dictionary of configurations to override</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">92</span>                       <span class="p">{</span><span class="s1">&#39;tokenizer&#39;</span><span class="p">:</span> <span class="s1">&#39;character&#39;</span><span class="p">,</span>
-<span class="lineno">93</span>                        <span class="s1">&#39;prompt_separator&#39;</span><span class="p">:</span> <span class="s1">&#39;&#39;</span><span class="p">,</span>
-<span class="lineno">94</span>                        <span class="s1">&#39;prompt&#39;</span><span class="p">:</span> <span class="s1">&#39;It is &#39;</span><span class="p">,</span>
-<span class="lineno">95</span>                        <span class="s1">&#39;text&#39;</span><span class="p">:</span> <span class="s1">&#39;tiny_shakespeare&#39;</span><span class="p">,</span>
-<span class="lineno">96</span>
-<span class="lineno">97</span>                        <span class="s1">&#39;optimizer.optimizer&#39;</span><span class="p">:</span> <span class="s1">&#39;Noam&#39;</span><span class="p">,</span>
-<span class="lineno">98</span>                        <span class="s1">&#39;optimizer.learning_rate&#39;</span><span class="p">:</span> <span class="mf">1.</span><span class="p">,</span>
-<span class="lineno">99</span>                        <span class="s1">&#39;optimizer.d_model&#39;</span><span class="p">:</span> <span class="mi">256</span><span class="p">,</span>
-<span class="lineno">100</span>
-<span class="lineno">101</span>                        <span class="s1">&#39;seq_len&#39;</span><span class="p">:</span> <span class="mi">1024</span><span class="p">,</span>
-<span class="lineno">102</span>                        <span class="s1">&#39;epochs&#39;</span><span class="p">:</span> <span class="mi">128</span><span class="p">,</span>
-<span class="lineno">103</span>                        <span class="s1">&#39;batch_size&#39;</span><span class="p">:</span> <span class="mi">6</span><span class="p">,</span>
-<span class="lineno">104</span>                        <span class="s1">&#39;inner_iterations&#39;</span><span class="p">:</span> <span class="mi">10</span><span class="p">,</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">94</span>                       <span class="p">{</span><span class="s1">&#39;tokenizer&#39;</span><span class="p">:</span> <span class="s1">&#39;character&#39;</span><span class="p">,</span>
+<span class="lineno">95</span>                        <span class="s1">&#39;prompt_separator&#39;</span><span class="p">:</span> <span class="s1">&#39;&#39;</span><span class="p">,</span>
+<span class="lineno">96</span>                        <span class="s1">&#39;prompt&#39;</span><span class="p">:</span> <span class="s1">&#39;It is &#39;</span><span class="p">,</span>
+<span class="lineno">97</span>                        <span class="s1">&#39;text&#39;</span><span class="p">:</span> <span class="s1">&#39;tiny_shakespeare&#39;</span><span class="p">,</span>
+<span class="lineno">98</span>
+<span class="lineno">99</span>                        <span class="s1">&#39;optimizer.optimizer&#39;</span><span class="p">:</span> <span class="s1">&#39;Noam&#39;</span><span class="p">,</span>
+<span class="lineno">100</span>                        <span class="s1">&#39;optimizer.learning_rate&#39;</span><span class="p">:</span> <span class="mf">1.</span><span class="p">,</span>
+<span class="lineno">101</span>                        <span class="s1">&#39;optimizer.d_model&#39;</span><span class="p">:</span> <span class="mi">256</span><span class="p">,</span>
+<span class="lineno">102</span>
+<span class="lineno">103</span>                        <span class="s1">&#39;seq_len&#39;</span><span class="p">:</span> <span class="mi">1024</span><span class="p">,</span>
+<span class="lineno">104</span>                        <span class="s1">&#39;epochs&#39;</span><span class="p">:</span> <span class="mi">128</span><span class="p">,</span>
+<span class="lineno">105</span>                        <span class="s1">&#39;batch_size&#39;</span><span class="p">:</span> <span class="mi">6</span><span class="p">,</span>
+<span class="lineno">106</span>                        <span class="s1">&#39;inner_iterations&#39;</span><span class="p">:</span> <span class="mi">10</span><span class="p">,</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-22'>
@@ -347,9 +349,11 @@ this give logits  of the the next token</p>
                     <a href='#section-22'>#</a>
                 </div>
                 <p>GLU Variant, one of GLU, Bilinear, ReGLU, GEGLU, SwiGLU</p>
+<p>These are defined in the <a href="../configs.html#FFN">configurable FFN</a>
+implementation</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">107</span>                        <span class="s1">&#39;transformer.ffn.glu_variant&#39;</span><span class="p">:</span> <span class="s1">&#39;Bilinear&#39;</span><span class="p">,</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">112</span>                        <span class="s1">&#39;transformer.ffn.glu_variant&#39;</span><span class="p">:</span> <span class="s1">&#39;Bilinear&#39;</span><span class="p">,</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-23'>
@@ -360,10 +364,10 @@ this give logits  of the the next token</p>
                 <p>Transformer configurations</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">110</span>                        <span class="s1">&#39;transformer.d_model&#39;</span><span class="p">:</span> <span class="mi">256</span><span class="p">,</span>
-<span class="lineno">111</span>                        <span class="s1">&#39;transformer.ffn.d_ff&#39;</span><span class="p">:</span> <span class="mi">1024</span><span class="p">,</span>
-<span class="lineno">112</span>                        <span class="s1">&#39;transformer.n_heads&#39;</span><span class="p">:</span> <span class="mi">8</span><span class="p">,</span>
-<span class="lineno">113</span>                        <span class="s1">&#39;transformer.n_layers&#39;</span><span class="p">:</span> <span class="mi">6</span><span class="p">})</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">115</span>                        <span class="s1">&#39;transformer.d_model&#39;</span><span class="p">:</span> <span class="mi">256</span><span class="p">,</span>
+<span class="lineno">116</span>                        <span class="s1">&#39;transformer.ffn.d_ff&#39;</span><span class="p">:</span> <span class="mi">1024</span><span class="p">,</span>
+<span class="lineno">117</span>                        <span class="s1">&#39;transformer.n_heads&#39;</span><span class="p">:</span> <span class="mi">8</span><span class="p">,</span>
+<span class="lineno">118</span>                        <span class="s1">&#39;transformer.n_layers&#39;</span><span class="p">:</span> <span class="mi">6</span><span class="p">})</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-24'>
@@ -374,7 +378,7 @@ this give logits  of the the next token</p>
                 <p>This is needed to initialize models</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">116</span>    <span class="n">conf</span><span class="o">.</span><span class="n">n_tokens</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">n_tokens</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">121</span>    <span class="n">conf</span><span class="o">.</span><span class="n">n_tokens</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">n_tokens</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-25'>
@@ -385,7 +389,7 @@ this give logits  of the the next token</p>
                 <p>Set models for saving and loading</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">119</span>    <span class="n">experiment</span><span class="o">.</span><span class="n">add_pytorch_models</span><span class="p">(</span><span class="n">get_modules</span><span class="p">(</span><span class="n">conf</span><span class="p">))</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">124</span>    <span class="n">experiment</span><span class="o">.</span><span class="n">add_pytorch_models</span><span class="p">(</span><span class="n">get_modules</span><span class="p">(</span><span class="n">conf</span><span class="p">))</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-26'>
@@ -396,7 +400,7 @@ this give logits  of the the next token</p>
                 <p>Start the experiment</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">122</span>    <span class="k">with</span> <span class="n">experiment</span><span class="o">.</span><span class="n">start</span><span class="p">():</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">127</span>    <span class="k">with</span> <span class="n">experiment</span><span class="o">.</span><span class="n">start</span><span class="p">():</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-27'>
@@ -407,11 +411,11 @@ this give logits  of the the next token</p>
                 <p><code>TrainValidConfigs.run</code></p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">124</span>        <span class="n">conf</span><span class="o">.</span><span class="n">run</span><span class="p">()</span>
-<span class="lineno">125</span>
-<span class="lineno">126</span>
-<span class="lineno">127</span><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">&#39;__main__&#39;</span><span class="p">:</span>
-<span class="lineno">128</span>    <span class="n">main</span><span class="p">()</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">129</span>        <span class="n">conf</span><span class="o">.</span><span class="n">run</span><span class="p">()</span>
+<span class="lineno">130</span>
+<span class="lineno">131</span>
+<span class="lineno">132</span><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">&#39;__main__&#39;</span><span class="p">:</span>
+<span class="lineno">133</span>    <span class="n">main</span><span class="p">()</span></pre></div>
             </div>
         </div>
     </div>
diff --git a/docs/transformers/glu_variants/simple.html b/docs/transformers/glu_variants/simple.html
index 46e54fae..207f5c42 100644
--- a/docs/transformers/glu_variants/simple.html
+++ b/docs/transformers/glu_variants/simple.html
@@ -71,25 +71,28 @@
                 <div class='section-link'>
                     <a href='#section-0'>#</a>
                 </div>
-                <h1>Train Autoregressive Transformer</h1>
-<p>This trains a simple <a href="../../">transformer</a> model for auto-regression.</p>
+                <h1>Gated Linear Units and Variants</h1>
+<p>This trains a simple <a href="../../">transformer</a> model for auto-regression.
+We try different variants for the <a href="../feed_forward">position-wise feedforward network</a>.</p>
+<p><em>This is a simpler implementation that doesn&rsquo;t use <a href="experiment.html"><code>labml.configs</code></a> module.
+We decided to write a simpler implementation to make it easier readers who are not familiar.</em></p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">13</span><span></span><span class="kn">import</span> <span class="nn">dataclasses</span>
-<span class="lineno">14</span>
-<span class="lineno">15</span><span class="kn">import</span> <span class="nn">torch</span>
-<span class="lineno">16</span><span class="kn">from</span> <span class="nn">torch</span> <span class="kn">import</span> <span class="n">nn</span>
-<span class="lineno">17</span><span class="kn">from</span> <span class="nn">torch.utils.data</span> <span class="kn">import</span> <span class="n">Dataset</span><span class="p">,</span> <span class="n">DataLoader</span>
+                <div class="highlight"><pre><span class="lineno">17</span><span></span><span class="kn">import</span> <span class="nn">dataclasses</span>
 <span class="lineno">18</span>
-<span class="lineno">19</span><span class="kn">from</span> <span class="nn">labml</span> <span class="kn">import</span> <span class="n">experiment</span><span class="p">,</span> <span class="n">lab</span><span class="p">,</span> <span class="n">tracker</span><span class="p">,</span> <span class="n">monit</span><span class="p">,</span> <span class="n">logger</span>
-<span class="lineno">20</span><span class="kn">from</span> <span class="nn">labml.logger</span> <span class="kn">import</span> <span class="n">Text</span>
-<span class="lineno">21</span><span class="kn">from</span> <span class="nn">labml.utils.download</span> <span class="kn">import</span> <span class="n">download_file</span>
-<span class="lineno">22</span><span class="kn">from</span> <span class="nn">labml_nn.experiments.nlp_autoregression</span> <span class="kn">import</span> <span class="n">transpose_batch</span>
-<span class="lineno">23</span><span class="kn">from</span> <span class="nn">labml_nn.optimizers.noam</span> <span class="kn">import</span> <span class="n">Noam</span>
-<span class="lineno">24</span><span class="kn">from</span> <span class="nn">labml_nn.transformers</span> <span class="kn">import</span> <span class="n">Encoder</span><span class="p">,</span> <span class="n">MultiHeadAttention</span>
-<span class="lineno">25</span><span class="kn">from</span> <span class="nn">labml_nn.transformers.feed_forward</span> <span class="kn">import</span> <span class="n">FeedForward</span>
-<span class="lineno">26</span><span class="kn">from</span> <span class="nn">labml_nn.transformers.models</span> <span class="kn">import</span> <span class="n">EmbeddingsWithPositionalEncoding</span><span class="p">,</span> <span class="n">TransformerLayer</span>
-<span class="lineno">27</span><span class="kn">from</span> <span class="nn">labml_nn.transformers.utils</span> <span class="kn">import</span> <span class="n">subsequent_mask</span></pre></div>
+<span class="lineno">19</span><span class="kn">import</span> <span class="nn">torch</span>
+<span class="lineno">20</span><span class="kn">from</span> <span class="nn">torch</span> <span class="kn">import</span> <span class="n">nn</span>
+<span class="lineno">21</span><span class="kn">from</span> <span class="nn">torch.utils.data</span> <span class="kn">import</span> <span class="n">Dataset</span><span class="p">,</span> <span class="n">DataLoader</span>
+<span class="lineno">22</span>
+<span class="lineno">23</span><span class="kn">from</span> <span class="nn">labml</span> <span class="kn">import</span> <span class="n">experiment</span><span class="p">,</span> <span class="n">lab</span><span class="p">,</span> <span class="n">tracker</span><span class="p">,</span> <span class="n">monit</span><span class="p">,</span> <span class="n">logger</span>
+<span class="lineno">24</span><span class="kn">from</span> <span class="nn">labml.logger</span> <span class="kn">import</span> <span class="n">Text</span>
+<span class="lineno">25</span><span class="kn">from</span> <span class="nn">labml.utils.download</span> <span class="kn">import</span> <span class="n">download_file</span>
+<span class="lineno">26</span><span class="kn">from</span> <span class="nn">labml_nn.experiments.nlp_autoregression</span> <span class="kn">import</span> <span class="n">transpose_batch</span>
+<span class="lineno">27</span><span class="kn">from</span> <span class="nn">labml_nn.optimizers.noam</span> <span class="kn">import</span> <span class="n">Noam</span>
+<span class="lineno">28</span><span class="kn">from</span> <span class="nn">labml_nn.transformers</span> <span class="kn">import</span> <span class="n">Encoder</span><span class="p">,</span> <span class="n">MultiHeadAttention</span>
+<span class="lineno">29</span><span class="kn">from</span> <span class="nn">labml_nn.transformers.feed_forward</span> <span class="kn">import</span> <span class="n">FeedForward</span>
+<span class="lineno">30</span><span class="kn">from</span> <span class="nn">labml_nn.transformers.models</span> <span class="kn">import</span> <span class="n">EmbeddingsWithPositionalEncoding</span><span class="p">,</span> <span class="n">TransformerLayer</span>
+<span class="lineno">31</span><span class="kn">from</span> <span class="nn">labml_nn.transformers.utils</span> <span class="kn">import</span> <span class="n">subsequent_mask</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-1'>
@@ -100,7 +103,7 @@
                 <h2>Auto regressive model</h2>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">30</span><span class="k">class</span> <span class="nc">AutoregressiveModel</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">34</span><span class="k">class</span> <span class="nc">AutoregressiveModel</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-2'>
@@ -111,8 +114,8 @@
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">35</span>    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">src_embed</span><span class="p">:</span> <span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">,</span> <span class="n">encoder</span><span class="p">:</span> <span class="n">Encoder</span><span class="p">,</span> <span class="n">generator</span><span class="p">:</span> <span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
-<span class="lineno">36</span>        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">39</span>    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">src_embed</span><span class="p">:</span> <span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">,</span> <span class="n">encoder</span><span class="p">:</span> <span class="n">Encoder</span><span class="p">,</span> <span class="n">generator</span><span class="p">:</span> <span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="lineno">40</span>        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-3'>
@@ -123,7 +126,7 @@
                 <p>Token embedding module</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">38</span>        <span class="bp">self</span><span class="o">.</span><span class="n">src_embed</span> <span class="o">=</span> <span class="n">src_embed</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">42</span>        <span class="bp">self</span><span class="o">.</span><span class="n">src_embed</span> <span class="o">=</span> <span class="n">src_embed</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-4'>
@@ -134,7 +137,7 @@
                 <p>Transformer based encoder</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">40</span>        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">encoder</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">44</span>        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">encoder</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-5'>
@@ -146,7 +149,7 @@
 this give logits of the the next token</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">43</span>        <span class="bp">self</span><span class="o">.</span><span class="n">generator</span> <span class="o">=</span> <span class="n">generator</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">47</span>        <span class="bp">self</span><span class="o">.</span><span class="n">generator</span> <span class="o">=</span> <span class="n">generator</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-6'>
@@ -157,7 +160,7 @@ this give logits of the the next token</p>
                 <p>This will be initialized on the first call</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">45</span>        <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span> <span class="o">=</span> <span class="kc">None</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">49</span>        <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span> <span class="o">=</span> <span class="kc">None</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-7'>
@@ -168,7 +171,7 @@ this give logits of the the next token</p>
                 
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">47</span>    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">src</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">51</span>    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">src</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-8'>
@@ -179,8 +182,8 @@ this give logits of the the next token</p>
                 <p>Create subsequent mask, so that the transformer can only pay attention to past tokens.</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">49</span>        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">src</span><span class="p">):</span>
-<span class="lineno">50</span>            <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span> <span class="o">=</span> <span class="n">subsequent_mask</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">src</span><span class="p">))</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">src</span><span class="o">.</span><span class="n">device</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">53</span>        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">src</span><span class="p">):</span>
+<span class="lineno">54</span>            <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span> <span class="o">=</span> <span class="n">subsequent_mask</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">src</span><span class="p">))</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">src</span><span class="o">.</span><span class="n">device</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-9'>
@@ -191,7 +194,7 @@ this give logits of the the next token</p>
                 <p>Embed the tokens (<code>src</code>) and run it through the the transformer</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">52</span>        <span class="n">res</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">src_embed</span><span class="p">(</span><span class="n">src</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">56</span>        <span class="n">res</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">src_embed</span><span class="p">(</span><span class="n">src</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">src_mask</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-10'>
@@ -202,98 +205,19 @@ this give logits of the the next token</p>
                 <p>Generate logits of the next token</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">54</span>        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">generator</span><span class="p">(</span><span class="n">res</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">58</span>        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">generator</span><span class="p">(</span><span class="n">res</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-11'>
-            <div class='docs'>
+        <div class='docs doc-strings'>
                 <div class='section-link'>
                     <a href='#section-11'>#</a>
                 </div>
-                
+                <h3>Configurations</h3>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">57</span><span class="nd">@dataclasses</span><span class="o">.</span><span class="n">dataclass</span>
-<span class="lineno">58</span><span class="k">class</span> <span class="nc">Configs</span><span class="p">:</span>
-<span class="lineno">59</span>    <span class="n">d_model</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">512</span>
-<span class="lineno">60</span>    <span class="n">seq_len</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">128</span>
-<span class="lineno">61</span>    <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span>
-<span class="lineno">62</span>    <span class="n">n_layers</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">6</span>
-<span class="lineno">63</span>    <span class="n">n_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span>
-<span class="lineno">64</span>    <span class="n">dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span>
-<span class="lineno">65</span>    <span class="n">d_ff</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2048</span>
-<span class="lineno">66</span>    <span class="n">glu_variant</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;GLU&#39;</span>
-<span class="lineno">67</span>    <span class="n">epochs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span>
-<span class="lineno">68</span>    <span class="n">grad_norm_clip</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.5</span>
-<span class="lineno">69</span>
-<span class="lineno">70</span>
-<span class="lineno">71</span><span class="k">class</span> <span class="nc">TinyShakespeareDataset</span><span class="p">(</span><span class="n">Dataset</span><span class="p">):</span>
-<span class="lineno">72</span>    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">seq_len</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
-<span class="lineno">73</span>        <span class="n">path</span> <span class="o">=</span> <span class="n">lab</span><span class="o">.</span><span class="n">get_data_path</span><span class="p">()</span> <span class="o">/</span> <span class="s1">&#39;tiny_shakespeare.txt&#39;</span>
-<span class="lineno">74</span>        <span class="n">download_file</span><span class="p">(</span><span class="s1">&#39;https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt&#39;</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span>
-<span class="lineno">75</span>        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">path</span><span class="p">),</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
-<span class="lineno">76</span>            <span class="n">text</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
-<span class="lineno">77</span>
-<span class="lineno">78</span>        <span class="n">chars</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">text</span><span class="p">))</span>
-<span class="lineno">79</span>        <span class="bp">self</span><span class="o">.</span><span class="n">stoi</span> <span class="o">=</span> <span class="p">{</span><span class="n">c</span><span class="p">:</span> <span class="n">i</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">c</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">chars</span><span class="p">)}</span>
-<span class="lineno">80</span>        <span class="bp">self</span><span class="o">.</span><span class="n">itos</span> <span class="o">=</span> <span class="p">{</span><span class="n">i</span><span class="p">:</span> <span class="n">c</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">c</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">chars</span><span class="p">)}</span>
-<span class="lineno">81</span>        <span class="bp">self</span><span class="o">.</span><span class="n">seq_len</span> <span class="o">=</span> <span class="n">seq_len</span>
-<span class="lineno">82</span>        <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">text_to_i</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
-<span class="lineno">83</span>
-<span class="lineno">84</span>    <span class="k">def</span> <span class="nf">text_to_i</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
-<span class="lineno">85</span>        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">stoi</span><span class="p">[</span><span class="n">c</span><span class="p">]</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">text</span><span class="p">],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">long</span><span class="p">)</span>
-<span class="lineno">86</span>
-<span class="lineno">87</span>    <span class="k">def</span> <span class="fm">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-<span class="lineno">88</span>        <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">)</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">seq_len</span> <span class="o">-</span> <span class="mi">1</span>
-<span class="lineno">89</span>
-<span class="lineno">90</span>    <span class="k">def</span> <span class="fm">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">idx</span><span class="p">):</span>
-<span class="lineno">91</span>        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">idx</span><span class="p">:</span><span class="n">idx</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">seq_len</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">idx</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:</span><span class="n">idx</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">seq_len</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span>
-<span class="lineno">92</span>
-<span class="lineno">93</span>
-<span class="lineno">94</span><span class="k">class</span> <span class="nc">Trainer</span><span class="p">:</span>
-<span class="lineno">95</span>    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">configs</span><span class="p">:</span> <span class="n">Configs</span><span class="p">):</span>
-<span class="lineno">96</span>        <span class="bp">self</span><span class="o">.</span><span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s1">&#39;cpu&#39;</span><span class="p">)</span>
-<span class="lineno">97</span>        <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
-<span class="lineno">98</span>            <span class="bp">self</span><span class="o">.</span><span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s1">&#39;cuda:0&#39;</span><span class="p">)</span>
-<span class="lineno">99</span>        <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span> <span class="o">=</span> <span class="n">TinyShakespeareDataset</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">seq_len</span><span class="p">)</span>
-<span class="lineno">100</span>        <span class="bp">self</span><span class="o">.</span><span class="n">dataloader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">configs</span><span class="o">.</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">collate_fn</span><span class="o">=</span><span class="n">transpose_batch</span><span class="p">,</span>
-<span class="lineno">101</span>                                     <span class="n">shuffle</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
-<span class="lineno">102</span>
-<span class="lineno">103</span>        <span class="k">if</span> <span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span> <span class="o">==</span> <span class="s1">&#39;GLU&#39;</span><span class="p">:</span>
-<span class="lineno">104</span>            <span class="n">ffn</span> <span class="o">=</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sigmoid</span><span class="p">(),</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
-<span class="lineno">105</span>        <span class="k">elif</span> <span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span> <span class="o">==</span> <span class="s1">&#39;Bilinear&#39;</span><span class="p">:</span>
-<span class="lineno">106</span>            <span class="n">ffn</span> <span class="o">=</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">(),</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
-<span class="lineno">107</span>        <span class="k">elif</span> <span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span> <span class="o">==</span> <span class="s1">&#39;ReGLU&#39;</span><span class="p">:</span>
-<span class="lineno">108</span>            <span class="n">ffn</span> <span class="o">=</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">(),</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
-<span class="lineno">109</span>        <span class="k">elif</span> <span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span> <span class="o">==</span> <span class="s1">&#39;GEGLU&#39;</span><span class="p">:</span>
-<span class="lineno">110</span>            <span class="n">ffn</span> <span class="o">=</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">GELU</span><span class="p">(),</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
-<span class="lineno">111</span>        <span class="k">elif</span> <span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span> <span class="o">==</span> <span class="s1">&#39;SwiGLU&#39;</span><span class="p">:</span>
-<span class="lineno">112</span>            <span class="n">ffn</span> <span class="o">=</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">SiLU</span><span class="p">(),</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
-<span class="lineno">113</span>        <span class="k">elif</span> <span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span> <span class="o">==</span> <span class="s1">&#39;ReLU&#39;</span><span class="p">:</span>
-<span class="lineno">114</span>            <span class="n">ffn</span> <span class="o">=</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">())</span>
-<span class="lineno">115</span>        <span class="k">elif</span> <span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span> <span class="o">==</span> <span class="s1">&#39;GELU&#39;</span><span class="p">:</span>
-<span class="lineno">116</span>            <span class="n">ffn</span> <span class="o">=</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">GELU</span><span class="p">())</span>
-<span class="lineno">117</span>        <span class="k">else</span><span class="p">:</span>
-<span class="lineno">118</span>            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Unknown variant </span><span class="si">{</span><span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
-<span class="lineno">119</span>
-<span class="lineno">120</span>        <span class="n">n_chars</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">stoi</span><span class="p">)</span>
-<span class="lineno">121</span>        <span class="bp">self</span><span class="o">.</span><span class="n">model</span> <span class="o">=</span> <span class="n">AutoregressiveModel</span><span class="p">(</span><span class="n">EmbeddingsWithPositionalEncoding</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">n_chars</span><span class="p">),</span>
-<span class="lineno">122</span>                                         <span class="n">Encoder</span><span class="p">(</span><span class="n">TransformerLayer</span><span class="p">(</span>
-<span class="lineno">123</span>                                             <span class="n">d_model</span><span class="o">=</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span>
-<span class="lineno">124</span>                                             <span class="n">self_attn</span><span class="o">=</span><span class="n">MultiHeadAttention</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">n_heads</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span>
-<span class="lineno">125</span>                                                                          <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">),</span>
-<span class="lineno">126</span>                                             <span class="n">src_attn</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-<span class="lineno">127</span>                                             <span class="n">feed_forward</span><span class="o">=</span><span class="n">ffn</span><span class="p">,</span>
-<span class="lineno">128</span>                                             <span class="n">dropout_prob</span><span class="o">=</span><span class="n">configs</span><span class="o">.</span><span class="n">dropout</span>
-<span class="lineno">129</span>                                         <span class="p">),</span> <span class="n">configs</span><span class="o">.</span><span class="n">n_layers</span><span class="p">),</span>
-<span class="lineno">130</span>                                         <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">n_chars</span><span class="p">))</span>
-<span class="lineno">131</span>        <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
-<span class="lineno">132</span>
-<span class="lineno">133</span>        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">Noam</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">warmup</span><span class="o">=</span><span class="mi">2_000</span><span class="p">,</span> <span class="n">d_model</span><span class="o">=</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
-<span class="lineno">134</span>
-<span class="lineno">135</span>        <span class="bp">self</span><span class="o">.</span><span class="n">loss_func</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">CrossEntropyLoss</span><span class="p">()</span>
-<span class="lineno">136</span>        <span class="bp">self</span><span class="o">.</span><span class="n">epochs</span> <span class="o">=</span> <span class="n">configs</span><span class="o">.</span><span class="n">epochs</span>
-<span class="lineno">137</span>        <span class="bp">self</span><span class="o">.</span><span class="n">grad_norm_clip</span> <span class="o">=</span> <span class="n">configs</span><span class="o">.</span><span class="n">grad_norm_clip</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">61</span><span class="nd">@dataclasses</span><span class="o">.</span><span class="n">dataclass</span>
+<span class="lineno">62</span><span class="k">class</span> <span class="nc">Configs</span><span class="p">:</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-12'>
@@ -301,10 +225,19 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-12'>#</a>
                 </div>
-                <p>Set tracker configurations</p>
+                
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">140</span>        <span class="n">tracker</span><span class="o">.</span><span class="n">set_scalar</span><span class="p">(</span><span class="s2">&quot;loss.*&quot;</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">66</span>    <span class="n">d_model</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">512</span>
+<span class="lineno">67</span>    <span class="n">seq_len</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">128</span>
+<span class="lineno">68</span>    <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span>
+<span class="lineno">69</span>    <span class="n">n_layers</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">6</span>
+<span class="lineno">70</span>    <span class="n">n_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span>
+<span class="lineno">71</span>    <span class="n">dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span>
+<span class="lineno">72</span>    <span class="n">d_ff</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2048</span>
+<span class="lineno">73</span>    <span class="n">glu_variant</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;GLU&#39;</span>
+<span class="lineno">74</span>    <span class="n">epochs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span>
+<span class="lineno">75</span>    <span class="n">grad_norm_clip</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.5</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-13'>
@@ -312,10 +245,10 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-13'>#</a>
                 </div>
-                <h3>Sampling function to generate samples periodically while training</h3>
+                <h3>Tiny Shakespeare Dataset</h3>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">142</span>    <span class="k">def</span> <span class="nf">sample</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">78</span><span class="k">class</span> <span class="nc">TinyShakespeareDataset</span><span class="p">(</span><span class="n">Dataset</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-14'>
@@ -323,10 +256,10 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-14'>#</a>
                 </div>
-                <p>Starting prompt</p>
+                
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">148</span>        <span class="n">prompt</span> <span class="o">=</span> <span class="s1">&#39;It is&#39;</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">83</span>    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">seq_len</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-15'>
@@ -334,10 +267,10 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-15'>#</a>
                 </div>
-                <p>Collect output for printing</p>
+                <p>Location of the text file</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">150</span>        <span class="n">log</span> <span class="o">=</span> <span class="p">[(</span><span class="n">prompt</span><span class="p">,</span> <span class="n">Text</span><span class="o">.</span><span class="n">subtle</span><span class="p">)]</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">85</span>        <span class="n">path</span> <span class="o">=</span> <span class="n">lab</span><span class="o">.</span><span class="n">get_data_path</span><span class="p">()</span> <span class="o">/</span> <span class="s1">&#39;tiny_shakespeare.txt&#39;</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-16'>
@@ -345,10 +278,10 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-16'>#</a>
                 </div>
-                <p>Sample 25 tokens</p>
+                <p>Download the file</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">152</span>        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">monit</span><span class="o">.</span><span class="n">iterate</span><span class="p">(</span><span class="s1">&#39;Sample&#39;</span><span class="p">,</span> <span class="mi">25</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">87</span>        <span class="n">download_file</span><span class="p">(</span><span class="s1">&#39;https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt&#39;</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-17'>
@@ -356,11 +289,11 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-17'>#</a>
                 </div>
-                <p>Tokenize the prompt</p>
+                <p>Read the downloaded file</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">154</span>            <span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">text_to_i</span><span class="p">(</span><span class="n">prompt</span><span class="p">)</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-<span class="lineno">155</span>            <span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">89</span>        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">path</span><span class="p">),</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+<span class="lineno">90</span>            <span class="n">text</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-18'>
@@ -368,10 +301,10 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-18'>#</a>
                 </div>
-                <p>Get the model output</p>
+                <p>Extract the characters</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">157</span>            <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">(</span><span class="n">data</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">93</span>        <span class="n">chars</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">text</span><span class="p">))</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-19'>
@@ -379,10 +312,10 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-19'>#</a>
                 </div>
-                <p>Get the model prediction (greedy)</p>
+                <p>Character to id (integer) map</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">159</span>            <span class="n">output</span> <span class="o">=</span> <span class="n">output</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">dim</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">95</span>        <span class="bp">self</span><span class="o">.</span><span class="n">stoi</span> <span class="o">=</span> <span class="p">{</span><span class="n">c</span><span class="p">:</span> <span class="n">i</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">c</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">chars</span><span class="p">)}</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-20'>
@@ -390,10 +323,10 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-20'>#</a>
                 </div>
-                <p>Add the prediction to prompt</p>
+                <p>Id to character map</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">161</span>            <span class="n">prompt</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">itos</span><span class="p">[</span><span class="n">output</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">item</span><span class="p">()]</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">97</span>        <span class="bp">self</span><span class="o">.</span><span class="n">itos</span> <span class="o">=</span> <span class="p">{</span><span class="n">i</span><span class="p">:</span> <span class="n">c</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">c</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">chars</span><span class="p">)}</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-21'>
@@ -401,10 +334,10 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-21'>#</a>
                 </div>
-                <p>Add the prediction for logging</p>
+                <p>Length of a training sample</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">163</span>            <span class="n">log</span> <span class="o">+=</span> <span class="p">[(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">itos</span><span class="p">[</span><span class="n">output</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">item</span><span class="p">()],</span> <span class="n">Text</span><span class="o">.</span><span class="n">value</span><span class="p">)]</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">99</span>        <span class="bp">self</span><span class="o">.</span><span class="n">seq_len</span> <span class="o">=</span> <span class="n">seq_len</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-22'>
@@ -412,23 +345,21 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-22'>#</a>
                 </div>
-                <p>Print the sampled output</p>
+                <p>Data in the form of a tensor of ids</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">166</span>        <span class="n">logger</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="n">log</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">101</span>        <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">text_to_i</span><span class="p">(</span><span class="n">text</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-23'>
-            <div class='docs'>
+        <div class='docs doc-strings'>
                 <div class='section-link'>
                     <a href='#section-23'>#</a>
                 </div>
-                
+                <p>Transform the text into a tensor of ids</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">168</span>    <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-<span class="lineno">169</span>        <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">monit</span><span class="o">.</span><span class="n">loop</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">epochs</span><span class="p">):</span>
-<span class="lineno">170</span>            <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">batch</span> <span class="ow">in</span> <span class="n">monit</span><span class="o">.</span><span class="n">enum</span><span class="p">(</span><span class="s1">&#39;Train&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataloader</span><span class="p">):</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">103</span>    <span class="k">def</span> <span class="nf">text_to_i</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-24'>
@@ -436,27 +367,22 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-24'>#</a>
                 </div>
-                <p>Move data to the device</p>
+                
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">172</span>                <span class="n">data</span><span class="p">,</span> <span class="n">target</span> <span class="o">=</span> <span class="n">batch</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">),</span> <span class="n">batch</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
-<span class="lineno">173</span>
-<span class="lineno">174</span>                <span class="n">tracker</span><span class="o">.</span><span class="n">add_global_step</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">*</span> <span class="n">data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
-<span class="lineno">175</span>
-<span class="lineno">176</span>                <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
-<span class="lineno">177</span>                <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">(</span><span class="n">data</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">107</span>        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">stoi</span><span class="p">[</span><span class="n">c</span><span class="p">]</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">text</span><span class="p">],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">long</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-25'>
-            <div class='docs'>
+        <div class='docs doc-strings'>
                 <div class='section-link'>
                     <a href='#section-25'>#</a>
                 </div>
-                <p>Calculate and log loss</p>
+                <p>Number of samples in the dataset.</p>
+<p><em>This will read the dataset <code>seq_len</code> times in a single epoch.</em></p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">180</span>                <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">loss_func</span><span class="p">(</span><span class="n">output</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">output</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]),</span> <span class="n">target</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">))</span>
-<span class="lineno">181</span>                <span class="n">tracker</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="s2">&quot;loss.train&quot;</span><span class="p">,</span> <span class="n">loss</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">109</span>    <span class="k">def</span> <span class="fm">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-26'>
@@ -464,21 +390,21 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-26'>#</a>
                 </div>
-                <p>Calculate gradients</p>
+                
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">184</span>                <span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">115</span>        <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">)</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">seq_len</span> <span class="o">-</span> <span class="mi">1</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-27'>
-            <div class='docs'>
+        <div class='docs doc-strings'>
                 <div class='section-link'>
                     <a href='#section-27'>#</a>
                 </div>
-                <p>Clip gradients</p>
+                <p>Return a sample</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">186</span>                <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">clip_grad_norm_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">max_norm</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">grad_norm_clip</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">117</span>    <span class="k">def</span> <span class="fm">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">idx</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-28'>
@@ -486,22 +412,21 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-28'>#</a>
                 </div>
-                <p>Take optimizer step</p>
+                
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">188</span>                <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">121</span>        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">idx</span><span class="p">:</span><span class="n">idx</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">seq_len</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">idx</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:</span><span class="n">idx</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">seq_len</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-29'>
-            <div class='docs'>
+        <div class='docs doc-strings'>
                 <div class='section-link'>
                     <a href='#section-29'>#</a>
                 </div>
-                <p>Log the model parameters and gradients on last batch of every epoch</p>
+                <h2>Trainer</h2>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">190</span>                <span class="k">if</span> <span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">%</span> <span class="mi">100</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
-<span class="lineno">191</span>                    <span class="n">tracker</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="s1">&#39;model&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">124</span><span class="k">class</span> <span class="nc">Trainer</span><span class="p">:</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-30'>
@@ -509,15 +434,10 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-30'>#</a>
                 </div>
-                <p>Clear the gradients</p>
+                
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">193</span>                <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
-<span class="lineno">194</span>
-<span class="lineno">195</span>                <span class="k">if</span> <span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">%</span> <span class="mi">100</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
-<span class="lineno">196</span>                    <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
-<span class="lineno">197</span>                    <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
-<span class="lineno">198</span>                        <span class="bp">self</span><span class="o">.</span><span class="n">sample</span><span class="p">()</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">129</span>    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">configs</span><span class="p">:</span> <span class="n">Configs</span><span class="p">):</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-31'>
@@ -525,13 +445,12 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-31'>#</a>
                 </div>
-                <p>Save the tracked metrics</p>
+                <p>Get the device</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">201</span>                <span class="k">if</span> <span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">%</span> <span class="mi">10</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
-<span class="lineno">202</span>                    <span class="n">tracker</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
-<span class="lineno">203</span>
-<span class="lineno">204</span>            <span class="n">experiment</span><span class="o">.</span><span class="n">save_checkpoint</span><span class="p">()</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">131</span>        <span class="bp">self</span><span class="o">.</span><span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s1">&#39;cpu&#39;</span><span class="p">)</span>
+<span class="lineno">132</span>        <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
+<span class="lineno">133</span>            <span class="bp">self</span><span class="o">.</span><span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s1">&#39;cuda:0&#39;</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-32'>
@@ -539,10 +458,10 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-32'>#</a>
                 </div>
-                
+                <p>Initialize the dataset</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">207</span><span class="k">def</span> <span class="nf">main</span><span class="p">():</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">135</span>        <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span> <span class="o">=</span> <span class="n">TinyShakespeareDataset</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">seq_len</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-33'>
@@ -550,10 +469,13 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-33'>#</a>
                 </div>
-                <p>Create experiment</p>
+                <p>Initialize the dataloader</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">209</span>    <span class="n">experiment</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">&quot;glu_variants&quot;</span><span class="p">)</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">137</span>        <span class="bp">self</span><span class="o">.</span><span class="n">dataloader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="p">,</span>
+<span class="lineno">138</span>                                     <span class="n">batch_size</span><span class="o">=</span><span class="n">configs</span><span class="o">.</span><span class="n">batch_size</span><span class="p">,</span>
+<span class="lineno">139</span>                                     <span class="n">collate_fn</span><span class="o">=</span><span class="n">transpose_batch</span><span class="p">,</span>
+<span class="lineno">140</span>                                     <span class="n">shuffle</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-34'>
@@ -561,10 +483,13 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-34'>#</a>
                 </div>
-                <p>Create configs</p>
+                <p>FFN with Gated Linear Unit
+<script type="math/tex; mode=display">FFN_{GLU}(x)(x, W_1, V, W_2) = (\sigma(x W_1) \otimes x V) W_2</script>
+</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">211</span>    <span class="n">configs</span> <span class="o">=</span> <span class="n">Configs</span><span class="p">()</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">144</span>        <span class="k">if</span> <span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span> <span class="o">==</span> <span class="s1">&#39;GLU&#39;</span><span class="p">:</span>
+<span class="lineno">145</span>            <span class="n">ffn</span> <span class="o">=</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sigmoid</span><span class="p">(),</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-35'>
@@ -572,13 +497,13 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-35'>#</a>
                 </div>
-                <p>Load configurations</p>
+                <p>FFN with Bilinear hidden layer
+<script type="math/tex; mode=display">FFN_{Bilinear}(x)(x, W_1, V, W_2) = (x W_1 \otimes x V) W_2</script>
+</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">213</span>    <span class="n">experiment</span><span class="o">.</span><span class="n">configs</span><span class="p">(</span><span class="n">dataclasses</span><span class="o">.</span><span class="n">asdict</span><span class="p">(</span><span class="n">configs</span><span class="p">))</span>
-<span class="lineno">214</span>
-<span class="lineno">215</span>    <span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">configs</span><span class="p">)</span>
-<span class="lineno">216</span>    <span class="n">experiment</span><span class="o">.</span><span class="n">add_pytorch_models</span><span class="p">({</span><span class="s1">&#39;model&#39;</span><span class="p">:</span> <span class="n">trainer</span><span class="o">.</span><span class="n">model</span><span class="p">})</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">148</span>        <span class="k">elif</span> <span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span> <span class="o">==</span> <span class="s1">&#39;Bilinear&#39;</span><span class="p">:</span>
+<span class="lineno">149</span>            <span class="n">ffn</span> <span class="o">=</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">(),</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-36'>
@@ -586,10 +511,13 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-36'>#</a>
                 </div>
-                <p>Start the experiment</p>
+                <p>FFN with ReLU gate
+<script type="math/tex; mode=display">FFN_{ReGLU}(x)(x, W_1, V, W_2) = (\max(0, x W_1) \otimes x V) W_2</script>
+</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">219</span>    <span class="k">with</span> <span class="n">experiment</span><span class="o">.</span><span class="n">start</span><span class="p">():</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">152</span>        <span class="k">elif</span> <span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span> <span class="o">==</span> <span class="s1">&#39;ReGLU&#39;</span><span class="p">:</span>
+<span class="lineno">153</span>            <span class="n">ffn</span> <span class="o">=</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">(),</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span></pre></div>
             </div>
         </div>
     <div class='section' id='section-37'>
@@ -597,14 +525,570 @@ this give logits of the the next token</p>
                 <div class='section-link'>
                     <a href='#section-37'>#</a>
                 </div>
-                <p><code>TrainValidConfigs.run</code></p>
+                <p>FFN with GELU gate
+<script type="math/tex; mode=display">FFN_{GEGLU}(x)(x, W_1, V, W_2) = (\text{GELU}(x W_1) \otimes x V) W_2</script>
+</p>
             </div>
             <div class='code'>
-                <div class="highlight"><pre><span class="lineno">221</span>        <span class="n">trainer</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
-<span class="lineno">222</span>
-<span class="lineno">223</span>
-<span class="lineno">224</span><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">&#39;__main__&#39;</span><span class="p">:</span>
-<span class="lineno">225</span>    <span class="n">main</span><span class="p">()</span></pre></div>
+                <div class="highlight"><pre><span class="lineno">156</span>        <span class="k">elif</span> <span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span> <span class="o">==</span> <span class="s1">&#39;GEGLU&#39;</span><span class="p">:</span>
+<span class="lineno">157</span>            <span class="n">ffn</span> <span class="o">=</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">GELU</span><span class="p">(),</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-38'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-38'>#</a>
+                </div>
+                <p>FFN with Swish gate
+<script type="math/tex; mode=display">FFN_{SwiGLU}(x)(x, W_1, V, W_2) = (\text{Swish}_1(x W_1) \otimes x V) W_2</script>
+where $\text{Swish}_\beta(x) = x \sigma(\beta x)$</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">161</span>        <span class="k">elif</span> <span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span> <span class="o">==</span> <span class="s1">&#39;SwiGLU&#39;</span><span class="p">:</span>
+<span class="lineno">162</span>            <span class="n">ffn</span> <span class="o">=</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">SiLU</span><span class="p">(),</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-39'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-39'>#</a>
+                </div>
+                <p>FFN with ReLU activation
+<script type="math/tex; mode=display">FFN_{ReLU}(x)(x, W_1, W_2, b_1, b_2) = \text{ReLU}_1(x W_1 + b_1) W_2 + b_2</script>
+</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">165</span>        <span class="k">elif</span> <span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span> <span class="o">==</span> <span class="s1">&#39;ReLU&#39;</span><span class="p">:</span>
+<span class="lineno">166</span>            <span class="n">ffn</span> <span class="o">=</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">())</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-40'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-40'>#</a>
+                </div>
+                <p>FFN with ReLU activation
+<script type="math/tex; mode=display">FFN_{GELU}(x)(x, W_1, W_2, b_1, b_2) = \text{GELU}_1(x W_1 + b_1) W_2 + b_2</script>
+</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">169</span>        <span class="k">elif</span> <span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span> <span class="o">==</span> <span class="s1">&#39;GELU&#39;</span><span class="p">:</span>
+<span class="lineno">170</span>            <span class="n">ffn</span> <span class="o">=</span> <span class="n">FeedForward</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">GELU</span><span class="p">())</span>
+<span class="lineno">171</span>        <span class="k">else</span><span class="p">:</span>
+<span class="lineno">172</span>            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Unknown variant </span><span class="si">{</span><span class="n">configs</span><span class="o">.</span><span class="n">glu_variant</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-41'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-41'>#</a>
+                </div>
+                <p>Number of different characters</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">175</span>        <span class="n">n_chars</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">stoi</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-42'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-42'>#</a>
+                </div>
+                <p>Initialize <a href="../mha.html">Multi-Head Attention module</a></p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">178</span>        <span class="n">mha</span> <span class="o">=</span> <span class="n">MultiHeadAttention</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">n_heads</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-43'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-43'>#</a>
+                </div>
+                <p>Initialize the <a href="../models.html#TransformerLayer">Transformer Block</a></p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">180</span>        <span class="n">transformer_layer</span> <span class="o">=</span> <span class="n">TransformerLayer</span><span class="p">(</span><span class="n">d_model</span><span class="o">=</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">self_attn</span><span class="o">=</span><span class="n">mha</span><span class="p">,</span> <span class="n">src_attn</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+<span class="lineno">181</span>                                             <span class="n">feed_forward</span><span class="o">=</span><span class="n">ffn</span><span class="p">,</span> <span class="n">dropout_prob</span><span class="o">=</span><span class="n">configs</span><span class="o">.</span><span class="n">dropout</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-44'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-44'>#</a>
+                </div>
+                <p>Initialize the model with an
+<a href="../models.html#EmbeddingsWithPositionalEncoding">embedding layer</a>
+(with fixed positional encoding)
+<a href="../models.html#Encoder">transformer encoder</a> and
+a linear layer to generate logits.</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">187</span>        <span class="bp">self</span><span class="o">.</span><span class="n">model</span> <span class="o">=</span> <span class="n">AutoregressiveModel</span><span class="p">(</span><span class="n">EmbeddingsWithPositionalEncoding</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">n_chars</span><span class="p">),</span>
+<span class="lineno">188</span>                                         <span class="n">Encoder</span><span class="p">(</span><span class="n">transformer_layer</span><span class="p">,</span> <span class="n">configs</span><span class="o">.</span><span class="n">n_layers</span><span class="p">),</span>
+<span class="lineno">189</span>                                         <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">n_chars</span><span class="p">))</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-45'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-45'>#</a>
+                </div>
+                <p>Move the model to the current device</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">192</span>        <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-46'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-46'>#</a>
+                </div>
+                <p>Initialize <a href="../../optimizers/noam.html">Noam optimizer</a></p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">195</span>        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">Noam</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">warmup</span><span class="o">=</span><span class="mi">2_000</span><span class="p">,</span> <span class="n">d_model</span><span class="o">=</span><span class="n">configs</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-47'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-47'>#</a>
+                </div>
+                <p>Cross-entropy loss</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">198</span>        <span class="bp">self</span><span class="o">.</span><span class="n">loss_func</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">CrossEntropyLoss</span><span class="p">()</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-48'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-48'>#</a>
+                </div>
+                <p>Number of training epochs;
+*note that our dataset definition repeats the data <code>seq_len</code> times in a single epoch</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">201</span>        <span class="bp">self</span><span class="o">.</span><span class="n">epochs</span> <span class="o">=</span> <span class="n">configs</span><span class="o">.</span><span class="n">epochs</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-49'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-49'>#</a>
+                </div>
+                <p>Gradient clipping norm</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">203</span>        <span class="bp">self</span><span class="o">.</span><span class="n">grad_norm_clip</span> <span class="o">=</span> <span class="n">configs</span><span class="o">.</span><span class="n">grad_norm_clip</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-50'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-50'>#</a>
+                </div>
+                <p>Set tracker configurations</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">206</span>        <span class="n">tracker</span><span class="o">.</span><span class="n">set_scalar</span><span class="p">(</span><span class="s2">&quot;loss.*&quot;</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-51'>
+        <div class='docs doc-strings'>
+                <div class='section-link'>
+                    <a href='#section-51'>#</a>
+                </div>
+                <h3>Sampling function to generate samples periodically while training</h3>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">208</span>    <span class="k">def</span> <span class="nf">sample</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-52'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-52'>#</a>
+                </div>
+                <p>Starting prompt</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">214</span>        <span class="n">prompt</span> <span class="o">=</span> <span class="s1">&#39;It is&#39;</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-53'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-53'>#</a>
+                </div>
+                <p>Collect output for printing</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">216</span>        <span class="n">log</span> <span class="o">=</span> <span class="p">[(</span><span class="n">prompt</span><span class="p">,</span> <span class="n">Text</span><span class="o">.</span><span class="n">subtle</span><span class="p">)]</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-54'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-54'>#</a>
+                </div>
+                <p>Sample 25 tokens</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">218</span>        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">monit</span><span class="o">.</span><span class="n">iterate</span><span class="p">(</span><span class="s1">&#39;Sample&#39;</span><span class="p">,</span> <span class="mi">25</span><span class="p">):</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-55'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-55'>#</a>
+                </div>
+                <p>Tokenize the prompt</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">220</span>            <span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">text_to_i</span><span class="p">(</span><span class="n">prompt</span><span class="p">)</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
+<span class="lineno">221</span>            <span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-56'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-56'>#</a>
+                </div>
+                <p>Get the model output</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">223</span>            <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">(</span><span class="n">data</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-57'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-57'>#</a>
+                </div>
+                <p>Get the model prediction (greedy)</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">225</span>            <span class="n">output</span> <span class="o">=</span> <span class="n">output</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">dim</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-58'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-58'>#</a>
+                </div>
+                <p>Add the prediction to prompt</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">227</span>            <span class="n">prompt</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">itos</span><span class="p">[</span><span class="n">output</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">item</span><span class="p">()]</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-59'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-59'>#</a>
+                </div>
+                <p>Add the prediction for logging</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">229</span>            <span class="n">log</span> <span class="o">+=</span> <span class="p">[(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">itos</span><span class="p">[</span><span class="n">output</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">item</span><span class="p">()],</span> <span class="n">Text</span><span class="o">.</span><span class="n">value</span><span class="p">)]</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-60'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-60'>#</a>
+                </div>
+                <p>Print the sampled output</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">232</span>        <span class="n">logger</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="n">log</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-61'>
+        <div class='docs doc-strings'>
+                <div class='section-link'>
+                    <a href='#section-61'>#</a>
+                </div>
+                <h3>Train the model</h3>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">234</span>    <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-62'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-62'>#</a>
+                </div>
+                <p>Loop for the given number of epochs</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">240</span>        <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">monit</span><span class="o">.</span><span class="n">loop</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">epochs</span><span class="p">):</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-63'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-63'>#</a>
+                </div>
+                <p>Iterate over the minibatches</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">242</span>            <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">batch</span> <span class="ow">in</span> <span class="n">monit</span><span class="o">.</span><span class="n">enum</span><span class="p">(</span><span class="s1">&#39;Train&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataloader</span><span class="p">):</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-64'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-64'>#</a>
+                </div>
+                <p>Move data to the device</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">244</span>                <span class="n">data</span><span class="p">,</span> <span class="n">target</span> <span class="o">=</span> <span class="n">batch</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">),</span> <span class="n">batch</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-65'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-65'>#</a>
+                </div>
+                <p>Set tracker step, as the number of characters trained on</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">247</span>                <span class="n">tracker</span><span class="o">.</span><span class="n">add_global_step</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">*</span> <span class="n">data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-66'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-66'>#</a>
+                </div>
+                <p>Set model state to training</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">250</span>                <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-67'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-67'>#</a>
+                </div>
+                <p>Evaluate the model</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">252</span>                <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">(</span><span class="n">data</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-68'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-68'>#</a>
+                </div>
+                <p>Calculate loss</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">255</span>                <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">loss_func</span><span class="p">(</span><span class="n">output</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">output</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]),</span> <span class="n">target</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">))</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-69'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-69'>#</a>
+                </div>
+                <p>Log the loss</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">257</span>                <span class="n">tracker</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="s2">&quot;loss.train&quot;</span><span class="p">,</span> <span class="n">loss</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-70'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-70'>#</a>
+                </div>
+                <p>Calculate gradients</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">260</span>                <span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-71'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-71'>#</a>
+                </div>
+                <p>Clip gradients</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">262</span>                <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">clip_grad_norm_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">max_norm</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">grad_norm_clip</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-72'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-72'>#</a>
+                </div>
+                <p>Take optimizer step</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">264</span>                <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-73'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-73'>#</a>
+                </div>
+                <p>Log the model parameters and gradients</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">266</span>                <span class="k">if</span> <span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">%</span> <span class="mi">100</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+<span class="lineno">267</span>                    <span class="n">tracker</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="s1">&#39;model&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-74'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-74'>#</a>
+                </div>
+                <p>Clear the gradients</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">269</span>                <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-75'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-75'>#</a>
+                </div>
+                <p>Generate a sample</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">272</span>                <span class="k">if</span> <span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">%</span> <span class="mi">100</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+<span class="lineno">273</span>                    <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
+<span class="lineno">274</span>                    <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
+<span class="lineno">275</span>                        <span class="bp">self</span><span class="o">.</span><span class="n">sample</span><span class="p">()</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-76'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-76'>#</a>
+                </div>
+                <p>Save the tracked metrics</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">278</span>                <span class="k">if</span> <span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">%</span> <span class="mi">10</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+<span class="lineno">279</span>                    <span class="n">tracker</span><span class="o">.</span><span class="n">save</span><span class="p">()</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-77'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-77'>#</a>
+                </div>
+                <p>Save the model</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">282</span>            <span class="n">experiment</span><span class="o">.</span><span class="n">save_checkpoint</span><span class="p">()</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-78'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-78'>#</a>
+                </div>
+                
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">285</span><span class="k">def</span> <span class="nf">main</span><span class="p">():</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-79'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-79'>#</a>
+                </div>
+                <p>Create experiment</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">287</span>    <span class="n">experiment</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">&quot;glu_variants&quot;</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-80'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-80'>#</a>
+                </div>
+                <p>Create configs</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">289</span>    <span class="n">configs</span> <span class="o">=</span> <span class="n">Configs</span><span class="p">()</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-81'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-81'>#</a>
+                </div>
+                <p>Load configurations</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">291</span>    <span class="n">experiment</span><span class="o">.</span><span class="n">configs</span><span class="p">(</span><span class="n">dataclasses</span><span class="o">.</span><span class="n">asdict</span><span class="p">(</span><span class="n">configs</span><span class="p">))</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-82'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-82'>#</a>
+                </div>
+                <p>Create trainer</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">294</span>    <span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">configs</span><span class="p">)</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-83'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-83'>#</a>
+                </div>
+                <p>Set models for training and loading</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">296</span>    <span class="n">experiment</span><span class="o">.</span><span class="n">add_pytorch_models</span><span class="p">({</span><span class="s1">&#39;model&#39;</span><span class="p">:</span> <span class="n">trainer</span><span class="o">.</span><span class="n">model</span><span class="p">})</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-84'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-84'>#</a>
+                </div>
+                <p>Start the experiment</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">299</span>    <span class="k">with</span> <span class="n">experiment</span><span class="o">.</span><span class="n">start</span><span class="p">():</span></pre></div>
+            </div>
+        </div>
+    <div class='section' id='section-85'>
+            <div class='docs'>
+                <div class='section-link'>
+                    <a href='#section-85'>#</a>
+                </div>
+                <p>Train the model</p>
+            </div>
+            <div class='code'>
+                <div class="highlight"><pre><span class="lineno">301</span>        <span class="n">trainer</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
+<span class="lineno">302</span>
+<span class="lineno">303</span>
+<span class="lineno">304</span><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">&#39;__main__&#39;</span><span class="p">:</span>
+<span class="lineno">305</span>    <span class="n">main</span><span class="p">()</span></pre></div>
             </div>
         </div>
     </div>
diff --git a/labml_nn/transformers/configs.py b/labml_nn/transformers/configs.py
index 2afdb37d..e11c984d 100644
--- a/labml_nn/transformers/configs.py
+++ b/labml_nn/transformers/configs.py
@@ -19,6 +19,14 @@ from .models import EmbeddingsWithPositionalEncoding, EmbeddingsWithLearnedPosit
 
 
 class FeedForwardConfigs(BaseConfigs):
+    """
+    <a id="FFN">
+    ## FFN Configurations
+    </a>
+
+    Creates a Position-wise FeedForward Network defined in
+    [`feed_forward.py`](feed_forward.html).
+    """
     # Position-wise feedforward layer
     ffn: FeedForward
     # Number of features in the embedding
@@ -44,7 +52,9 @@ class FeedForwardConfigs(BaseConfigs):
 @option(FeedForwardConfigs.activation, 'ReLU')
 def _ffn_activation_relu():
     """
-    ReLU activation
+    ### ReLU activation
+
+    $$\max(0, x)$$
     """
     return nn.ReLU()
 
@@ -52,7 +62,11 @@ def _ffn_activation_relu():
 @option(FeedForwardConfigs.activation, 'GELU')
 def _ffn_activation_gelu():
     """
-    GELU activation
+    ### GELU activation
+
+    $$x \Phi(x)$$ where $\Phi(x) = P(X \le x), X \sim \mathcal{N}(0,1)$
+
+    It was introduced in paper [Gaussian Error Linear Units](https://arxiv.org/abs/1606.08415).
     """
     return nn.GELU()
 
@@ -60,7 +74,7 @@ def _ffn_activation_gelu():
 @option(FeedForwardConfigs.ffn, 'default')
 def _feed_forward(c: FeedForwardConfigs):
     """
-    Create feedforward layer
+    Initialize a [feed forward network](feed_forward.html)
     """
     return FeedForward(c.d_model, c.d_ff,
                        dropout=c.dropout,
@@ -70,7 +84,14 @@ def _feed_forward(c: FeedForwardConfigs):
                        bias2=c.bias2,
                        bias_gate=c.bias_gate)
 
+# ## GLU Variants
+# These are variants with gated hidden layers for the FFN
+# as introduced in paper [GLU Variants Improve Transformer](https://arxiv.org/abs/2002.05202).
+# We have omitted the bias terms as specified in the paper.
 
+# ### FFN with Gated Linear Units
+#
+# $$FFN_{GLU}(x)(x, W_1, V, W_2) = (\sigma(x W_1) \otimes x V) W_2$$
 aggregate(FeedForwardConfigs.glu_variant, 'GLU',
           (FeedForwardConfigs.is_gated, True),
           (FeedForwardConfigs.bias1, False),
@@ -78,24 +99,40 @@ aggregate(FeedForwardConfigs.glu_variant, 'GLU',
           (FeedForwardConfigs.bias_gate, False),
           (FeedForwardConfigs.activation, nn.Sigmoid()))
 
+# ### FFN with Bilinear hidden layer
+#
+# $$FFN_{Bilinear}(x)(x, W_1, V, W_2) = (x W_1 \otimes x V) W_2$$
 aggregate(FeedForwardConfigs.glu_variant, 'Bilinear',
           (FeedForwardConfigs.is_gated, True),
           (FeedForwardConfigs.bias1, False),
           (FeedForwardConfigs.bias2, False),
           (FeedForwardConfigs.bias_gate, False),
           (FeedForwardConfigs.activation, nn.Identity()))
+
+# ### FFN with ReLU gate
+#
+# $$FFN_{ReGLU}(x)(x, W_1, V, W_2) = (\max(0, x W_1) \otimes x V) W_2$$
 aggregate(FeedForwardConfigs.glu_variant, 'ReGLU',
           (FeedForwardConfigs.is_gated, True),
           (FeedForwardConfigs.bias1, False),
           (FeedForwardConfigs.bias2, False),
           (FeedForwardConfigs.bias_gate, False),
           (FeedForwardConfigs.activation, nn.ReLU()))
+
+# ### FFN with GELU gate
+#
+# $$FFN_{GEGLU}(x)(x, W_1, V, W_2) = (\text{GELU}(x W_1) \otimes x V) W_2$$
 aggregate(FeedForwardConfigs.glu_variant, 'GEGLU',
           (FeedForwardConfigs.is_gated, True),
           (FeedForwardConfigs.bias1, False),
           (FeedForwardConfigs.bias2, False),
           (FeedForwardConfigs.bias_gate, False),
           (FeedForwardConfigs.activation, nn.GELU()))
+
+# ### FFN with Swish gate
+#
+# $$FFN_{SwiGLU}(x)(x, W_1, V, W_2) = (\text{Swish}_1(x W_1) \otimes x V) W_2$$
+# where $\text{Swish}_\beta(x) = x \sigma(\beta x)$
 aggregate(FeedForwardConfigs.glu_variant, 'SwiGLU',
           (FeedForwardConfigs.is_gated, True),
           (FeedForwardConfigs.bias1, False),
@@ -236,7 +273,7 @@ def _generator(c: TransformerConfigs):
     return Generator(c.n_tgt_vocab, c.d_model)
 
 
-# ## Positional Embeddings
+# ### Fixed Positional Embeddings
 @option(TransformerConfigs.src_embed, 'fixed_pos')
 def _src_embed_with_positional(c: TransformerConfigs):
     """
@@ -253,7 +290,7 @@ def _tgt_embed_with_positional(c: TransformerConfigs):
     return EmbeddingsWithPositionalEncoding(c.d_model, c.n_tgt_vocab)
 
 
-# ## Learned Positional Embeddings
+# ### Learned Positional Embeddings
 @option(TransformerConfigs.src_embed, 'learned_pos')
 def _src_embed_with_learned_positional(c: TransformerConfigs):
     """
@@ -270,7 +307,7 @@ def _tgt_embed_with_learned_positional(c: TransformerConfigs):
     return EmbeddingsWithLearnedPositionalEncoding(c.d_model, c.n_tgt_vocab)
 
 
-# ## No Positional Embeddings
+# ### No Positional Embeddings
 @option(TransformerConfigs.src_embed, 'no_pos')
 def _src_embed_without_positional(c: TransformerConfigs):
     """
diff --git a/labml_nn/transformers/feed_forward.py b/labml_nn/transformers/feed_forward.py
index a7c92afb..57cedc9b 100644
--- a/labml_nn/transformers/feed_forward.py
+++ b/labml_nn/transformers/feed_forward.py
@@ -21,6 +21,15 @@ where $W_1$, $W_2$, $b_1$ and $b_2$ are learnable parameters.
 Sometimes the
 GELU (Gaussian Error Linear Unit) activation is also used instead of ReLU.
 $$x \Phi(x)$$ where $\Phi(x) = P(X \le x), X \sim \mathcal{N}(0,1)$
+
+### Gated Linear Units
+
+This is a generic implementation that supports different variants including
+[Gated Linear Units](https://arxiv.org/abs/2002.05202) (GLU).
+We have also implemented experiments on these:
+
+* [experiment that uses `labml.configs`](glu_variants/experiment.html)
+* [simpler version from scratch](glu_variants/simple.html)
 """
 
 import torch
@@ -31,7 +40,7 @@ from labml_helpers.module import Module
 
 class FeedForward(Module):
     """
-    ## Position-wise feed-forward network (FFN) module
+    ## FFN module
     """
 
     def __init__(self, d_model: int, d_ff: int,
@@ -51,19 +60,32 @@ class FeedForward(Module):
         * `bias_gate` specified whether the fully connected layer for the gate should have a learnable bias
         """
         super().__init__()
+        # Layer one parameterized by weight $W_1$ and bias $b_1$
         self.layer1 = nn.Linear(d_model, d_ff, bias=bias1)
+        # Layer one parameterized by weight $W_1$ and bias $b_1$
         self.layer2 = nn.Linear(d_ff, d_model, bias=bias2)
+        # Hidden layer dropout
         self.dropout = nn.Dropout(dropout)
+        # Activation function $f$
         self.activation = activation
+        # Whether there is a gate
         self.is_gated = is_gated
         if is_gated:
+            # If there is a gate the linear layer to transform inputs to
+            # be multiplied by the gate, parameterized by weight $V$ and bias $c$
             self.linear_v = nn.Linear(d_model, d_ff, bias=bias_gate)
 
     def __call__(self, x: torch.Tensor):
+        # $f(x W_1 + b_1)$
         g = self.activation(self.layer1(x))
+        # If gated, $f(x W_1 + b_1) \otimes (x V + b) $
         if self.is_gated:
             x = g * self.linear_v(x)
+        # Otherwise
         else:
             x = g
+        # Apply dropout
         x = self.dropout(x)
+        # $(f(x W_1 + b_1) \otimes (x V + b)) W_2 + b_2$ or $f(x W_1 + b_1) W_2 + b_2$
+        # depending on whether it is gated
         return self.layer2(x)
diff --git a/labml_nn/transformers/glu_variants/experiment.py b/labml_nn/transformers/glu_variants/experiment.py
index b4fb4e47..89c21b70 100644
--- a/labml_nn/transformers/glu_variants/experiment.py
+++ b/labml_nn/transformers/glu_variants/experiment.py
@@ -6,9 +6,11 @@ summary: >
   for the position-wise feedforward network (FFN).
 ---
 
-# Train Autoregressive Transformer
+# Gated Linear Units and Variants
 
 This trains a simple [transformer](../../) model for auto-regression.
+We try different variants for the [position-wise feedforward network](../feed_forward).
+The reusable & configurable are defined in [`configs.py`](configs.html).
 """
 
 import torch
@@ -72,7 +74,7 @@ def autoregressive_model(c: Configs):
 @option(Configs.transformer)
 def transformer_c(c: Configs):
     """
-    Initialize the configurable transformer encoder for our autoregressive model
+    Initialize the [configurable transformer](../configs.html) encoder for our autoregressive model.
     """
     tc = TransformerConfigs()
     tc.n_src_vocab = c.n_tokens
@@ -104,6 +106,9 @@ def main():
                         'inner_iterations': 10,
 
                         # GLU Variant, one of GLU, Bilinear, ReGLU, GEGLU, SwiGLU
+                        #
+                        # These are defined in the [configurable FFN](../configs.html#FFN)
+                        # implementation
                         'transformer.ffn.glu_variant': 'Bilinear',
 
                         # Transformer configurations
diff --git a/labml_nn/transformers/glu_variants/simple.py b/labml_nn/transformers/glu_variants/simple.py
index 0173cb29..0cf7f494 100644
--- a/labml_nn/transformers/glu_variants/simple.py
+++ b/labml_nn/transformers/glu_variants/simple.py
@@ -6,9 +6,13 @@ summary: >
   for the position-wise feedforward network (FFN).
 ---
 
-# Train Autoregressive Transformer
+# Gated Linear Units and Variants
 
 This trains a simple [transformer](../../) model for auto-regression.
+We try different variants for the [position-wise feedforward network](../feed_forward).
+
+*This is a simpler implementation that doesn't use [`labml.configs`](experiment.html) module.
+We decided to write a simpler implementation to make it easier readers who are not familiar.*
 """
 import dataclasses
 
@@ -56,6 +60,9 @@ class AutoregressiveModel(nn.Module):
 
 @dataclasses.dataclass
 class Configs:
+    """
+    ### Configurations
+    """
     d_model: int = 512
     seq_len: int = 128
     batch_size: int = 32
@@ -69,71 +76,130 @@ class Configs:
 
 
 class TinyShakespeareDataset(Dataset):
+    """
+    ### Tiny Shakespeare Dataset
+    """
+
     def __init__(self, seq_len: int):
+        # Location of the text file
         path = lab.get_data_path() / 'tiny_shakespeare.txt'
+        # Download the file
         download_file('https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt', path)
+        # Read the downloaded file
         with open(str(path), 'r') as f:
             text = f.read()
 
+        # Extract the characters
         chars = list(set(text))
+        # Character to id (integer) map
         self.stoi = {c: i for i, c in enumerate(chars)}
+        # Id to character map
         self.itos = {i: c for i, c in enumerate(chars)}
+        # Length of a training sample
         self.seq_len = seq_len
+        # Data in the form of a tensor of ids
         self.data = self.text_to_i(text)
 
     def text_to_i(self, text: str):
+        """
+        Transform the text into a tensor of ids
+        """
         return torch.tensor([self.stoi[c] for c in text], dtype=torch.long)
 
     def __len__(self):
+        """
+        Number of samples in the dataset.
+
+        *This will read the dataset `seq_len` times in a single epoch.*
+        """
         return len(self.data) - self.seq_len - 1
 
     def __getitem__(self, idx):
+        """
+        Return a sample
+        """
         return self.data[idx:idx + self.seq_len], self.data[idx + 1:idx + self.seq_len + 1]
 
 
 class Trainer:
+    """
+    ## Trainer
+    """
+
     def __init__(self, configs: Configs):
+        # Get the device
         self.device = torch.device('cpu')
         if torch.cuda.is_available():
             self.device = torch.device('cuda:0')
+        # Initialize the dataset
         self.dataset = TinyShakespeareDataset(configs.seq_len)
-        self.dataloader = DataLoader(self.dataset, batch_size=configs.batch_size, collate_fn=transpose_batch,
+        # Initialize the dataloader
+        self.dataloader = DataLoader(self.dataset,
+                                     batch_size=configs.batch_size,
+                                     collate_fn=transpose_batch,
                                      shuffle=True)
 
+        # FFN with Gated Linear Unit
+        # $$FFN_{GLU}(x)(x, W_1, V, W_2) = (\sigma(x W_1) \otimes x V) W_2$$
         if configs.glu_variant == 'GLU':
             ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.Sigmoid(), True, False, False, False)
+        # FFN with Bilinear hidden layer
+        # $$FFN_{Bilinear}(x)(x, W_1, V, W_2) = (x W_1 \otimes x V) W_2$$
         elif configs.glu_variant == 'Bilinear':
             ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.Identity(), True, False, False, False)
+        # FFN with ReLU gate
+        # $$FFN_{ReGLU}(x)(x, W_1, V, W_2) = (\max(0, x W_1) \otimes x V) W_2$$
         elif configs.glu_variant == 'ReGLU':
             ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.ReLU(), True, False, False, False)
+        # FFN with GELU gate
+        # $$FFN_{GEGLU}(x)(x, W_1, V, W_2) = (\text{GELU}(x W_1) \otimes x V) W_2$$
         elif configs.glu_variant == 'GEGLU':
             ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.GELU(), True, False, False, False)
+        # FFN with Swish gate
+        # $$FFN_{SwiGLU}(x)(x, W_1, V, W_2) = (\text{Swish}_1(x W_1) \otimes x V) W_2$$
+        # where $\text{Swish}_\beta(x) = x \sigma(\beta x)$
         elif configs.glu_variant == 'SwiGLU':
             ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.SiLU(), True, False, False, False)
+        # FFN with ReLU activation
+        # $$FFN_{ReLU}(x)(x, W_1, W_2, b_1, b_2) = \text{ReLU}_1(x W_1 + b_1) W_2 + b_2$$
         elif configs.glu_variant == 'ReLU':
             ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.ReLU())
+        # FFN with ReLU activation
+        # $$FFN_{GELU}(x)(x, W_1, W_2, b_1, b_2) = \text{GELU}_1(x W_1 + b_1) W_2 + b_2$$
         elif configs.glu_variant == 'GELU':
             ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.GELU())
         else:
             raise ValueError(f'Unknown variant {configs.glu_variant}')
 
+        # Number of different characters
         n_chars = len(self.dataset.stoi)
+
+        # Initialize [Multi-Head Attention module](../mha.html)
+        mha = MultiHeadAttention(configs.n_heads, configs.d_model, configs.dropout)
+        # Initialize the [Transformer Block](../models.html#TransformerLayer)
+        transformer_layer = TransformerLayer(d_model=configs.d_model, self_attn=mha, src_attn=None,
+                                             feed_forward=ffn, dropout_prob=configs.dropout)
+        # Initialize the model with an
+        # [embedding layer](../models.html#EmbeddingsWithPositionalEncoding)
+        # (with fixed positional encoding)
+        # [transformer encoder](../models.html#Encoder) and
+        # a linear layer to generate logits.
         self.model = AutoregressiveModel(EmbeddingsWithPositionalEncoding(configs.d_model, n_chars),
-                                         Encoder(TransformerLayer(
-                                             d_model=configs.d_model,
-                                             self_attn=MultiHeadAttention(configs.n_heads, configs.d_model,
-                                                                          configs.dropout),
-                                             src_attn=None,
-                                             feed_forward=ffn,
-                                             dropout_prob=configs.dropout
-                                         ), configs.n_layers),
+                                         Encoder(transformer_layer, configs.n_layers),
                                          nn.Linear(configs.d_model, n_chars))
+
+        # Move the model to the current device
         self.model.to(self.device)
 
+        # Initialize [Noam optimizer](../../optimizers/noam.html)
         self.optimizer = Noam(self.model.parameters(), lr=1.0, warmup=2_000, d_model=configs.d_model)
 
+        # Cross-entropy loss
         self.loss_func = nn.CrossEntropyLoss()
+        # Number of training epochs;
+        # *note that our dataset definition repeats the data `seq_len` times in a single epoch
         self.epochs = configs.epochs
+        # Gradient clipping norm
         self.grad_norm_clip = configs.grad_norm_clip
 
         # Set tracker configurations
@@ -166,18 +232,28 @@ class Trainer:
         logger.log(log)
 
     def train(self):
+        """
+        ### Train the model
+        """
+
+        # Loop for the given number of epochs
         for _ in monit.loop(self.epochs):
+            # Iterate over the minibatches
             for i, batch in monit.enum('Train', self.dataloader):
                 # Move data to the device
                 data, target = batch[0].to(self.device), batch[1].to(self.device)
 
+                # Set tracker step, as the number of characters trained on
                 tracker.add_global_step(data.shape[0] * data.shape[1])
 
+                # Set model state to training
                 self.model.train()
+                # Evaluate the model
                 output = self.model(data)
 
-                # Calculate and log loss
+                # Calculate loss
                 loss = self.loss_func(output.view(-1, output.shape[-1]), target.view(-1))
+                # Log the loss
                 tracker.add("loss.train", loss)
 
                 # Calculate gradients
@@ -186,12 +262,13 @@ class Trainer:
                 torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=self.grad_norm_clip)
                 # Take optimizer step
                 self.optimizer.step()
-                # Log the model parameters and gradients on last batch of every epoch
+                # Log the model parameters and gradients
                 if (i + 1) % 100 == 0:
                     tracker.add('model', self.model)
                 # Clear the gradients
                 self.optimizer.zero_grad()
 
+                # Generate a sample
                 if (i + 1) % 100 == 0:
                     self.model.eval()
                     with torch.no_grad():
@@ -201,6 +278,7 @@ class Trainer:
                 if (i + 1) % 10 == 0:
                     tracker.save()
 
+            # Save the model
             experiment.save_checkpoint()
 
 
@@ -212,12 +290,14 @@ def main():
     # Load configurations
     experiment.configs(dataclasses.asdict(configs))
 
+    # Create trainer
     trainer = Trainer(configs)
+    # Set models for training and loading
     experiment.add_pytorch_models({'model': trainer.model})
 
     # Start the experiment
     with experiment.start():
-        # `TrainValidConfigs.run`
+        # Train the model
         trainer.train()