From 53dde02827b05f26ea4a6369ab321e73866bb7c0 Mon Sep 17 00:00:00 2001 From: Quinton Hoole Date: Mon, 9 Mar 2015 19:22:34 -0700 Subject: [PATCH] Cluster Federation RFC. # *** ERROR: *** docs are out of sync between cli and markdown # run hack/run-gendocs.sh > docs/kubectl.md to regenerate # # Your commit will be aborted unless you regenerate docs. COMMIT_BLOCKED_ON_GENDOCS --- docs/proposals/federation-high-level-arch.png | Bin 0 -> 31793 bytes docs/proposals/federation.md | 425 ++++++++++++++++++ 2 files changed, 425 insertions(+) create mode 100644 docs/proposals/federation-high-level-arch.png create mode 100644 docs/proposals/federation.md diff --git a/docs/proposals/federation-high-level-arch.png b/docs/proposals/federation-high-level-arch.png new file mode 100644 index 0000000000000000000000000000000000000000..8a416cc1e687aac784afb03edc6eac295f433a51 GIT binary patch literal 31793 zcmb^Y^;eYN_dbr#fD9c9(kTcA-OZqaASoc-jkMIzLyMG%(kUg<-QAti3=G{hbbf9g z@8@fM|AFsX@C&T_oPDmd_dYwW9jdJO9v_Di2LuA)%gVe{0fEp)fM04X2=JFh?*vs4 z$n!$>ounFUcDE5fKIN19K_hp9y!j7V@*iNdM-k-DgJhzEDQ#8*rKxyHI}8IN;zo0L zbGzOuW_;z>mqxC2ey}A_!4Xqld0aa}=5B&d8g&xS_*3QEH&X8-k|@|WvFW!@pK?Lr zxBH7cJE>abWvRD(*DEug_xDpS2IU)r^SU^vA560U|N2qs=hLk+LnywS zn{BCDonE)B|JBL`aglp2`&l`uuPXnW?U?PuUfM4WU8QCw!A)w(d;}hMrP*PCd(41o^qEcD{{@26jOy9 z+)$dcyToQuq|L-z`$W}z>_QVS;GPF(PMHb zLDcTp-h=Fvqbf^IM;DK;51BXMpSNnvx|P}vG02*7D|Q&CVv8EB?6)RG4qh*u7SjEs zz1Uz$vb%0Rx*f|ssL4GkRr0E;K=tLz9e7Pi#f;#a<`%h37HAC*rik3!|B_f-Vt(_k z>y%hd#Vb+4J1rvKy72-<&;sTb1eR9sX= zUuH@;ecbG{UOdbh$PBMO&tLTX{k=ptAAAj!IK1DkEEZuR$-41}$ECow|Gj$hg9$$Q z9|p$wQQ_fDA_Rl<)F!COz3k>1GlMB~CpM@878dPkW~4a(nc{t868XEVa5~_lSOB&w zT{%)V%0)3z?t_;BQcI54&*@(5E!15(!1NMI8V+aNi6; zo98EI;BG4QR_jy(GyxDo<=jf_>HXhv1I8NL%m_Ni2@bD-{2Smu|J_)fynxD)Jq)FT zQDs{(b#ryK;T7PS@-6-Uo%0K?I$-IsakjRGN&QvNjgo%v#bWUP)tRMUc+=@SUlxaB zGiPZk@)&>_IB-_ezaM3VujabS_Q>@drccAVqi&LEpo?i8H1Ceuus}H(cC(3yF~ESk+w3{`i#s2p31 zYZFu5{L}fh9Z^m<*lKjGQ^Du+9K*2oqI9w1y@Rv?C(Ep**BdnR2}w^ z9b7AJ#iU(-*=nEid1gXvt|)G@pp{LmC|p}%h%~A-)VtFk#=|4CVf!)jtGcKJ3E}QD|s{avOg5yYyg!U0Mrc zD@v|aItLBK&U*@;atDn!nb}T@bafrd(Rihv*XB_u#;w_ucYlgKz%F~s%5AWlEK#>ar?;x- zb2>X^mO&`swO)Mf!yZxJbQ&n zW(Zm4*imfVc4n^x?^}4z&XiBNamqY=wW1K$vk_O?kO>s!>I z5eyP=`}OvAnnheB^Lu#uf)eFNP=}1b7n?TwcI+co`3e~ZIy^NdYb*|i)^qVCX5*YV z7cpoO*OQ4aPf=!}N2)f?wr7|1O$qqqCHuHI{0kzK&Vc>%?x${V-yUC%jb^IunF$ay>gx0wV?>4S^Wc1K zyMcr1)?AsU`K+DCkEB+}H(5;$LXgo!B+Zjuuhl6ADym98oW;# zI|bS9pHDR*6^z7N21FDL8=#lUD}&KR-wq?5d{=ZqVWmF@pYX!Pm&9^oJMM=%T>Hg1 zsdS59&#n0wh!(exx`=Kgk7HZf^^P~(JLpWT9f}qni%M)&-xe3#|I)bCR>xd^{#!EJ zVkmXxh%xdTi*()Cee zO5&zx`-}>`ecOIsLtjL3Vbdwdna^?DW}-FZQDdD0T2NSN-(6!c%x!$SowpCN&NN2ylndfcUQ=vJ#FXOqVoh#*o0rf%tSn1jNQ+>5g@0l?%##V-S6v?gg zVHuQAl<~e-u}e)d!naJ1Nc#B9R)qV&mJe>8G~tfL#up{9QohMmO@vz&<=H=o)hv@z zt<~kW+G%ErXdpxT-ZcKytHtOOoS>OLgSwS>f6lR~bWjflL4d9Zb$*`q9%+!uGZl~BCf&F6QI;7*a&UqHgc1dliv&@A)fDs3GchQ=rD z+Bx=Q1T^r|{WK>zJh%FDgT|jo%Qe0gGr7TV3nCcMHckgSY{a~4;Sjv$OnMi^^khCt z&=g}m?-y}WbZ-EPn%}jiGhJf;Rj%|N0i0k(ceix27ahRuzFK(7pex}`u_!7Bk5{y1`t=8V+4R#TJ70!0ojxLzRJdY|JC9NKuYQ^KDhSqs$1Xnh! z=qZt(AK#~v(5qW&J2%zO=~OL^7#AJnp^1JCkM&T_t=WFLRrP#{|3F9veA4*%rjvde z4_;NXe0x_jBTUIebr{h77XnAr)>?y>-yirzVe* zaMH6=-CNUmEcTmc`%B#Q=By_7S8ksNwuQ%A$T8qKk+Xp7S>+-`(AoX0<@S~F(-ZI? zONS+3AdokK$!!w5?R-IVEutykip}373ONN2P~E1#o~Cc706TyF`*|q_?Me2l1W3BG z#37@j{_&&Plh{HZs*cxo<0{9}U6o&GNFpb2;bJzr930>dLOzrv@KTFqm4d#YoKuo; zjIc7!VmqpgutI1&4Tc=!m_?BhXeU256LbDWmy$7qM>H;n^NBZF1D*tbW({q4)$V8E zZV+@`)0Vwbpg|VR5jt&S(u> zs7Tsv<}6>Ea7mIMzm0mYHKM3wChm5uZv%a!EhqKS<^LR1m94|!5w-489J-yp-A zP2`+*@?GbE;WQp$8BGBDchPjhh_*f->M((F^-dSSGtvGsOVmsQ-knQqH%m*u&e~3f zN3lhAvJN({hSx@2^Pj~hzl4eqrecc5D@{h@l{hx-2-w8?OLo-zqjL3eY`UNgFQ{%d z!ZK|k3mKR3CKOp|s-183zNo%DdNiUU(U)3oGoo9rGFJI4kdcp*lUAABVzt(GrSejpdQVCyMe_Kq zBspXw7jzu^Z|Lv|px&9WONOS+UjThJG^3FAn+J>8e5qhQ4+ zI0vDIM2Hx>S0e9uKprU)Q|+_^{T#>52`L{cfZJHC@ES_a$ekS&97MC!47z&GQIVle$a}c; zWo+f>&EsH~Lkh*V=pKvhdhM2ay|bhZTYe>8mvLAw?sU;_k|NFle+Z5}VK7TPnnmn} zA9KgE`AC1W?v@zcte>91^4$zqcOAH_R%`t0(7jJ!XG-M*d7JtnIu`N$?adn=49kti z&G8|rxX+@!V7)vbJgpjB@G)rPcCF9DR@oKlL^o1@lxkeg@esMAHdd9mDS}Pd*v-{ zJ#ANLtrV#YtvNYcoQu8|wf8=!n?>DZSS}hJe3Z!Ub1U<#EE~u7XFB!oI~*uTPavy) zF;wmWPUJEhA{54JMh~S*Rx;DDCMD>f_sisxM{8YrQx+P;qp-r{-~%<$$C716t89=8e?GHG;x}Fi zsYZ~km@X{fk{vdd&@rTs_hz#YG6J)+%`rVQQ?~*onCLV(S;7kRT7MJ2383qrvICSq zHX(BjMDzBAgm4h(tKIwKFAoVQ5U+0&4$@FumOpEw6K&=Dmoyl*M*;1gW`l6@_^_I@ zKt&!o;kqddzL2rR>Ryg_pE5>2n@zZ`-G^;PG~XuYi=RX7!Mt5M3_CBdj2Mgz>FgCd zmI56m8pK91TB%67EOy4ez7@O}Xlf|@sQfk0+;>Q7p6I%php;r&OdVQF`hP;OvuV~G z))yI;)-RaHVi)#IS#r(pj|?a_;*ap#W({V&RVS!>67Lt1gx0}*HI9e`eXb+vSqLvJ zR>-9aaX3skZQ+JckLww!y3XE_5Xf1fTaKr+dZ;Xv6h7pzz%DMo%94+a*iE$dD8?LQ zesslo%Ake(HYiS_^z#0|Koz16uIQAalAk;IWezHPe#hLzl6;F>V(!r^3WwCJK)t@G zgb1W++Vr39{@!|RYx);h_P-LvHf1!G*p31z$&|!S8rVt@TH52f>&Q_h|9<1-DU_#- zM5d}5i&+PId;}T9OlK6Y9<+BvAL1zV63#r-eP=Q8vKu=p0%^uP6f)f6?B&f0#PC2C z3MADFrwz+ePf4cIM#B#^&+)#lgsm2m;yn|>D`5<4L0hVO$qWha)#CoOZdWQGzKNuGny&uoa&JKx4I&eF#=uSPLPVjp6_ z+pQ~V%#y51o;+KA9uzCO$wO~-=%OL|GaF)oI+DFuvnvvY+R2B zI1O|Tw7Vg4zO+Ho9z9~O@{NyX)Ey2N<^NX`1OtNhyy6PB)8%3E@P>^9mpt#jD#dEm zAG7R`pJp?S+W&&5jKjN$t&mSEP8$0E^8k8n5&7^V@J|$Hrw7hVrb35vjK6Vyl{! z33^oi@l|mgn-DRjSP_LKUZ!zsMLt$!BD{_K29IFNda_L2u<_{W4xA9+Xz zdW%Ukn6kvo6Z)%}Np-32Ql5r5k`CQR>-`suAGC)QjJ^K2#;GghmQIME4SGMcG}&~b zHm;Qx&0*moqf_c$Qu5?!J3HelOHYPZbFpnC>|p0^(GiI+;0Q~$|^kAIyEv3bEh_{*tbyff6;OHPfaP9KRq651Pq9Uj-|OVU z)JiDPED_7c(m#i?lCu6Ws<7p-4BS(9VYjZ&7e74h8Sn+<-fbcmwUwXGsG;u-E%emeOrws<&4~`GW-7ui zl}_ZiluYQT(y94KiecIXn}A{I^}%XzK$M62tDg@-eBnf-8Yjt?G+7^g{X|P{?B3S) zFO-BMR5Fmt8oq)4`Oot_lV{1>cbf?$qH-#3R$B#H<#&^p-w9@y)fG&~Eb*KhyzxeU zeG%b#Q9=rjr1`}K<@u<`$87Zr^_XQNr`q69*P*DoyTXRpW*A68?(5H zK`YD2{#QR6hJUDGf%^$@Q=jMMOGSx4Cjj%;1u7<vjp8czgDAh6iO^GgH=nQ& zd`;xgpG3D4T8?zmguS44E}h0whsWCD{RV5|;J*AM$eyWw`Ee$gF3p?#Wgi##2zXT( zs9SiZ`j2W5dvRX$atCNkb0o;2+!!N@Dcd=L$LW~ft)-_DVTOqmV^!CCKxvyvHZftc z3M~s?AFK#t9NrbJMB`;(DZwl-nGu$8BqPuUB69S~L&6R0YBMO~d5F_6?@5qT!>OCk zy&+n%(pIehI49JJn2mIHz;ImJIoSRMoC7K!&db?<#mPnj-cpBM7zv8gQ>T(?R!0AU zb8O%>&cb>1*Bt14EUErchByR*@c)#v4^)0VPqh9$LJkeF_fGC#dxRP`eD>6s?Z9Q+ zByDjwr`upGqCx94^+94jx~%tPtX_Vt&WG}hF(DH_f*4newZW8`-3twaoI@k?$N||< zKJ@g6UO{&0EKrw4OWi0qzJzApc~Zz;NOQe4U;$@{2V&V!d$cKEpV#uD7H{R0mGPaw zS1{Gd1_kGero7p+_P2w_(?VD~M8>Vc(-6TR3&Ye4$~We+?o)fzTn+6a zoKy`VN45G0?2H7jt5)wdo|kVlT&?kW8-U3MOPDwSoo=N@p$nLZBnE$bDJpM&a5rq^GpZ1jfK$G+W?a` zd6N`weeTUxgZm_^G+N5UYa3_Z16Sez^f*^fT^jmJLhS*$DhM#up< zRn?h`y&$Lk-3@vp&LA76CHH4g55oYZ&9QnLTnu+2Dq{?>Z8MVbRAxC`h{N9JCA_L6 zijyBx+3jQaW2kL69yrkjR^0w*CGa;3Z_5s9!vAlr6&QRC#;4yYb@ra0?~pW95jLB8r&m78+B$02V>lS6L=G z(j20xE`nGG$IYG#R6buWmXBDR_8D5dfEFOv9xkT-TTn!GHK`>N3Zie=P@hK^S55A} z6;SN;O zo|J_1v!MvfKapVGxOb_Ad;n1OW?oLPAMz7d`D1IlTx)3|aKV30An=dJ>+3_mWQ$n+ z?wG@8CBoi^Ac{}y3{QoLx{^lU)Pz+KE94pHUDcxZdAYTZ_pMS~?X2iInx$61`7@g;^|~#gZ1?Y>lG`VytSfhZbI9gqdFGhjx+zy!@rtxY0d}xr z=xoczGpw5+`!MaoU$kSzDCOJl{d6tlfdK=a5c`8=#z?PiscsJ6oni5V-l??uEs*1I-T*C?Kr_!AL}c%T$EU4ndKADtmzr1(Z(oiAv5whaHLM_i zQZ@5ZR1LI%39{AR^Pi8kGv+3JRqv|Q#VuZEh%~63#dPX1moJy*&fFdwg1u<$bxG$U+1&ZhrFfhj5-4u^&IGf7_sJo8QKwK= z$b0Fs4-w!b{-^a1n0oZv7&cbB!Z>a912Hr8CqJzCK!SW8QYo>R4;J~wmTkS7>Xk17 znfkqqDb#?2%1v45m$^bVgE9D9wfZq<%!lWSxZU@sIp3^5 zp`mEeH%nCN<`sq+6kauCH!64iw=Hcs#ZiMmjh6QE+#i^!dpGTo;T`vDBe9rYMAmZb7GUSMzb5$XJP5WHc*4zyq5Z&O9-366+ydVI zoVWtIEM75Jr0KFGiqgSAR58pDlMfoLqIPg~89M7eL))aVsy&B>WF}ChgiC?gfa9QT zdQ`M2q!A6Y;cDQ{Wi?@1LVh~hnPY@cahWmkkSu#w?bF}Q?)nc*1av2s5aLbqNAX>k z-6rfT3Sy782&ZXvosl&il;y~0&$NSVpREmw3wS#_H3slL8Ues%h8I;1VzWq;I<%<< zTK@nEbG{Mi87g_`Yn6^1o&0(Uhom0qHlN|!_oZL_J~#NN?bY#-f_wwaTl&wYUD&t) z`!T~nrrx-jD;G3rvTiGMYlotO0d98;qM`b3Wu#OUN%r=x_~#xI!s0w4t*B0EG$IbE zt|u`*|HU`nd`jFY&AO4@)6w9e$Jf|yk&M3g*K;4}EGxK>I;7QN{!IEXr&arb;#5qP zG-S^Ari#OC=OTZjfdedTXVTWI1hlnNC^z3{11W%@rzOaQl6>8)+IE>JQlj=wb%r15 z2sb`c2yy_e-}nN_a)PcByYY}#;bS5w-7|l}q#Rz*EJ_SH*0nKfTw*r_c}}o;f;~lhTCp7HWtq-cr>+A$K7+ z9$%gOsC=RzBG92B6A+2od@ zqjc9D7Cgnrua!i+DMJ~ zv>H9#9ebnv)OC%5WAg-dB!aY!+^It$s`x~P0BVADf8!PtG}nhrpIzT^+iHB<9qeoX zc4mx8lL2nG;OmNALrD_lrUICE$!obc*PPxFpt4vXDKu%Js0Bgwzptb1w7|~olqW|c zM&d&vPf9i8t?_K&LEVwC02a%v8!Pla!a_ z=|{6m*#uN-av^bOmd07&%^I%soodzE-=2@s+dnwQC8egJV(DEKKnuUgEurjq7KzwU zVoEb*3&R+HkyMs8G#imNl4*hYU<;B=x33n-aeArm?>25=^o3JBk&^Z*c#a*EAsfqA zZHMebt>uHlqD|C1yBW&}2cJnN{5(zjqOdq35ljZsj?j=(jE$0mo=SC(Bgnj<#)WYV zx3M=i1Fq!3xZ8Zpf?K`0==2evs=Xtjqj(_FUf1LzJm@^w(-L8KZF@nm+h6-fqOjp3mmV=}`h+uI6YXP+}ruqmQCVElfx(BU2yXAJ>ZPj;QiQM&# zI^%xOEk7$^zb+3b(Mg-y87KV+50xS)G1YonKGX+;`NTRdUNpYMw6QSare?HxT@yF2 z+Qc~3a=UUJwD@td;w+-h05A}=O?7g5Yg+VIoY(SitrlF=Jk(2_Qq*?U;A)AyuDp=R z=~?mh&I~`!sHz~ZlE~pe?-vnzave)ioW2{9U}sMoW)MyH%CSuuH4%3G{K_6HH5gvH z?@&(=W*;%^awk$nGe*_<#dE9Dulb1Qv$B-8A?-5BAF!F|{n;3ed9b05X99oExBBZm zM#>Zq1hfQkteEn?k;GGe4+c4bjKB>yp9_>!3UhD-w);?}3yVhm%g6|k5KZ&5YXl93 zMKzun-7w!EZi0~qr0BdHZ*PYCb#)<@xNUyPXDQ42c%Bs82i)|jIK&w?s`}bAF^CWl z&AJ1^z$iH!XuwmYC|nRTS1Gni?a$7m}bp!Xgx$2DB=Q-WmTug48v+53`B%!y2Q zRHiTtgOJo2;H#F;H^(CdCsq-linR^_nF##p3T4cCjrmNNh)+Hd>ua#{bh)9?tBh)X zc8!+CPU(7PQ{)ottnDXRTdQ}d$syoZ_|1#XH~c2> z)Ghg{2i6ZpjMfbzX!!cu+A=M>j&&Qm=&#brBo{3WG|P0}KppE{l^HBN)3;yI@ZQqn z48CFh3s&HRGvl@16}uJzRqk-q8Gbt&$V_x2@RugSoXj-D(mD8_C?QOceB_H<_hK_C zT+uAwk|dpNTtGy^Esf_!+D63h znpeWy()mVkcRH^Gr(wz{=W7BQFK3dQg5!2Q`48A*(9dHKIVaD-msfh!M`iq)Tu&!e zl$4;X6Tp6TUJ7q;hb&!%buO| z?P>+CjYiZr1bMtE4NLI?6WeNT-~f%HveKpAJp_N&2n8+c0N_X^zfJzt`kFyT{*s=)q$j%vfA-GmTeNPJRq&?tF3teHWcA60@|I+fM z-eM2Ul8Uv@ez5YW?cGg9wy4dcbX(A~?(~8~8h6Q3-aqcdi#>_rT^q?*SqYj_LK)fn z}B7YRN^uVgsiR~Xk@J#lvExif{%X~yQd4h8fvl0Ol|$HbcLe=Biswg~f^J?1#hN3HH7h0}Xmt$r`h^zoUiDg@v@5Y7)E| zu_uF~M6~6b6B>0jVIjXVwZXG*oOjm-@cvAaEYnahaeR-hGgx^~dn&X;n(tDlrsXXi zAmr=ZiYjs|w-1lqlf1tz!i~jE7cp%?JA~&JlG?wTTe>XZ&v-QFC|_mkGXJ7`L-$`3 zI_>NPh;y4qU-z|{2B@c218&Tw8YaM9^4%naxJ-?^s4`*dhnxK?g)g0#VI!o|jn=NN zcp^pS^sMh<^9Jp%P7q!QChnLzsemxjDBrou(VY)#<6bcFFTGqGC$P;f{JU%LXgiuE z5U~U9t&bjYP3>P{&i1oTT8@-eboX9UFr=;`kw+G<)Kya8q!FPyXFa>xOoya;o$2zw zuW+-nQzxf^MD@m)99OeYL?f2Jt~!%^V4+B0{EYSWT+E$_om>IgL&Bn!mObvwle7V^ z`hiCig<-pSt2pi_Cp9(F(_@bxHQ?EI*~)_VLB+;-&oLM8i&4pU-%YwLWQfm*#%Hf# z9jB^Zb4K}WCLnp?*0QGvF!fFwl9&=({b3!{ zFq}HE>oC!z4Pr4PZyV#l6{@TKP6I90O0K? z)ISiJD+8R?@x1dt8Rdwxaz5JfE^@Ot{pImau8Y9nh6{I(vzS+i&)xFsIStWPMO*00 zfj$oMdzCv+D-0zYrC&kma(iQxRV<|J{nfLZyLJz0)axQ7Jw<}RaG8RwjU&}hw5>)o ztGrE0+IXIgb3y~&yNHq3whJeC^z_LYsSTM@=iEFxPq`Xadn3~?6OI=}=HHQi^Z9UR zmm%ghS#z7d$73-xg0l8CX~)HT_45HnH!7agyY1rJdh}K%qx!jNQt+5YT^@D=UX}*V zzpUS37YKzwSRlZbfycabJ(@<#D2&faPmsQ|M*Pkr`fAZ-Lg|}36A%E??{#lzJvCR4 z_*^%!7}njuQkPm*#?qAbaE2Xh#$@~Vn4VH@nDpKw)dKxNRw~t&gEttsMaC+zK-Vc% zR?PH)Pf2>#TG7_fxPR4Cnb4isE3HkV8ljO)gqJh$+?7IkU07y{Ov#%Xgo+6W2rzIZ zo?*Ib^t7xjj;|5ep%U%FG3ed@@o=W8ImJa<5~G$Id8o&H8s+wm^Shx9?xPu|=u`iTmbTBE_$wTkB?N4xuPw~p18(B9N z;+#e2HiiV{I!PV2rx8E>`e8u^=}P5q!a9|-7G5!^!LD?LS)!W$Ofz~m&xTaNqg16) zpVciwNsCqX@9Qw|Sb-;djeW|lk>0iuOk}Fsu+`{#BPvo{yQ*6+hUOcSAGu!1(xQVt zEs&g^vrS$M7o06cqj~{Kct#_6JKnOA*Z(!F$50k?`}|iAru=*Re=-^&FiTvOX2^SK&XtdSU zvLWsA08&WBeI8!7nn~Jg3jzbtyW#h&LLT>5W!#KwbV;nWAGRVY)?gYvvtMX>*G6eF~&I~J@661mK z{`d7<;c&ed2dh|d#CnXrdX=ryM&!JD70a?on!{LcHom>Mgxt|J%arfmlP!_`m-7Qd zx`b1mr;aJQ98)yDakcG?O2lYohOPeRf@ZH`vgD~2)W|gQ>2UaP9ysV~8da)}yH0(y zE|tS>C7;idh3?&9rU*hkV@14%db*}F;-ECb%vk3ZSiz;ZsbE<14q!|^J_Ds3S7QK| ze8?5&{QH%1gV>x7$`x4AP|8f@3G_VvbINqsfU3J#36t2gIJfVKZ_Nu5liQgMGn8({ zvzy;CX95P-c4x-W0T)pnlz&jj?e6A^{>?@aUoR$8p2Kp)k-osMvG)=FOqw4Pem)v$ z%n9pvjj-QkRSqn5$);lL&*~4K$`=1pFv;c4d1ue%-XnJ4!aov+ZN;xlgR5> zRj-PP=f=z#0ZoT?%9->J93Id@h&J~Fg+Y%~kccCXzWP*J29ImZB#cz4>pflhiIkp} zkH96Sk~;O=#P7w#`Pirn9FuB5m4RK+HpM5#+)?lEXD-41@)>L;djm{t9q;Gl2Dmsp zj&gO$6uQcK|HW$RPWo75yjxt3x`l?|=KY@;h-5rVgey>0e;HJj` z6&3Tfi%pZr`u;T~Q`LqLr+-_;y|iz%HX;^(OZ`HU`p&Z~bo|WuO!6>5f}MUXYrQ@| z;n8)8NA=BxJ#yhNp*eMc&U^J^_f1Sv56R>LVQxX=}R` ztJM(94bkpKk3A}cTHOJ%yOSEDyz#m@)(nai(01;KzE=lXq=D*&!HK$Igcm0JAO8NARG|$@r{HoeMqw7|oL}Vd~OhXxg;z1WLJ0{eQCD zl)C=;`2AsD_&yIC4WJaP=MUvh(s^|Z)k7b_JE>5P%7hMDjVt$tgeYhtVJFDd2RlRoDWB(oi@9S&51a`(kW&i#~Im3Gh8FVpV}q$Rg8^mYxf3U#Sp z%;^(OWW@jVaY+ItaxoS50YegW22Y^*MK;h(-0r)Aq>Bx}&R(UI?ibj2SbO|Ua%{K| zT)-Nc1`Ns4%*&#@KNN!PillEP1)sn`6!m}`5&ON@#Hzd2_@RN^?lqO8BjJ05r9O4_ zgEw&Y-T9gsc?UVqrpnnefAYBK^atVH(dxwH>N`6==t{ytu-EiFS&=skwwhtyUJhAv z`F|^mH*+`|1bimRq8$xD%u!pXa4=E@m;v4ea23Jp1mkj_X73{o$Y_ZK8ZK{G<-zIC zP5pKd4KAKvlxYm9L$XQ;`>}5gY0`xBFKJEFz#~NI9A1lN3Asu4Gm$7VcXjD{7DQLl zl|yg)ym!DO9f0$F<70}@{0kt(1#c3upebo6u#ppWP%%GL**)8Nf4dQKTi4z1n!kv^ zpl?8=d~Z*qQTu~C-UZ@vj6Za<9H!-k7@0@QfBSLatnOJ-*z&BN07$>OV2=tW1u z&AYPqm(1V_;GHz$3ebBP2_&OUrP##WZ1ULKVUFW+h@9S=7QwqYZhV+sQuHX{xE|jr z_XukSad#t8z+|Aslk^1kBo`>}4y)+zo(!6KOz2`9iqF=w#$~#D zU5xo9CEdZp6YFjx%DcFiCWOAp!>x_ZgEau+M^R^UF z$?d+b4!O@7ri;_=YyUz{w9%^}9j9o{l|%%FV)JQN5oGxA#-8)7ju@rNFHIjoY;Zeu zGeJt&R1x3`PXF-N=DpmKL#TSjgcy^xmIOKAc_<@@Y@H%PrXA9B+P{Pf^ia|nztCzr zs$Qgs?d9^thm7QlUUV&lMPZmtgFJLfe9V4-BtMyAfyb(8oC#8agG;LuIPvtCwWe&~ zKETuv%Uc;ujn(US#pxUyIuBhk^H0G77>HUUz_g;Q^Fp@75HVgY3d42L-~4D#=SmjW z?4CJ4gKX*<)n;Xsb%oqpEj=+iCUCRFJE=*cT!jI2?Q<-!hgn;_I4RAB@c+Bc(VZAS zqiA;|FwxoXBVsO_HG~}@9O5Fdc=sy@&H0sDd!){)C&qnImfDW2GKvxabwu6+J?-J&TK_`FCjdcXE7-#_+I2V)y zcgQ=>Al5?~@M-|~KTmf#?J{~ATA-5XmfP_onq9)%X+D^iZSq)RM{8bc0OF}uW;OEx zDETr1z@vRD`d9SD=DzT;W$kk0^8oAOJVKZKNf7`1UTFiv%4b?JyNV0;NhW!kLWn_)|!cUYK+m z?ov$HzltN^O@yTNomY-Ew^zb|)1D)aUNa6~tpG2ri`fe01KDoB-(bG1_<&p~Q6q6SVtiXaQ%$%eTnh@NBpFzZNFOrVC=+T#u}N zVZ!q6BX9$BunfBM7LQ`gG0%Yeb_@6(paS97_kQ+cqWdSq1Glsd)@YWyK#S)OFDIv= z*|DAPdArK=%+;e%3XU8(G_6vuz)ca;Wmu*iw*3b`f6O7)z7Q3Z|c$7vGlJ2 zQxx7ZICcQ zb-kIAGTIk!n7)yANyg$A;rA5sc_&F8p2>+5b)j4A;1{$D;}TZeT8zv`fCh1;54|-i zPLVit+`A|@Z=Pxqzf)43_5r@LfJ$yik4iPs$`~#O8XL$GgzUC^qSattExsc}<`S*whQ^aXsBimspLrkbx^Css-58tsPlhgKt> z72>v}&q$|Ke>tP6V<-R}uCL;xN@u?XH4f1Te-b2WRQ+EtQqgBtWK+<_;MiWaOod)^ zxo19@c5oaW^o4N6hbwRq-nR3uu5f zQn7OTs9ldHrpuPKN$N^77zs77MmmNLr`Dwi(JO3Y$pGqKU_iCNvMH9A1r3(~seQYv zSJ#lHn|iED&|2Efe4N}yvx&~uC+nesoC(vu4-E^jG+XDFzB3X8NJEh-i&WRhUMRfe zu$)aGeodEnijGr2+ zyl^?~m=YpzHY|+|OYCV|t8mV*M`do_tFt!F!q)eD2r62X^#6R409P~1fm2HvWPpI0_b11oOB*PDJpU3SAJC;5G#&j|11x0{Vw)s#2rbMwKeoPcM9#3FzQ7<`Q>!7C^zX>L{{?dDwFPoi z-Xe*oU)-!+zv)M~S8es{c`O$Uq4TDN%C6(%2{Ox29oW%AyZhTi_#atT5l#{F;H$0h zF)J}xnI4)>p^d=mS6;qg)5vfyQ7xG_0o@urgZZTYvo`_46i8)fHVYO5|GoO6p>6&O zEG03ui59NEJUfRARJasU)wTq%_OKU;qRBi(?^0h?{e69{X5D|rbiS*hUhEDgc+%oJ z`Bj)4C^)H{Aog;;oRATwW3PZ<6uTCxV(K_ZthX$0;tL;*kfu>LA(3K580kmqG3p$@ zl;@9y&cIOL!Ta0Vq@GJr0Nd?nZ~|nl{>Ux_-ZWn6;2<|tDYkq3K5Ne2q4eIS%f?3j zsShwTgHv4Gxs(NOXkJy7iBhY?ulvNJ<<$T5wcm`?nGj6h!_ z$zXrkb|9%cA3%d15WL|1OAK5>kw@v+M_<8-X1WGg?WAOb6TiICMwDvzX_)>-8Bq0qr67VS$E9> z)?xu>59I%A?<>QiYP)dvFfeq9bO@p#(p>`#s7M-ghe&r1JxGa^O4l%`fGE-((xB9! zAl-<7G)N1aJ-qMto$v4YfBfaewf7TiuV+1L-D|CT25w%ID;T_w;-wTIjsUbmR~-ON zdP2)<2;trx0h$$4Vhp$#cV&jP=$dLsEDH^$3)qk%B}^9|%kgyklqJiOG)4%VF^ZEo zT&<@9QzMi=TlV;gr9KY}QbJC&C|?l5;bu%5cN}i#uLTc>$*_&M8ExET^GK&Vbq|SM zI)|IkTat%M!BhYR>-+ zIFqJ4avuHR7Q)X=IrP$Xf7aKY6=)hMndAe_ERW_;8et8efWKF3n=SuzF?)@7*#Ky9 zV|>lbzeM$t2~lV_Li&E%Qw)ul#WjVN4Yct(F`8#BIjJLIh%A{D#$Lb(Mvd3U66I+y z|I?p{vvsj=S!#eOL2^x689uM}H)uCyzvb?2P!nAsD@$x>b=un<_pdasvPsHSkHY%Q zR&#frT!avPJB7En>`t1j=0{2Ld1`ak`39u1{#o{$%JIx5{LX!LVOVG4*YknTiQJBh zhRQUyv9IKKe_xO5YkYfJxGX%2=^fjjZo?(Z^IW3R@#P228y|rDk9qd>op&1c5Wy`W zvmhG3*{=qv{H?|_${bXnINcj!zcj>?|Ek~Vz`kpSV4mq=Ek#tp2-%=$nN%F>%N{_n zQS3GYo}D-eo}Hxl(c)L&hArM;jq>FB!JN8v!-OkMfUE27OD4QO10Ba(aVe(sCiK&F zjPR(XSA7VO4&Zi#+tfy=j7w5`YP8m$D?*zW=31S1d|D*M<_`n;9FXghV;d!u4O8aY zU$++V2S4!Uy_i^8d$x{7dms2=uUbBRMy+IN z$xPQ<^z9>Vby|RH%N`Tur<+@>E>|(&`idwU->1T?IV*#LkL`fLwJXcBiMJ9G@Ga8) z)NX~@{Hd)g6zIN+xb~-a!BK`Ga5m`tLiWtpz_7h(nr*7LYG(c1MdL3l;biLx46*gx z!Wj5euhlbJ8gdM`!2aD4l%2N?_j&I3JI3AYFw)a3P1JVf7_-?}(dA#_3rx&hPcwb# zM&j*Eg!yqxg%~ONaLr;a&TNJq(e3`-@ocv$_yWpipj~|fSgDEw#3-JP#T%Y1z)@_z z+GT--Yjt9FX@Uobzc91Q=eOBbPEF4>-ks7kx0V~_D9{@JNY=Wy!zVoO#6vzZH@6P^ zd9`NdJgM=!J?DCgurr7kuPaIhoxf-*zT-X+0pQvs;-2XcU{deJQ37X(pJy77Kbpj- z88+vNuY%?a`Q>UItMNx6d{QA`xzc!OoAC9mqr;1`*MujC2ZrQ?r~Jg51f%uiTfVvj zL2{j%*Z!tHo1?`wKm6`V33w-YY#Y2w;4`jnevp`baTxOZw2yT-?k-P}q3_6^Vqk^z z)ejNxi80of8nlAn7~x)*AJr&%J$$D73J~NAWd3a$(p(ZTTcwEORu1Qd%V?u)S};qQ zl8YfJ`oK!lIRnSA@z__ct>}byBiNa5GuQHGvqf_yWzR;d`6gO0F4i#}_y6>~>HCE31;xMVFJgW`n?G?!VPIa|$A( z-00P~^Sm5U(`hxZGap@oL0J+$F_QW8@{wD05)t(MNE6mh<{&I{DTe)xe((A~rzrIer z>`YsG-L9uv4!2#+02eoH)1*a+3G}n^^q=iy#b&33=3l+Mz?Mizks8!R@6aW1_Z_&09kT;>LHfIQ+1gQ7m z?0bJMqzT${g2Dd)TkFL9J9DZhs!F2_y*qaa4EmkG=0pX#z@4K+b@0?Kn)`S1D9G>p z)R*7MFgL$&y(~Z_Uv@beCy*5g;`$W@V%`^wBRM-0qZfbQB|bbOK4l47cUM@Jii%@B z-*Z_0GwR*kJYafWpZaG5vZ%v;#V+2aw9v z?OmD&ebqe8tk!_$*cbQSc)=zx0JNEL2XefB04KP>B$MD> z6|W!4IHpT>DHiQ_$&D}MN_u0fk&_JT6>ZmS3)z-n027t9>|AZik@cJiiviIbULi6L zq7zT+pXVHOJG|XOs!^GpZpB7zn_*YlpwIhL-uPU44y)77W_qKcJ((T55%Igqi2Qdj@O!0QthjAObS;X@&?;6lI( z;fP|wtLFcRKWIru4sQ?F>R5((@t-P#x&P^E#TPc=Ii+%=_H4V`wZHByXSFm~AFh-V z)~Cm9-+*6h{=1atdoe$tG+FZbBsPPYpql^Wkmp18#ldVGGiPWHVB`RxWpzsqj7#yJiV#;esr=0t zZ>b=|BAiRvRH}}iVv;miIvAKSBY1;kW4%zjh7>o(Tv`*tQc!hv9_@uNDD>}_@lj$J zAE_|hUw>6%{b-2{sHY9XFKG2V4wjp7-sVHO2*SbPm(`d-C54@zX`L0B6iV9q~u@ zqR@>!j4Zxgal%IX+M*{iNEnfb$?}|!u)StRCFRO%W8tLDIa&43yt9F)rk7LFw|OcI zCxh@eDC8>`as)&Tk+{mV)0JNhRT+A{KEd6NbB*kakdx9fb^?EXJXJ9M)eelgW}I^E}4%bQa?XXw2|HfM-S zRWtxE`eXWIpLeJvGKOzcks7;;_L`5cr{n80!%IRI>+)O`PKJsJA|ZS>~D`*d{&w5fbcWhDph4kiBsGi@r1Co!2knLfb>TvjM$6 zrsFN zI1&8glp5ZOjMLV6iM1#g!@!>sTb%nHgMRe&2Y%l45f)-^;+D2fvsyg`pJ8yPacU{< zaqnDu>i3Jaw;^og*8?@YhO-$jU8Y~J55T9&5N+ePGEDVjp|niHef?4H+G3Oo3X zb%k7x+Or~DXf0?o}l0A)9f0M8qJ}Y z*FFZ6lmrwfsRz+x7T+tKw`WY)_eaX7Wrc*cONyIB&E+0fpHAOJTZ>o;jk(qZzqwO6 zK6kv34bm=9loV^zB1qNe z3%m$jRNJ0fJtxd~ zT_Tp{@n4}smj_W#90l8)Snj80anxDImr|QEd^S8={qKmdA=?I@vdNgoJ^%!@tH&j-clva zzBTVWC~j4)qUjO#koB%i@dh(IJ+sR7qxFz~wn0k=zFlrSU-uC4PH=9V%z*81kLTrP zG5G4o%U55|)x}R0J(@-p{ODsMXw3K{t z%y6sa<}+XQt@HNmaynyfoSvzIV}%wiZcnv#*-gzEwrG!^2oWMV>xX+@g9-8b-k@qc zvrV6?Ta0^!Dga)7zr)_Pe@P(fB;#nf78lSjS3tI}ntfAMt6eTc(gu z#zW&F)Jh;?hwGss!CE7*MKkb5QgujH?ya7ktDCj?mPmUIElWO#haOOR0T^-NSOuSN z2iD9P>OBP&ixi|15CQX`m(zm_RiQ@o;>ZWVO@TJJlKme0o85n?91`01zIF2jfsN$6 zGT`L|=D#i}4cC|0;EDp4)4{YJ1A4hXC#Q8=zxlA^r)x*Ky8Kah9&hpIvPv8sXA{W( zm3aEMlmAq$f{;)h43J8^#&gVa#v>(ja;7Uxuhr=RpJWWcj5kP0qB|o3@p8rUB0-md zbfW6GR;sTT3dePVwug(j29L9luKEg*1kb>~U<*!_7g$X~g3%@X=z`-1dDmmih1!%u z4TRY`?NHy>gs$Ky9DNjAn)W`ghmdYOPq+Yy@C;A3t8%CiA1U;vaUV}LxdtRyFb15r zH4K!=uyhxVAfKZl5g`X-xEs|Q%e824u5c@mY1U>`9sVbXQlVHe{6m=NX=X`K&7X+o zQ=d)$9+I`P+P@nYU56D*xRXwcZ53W&dz`7%G5;4xm~9_2Pi$bAoNM zt021!`p@ksSX${hm~2jc06!ik%a8Vqw<^GzTy(7)m`HZxl{X`2eZk5bGeYD1>|q($ z+t0g|Llyh;C_R5cCk0V|8Pv`Z1AWuQiz1EsTjE^SDIPrk*$|k^l^FvBklWkrR+f?E zD1uuQCG9Zs{Box`Mf}=v^q9SfN9A{Be$=KJPFy{tOOYOjGrbBK`*y(BkzQ*5!c|tw zkd!^G?&>5gt{cBPWJW#rKOYF(hh=H&Tc-=rTU08R+d2Gi)w9-~bEE}fgVs`tVzM1I z{4yZ)2YKqIpxzFIEem~SJ55|uNaiQZ&~ElVT7cQnyuDVS+kvCnt(F_N{I5C@4Dzd6 z5mvgHG5_s0KTm)V96>PAngsP$MCC`iAmlX1fVl%E+2m)?QUN27mU)jJ>fZKkj`NVt z!UA15j(p`(fxz|U0nWYAJXNj6fv}W*QfD)JXx83r8Go2vF>Q(HDRltJ*sbt8-fl#L zNd;=#EcvRjt*IBLFXgG<#y&nET2diL{a{xDkMSjR1@STlGTg{8>Dv?-dZQcf(2N~I)`!cZSb90Ix4U0Ht zou%|A3Oa@d3w@>R)uwV5bko=kBg~kg9Oynkmg0;p?i8bNy6H6v5!-g^MVX{fXHa(w zEEaZUvK)GCEA*-w>N_Sx`!(=t{sehq{y8rYcy3yRYA` zRDZgrVx7#PfYbQPPw^6GA~zktL>e?fB$rMj_{uLIt{G`>zJ&~D)>AhVCE48pO((HQ zstJz&J2+@q8kjKcs@iE}oiOx-YdzQ>$?`fqY^hJguJ_B;c9Hxz?x&o6;{69Leo}_u z6#t!T={_+11BiQNHAPUk@!Z>n3bU6ki?UikXFrT%=u%E(!m=}2SzKpxIL$d5Q~uW; z0xE!MB7_M!@1#Uy@f2i39@6Rqd6@c7}Fw$X*F(fUnk&R#7n>GblMo${Lf9I;{x8nOJEOC z%|<%M)i<<2{|-@AOH7|UG7jtj?ruBX2y_rh$)xUr7)bKk82$I6099681Pp!@M8ma9 z1!v}m{gs5iq=lx|vZ1=R<#F{}hGPWmeRpZY?K%(|GO`D$|B!^##D)c%r8mJY+(<}G#WTZ9U+8#WXZfXqAg-S z5-p(`#+GCcoR@Yk{UeOE!_yNBE~UmMjr=QZ zU_vR9@6(%0s;x^m0+8dh+Pw<2ms{(IqH48#8$Km)+m{ue9KoxDq#|m$v{q+i(o#9+ zq{Sa6Bt!h!q;mFyE159T#s4$JCtGE}^vmvDETo%cabCOb3&aUJ!GM9drWhh0u1`~N zPn8`7fWTDdRtstFC+K&^tGzp_iC@k~dM%f)r{s7$+0Lu)NJ|g&*M!_$i{LQf(HOmj zvlh7CGg6FhF#aGpv->s!hur=zE=g4JPhlHBhKW6px0I_r(qN#jor>Bb)s1s`u>|6~ zMgaLq%-e?R)lHmqM(!OC70{hpr^p>NI>iP zp#IU8T$Ac#^@WRNpmE`gA*b|TY5i5PBubA9UvT~)!QgELyjS@t+za8+qQ%wO)^ln0 zWMYga&kZRS6L;t)oihBoD~6+!`#e(i(jhM{gBQ@$zXW8~KH~_Mz1I&3!WUvm@K!d9 z!_s5Bbs3-94t=fI(16Iq2H7aUC^Pi^LBDXBEkVcMGEG_hpQT8$T~WjaxNYb5MPhisH|wyHCIK;3`D&kpgjLETCFKx*}d2&VpulD^g^D6bFdgmoWd zW5?@(Bn*>z{eTZ<*0@}P(iu`^Xv!Z|-{Su1mS6OEK&e*zp$G>fAlvN|{3th6IHaAyzQ;MOJl+brEP;IKNZx9W zGcY)lK!>p1Zad$JbwEMTRN%W$-w?fuwax^J!{hYF_o9|o&>8bkwMxlK8Ttu}7 z<4#{B3o}*{N`fmmxs#tknBgaF)gUCF>FF5i+rgPmmS7c=wGgBQv_8l^@BIvo(3>6B6q;6f|>&xJuh2)s11%uR8@8Z~_W@r4Lu zyW0nRLI$*3dl}IrdVY4SY&Z?=}tFkw@0i%(^aWC_qVdI7#`FABN+jryCbo_i?RumrMTs5R5tqG0~^75GISIw6Z)@n!quGi?f3@&dt6I$9iwOGMi>Q zAVY1T<|wgn+m>X82dEUON6de&6pUZ*8aP(u?HF*M?)X71*yC~VNnD=lxqEO5Zx}5* zde1{{zV1^SIpKuZP%UBLg}*r-{@^s>V2V9ESu>!=6W~4})Fz_E!2;)Yr=xrVa(jFT z1CyJoul@DdEts@1Cf& zRzaFt)jvY&zyBlvsmbx|NUMl!!=S-_5>7Aam*UORga*?043FC3tafQz16T!uy~miICt|Xbd;-oE$4YDQwheM4(=)LW1Ex zpV)oZg}&1!{G`R!y!e&S%WVR{U?H!qIQwjo9^Xu0EY+%mb7~`tr8{1CtJ`p7lT3%vJP3G)(y6%iSw{IiMHKvlmOU1ZL-!g8yl%9PU# z`vZuVFq$z~y~mBA>zfJ}Wtg2m{MHa$G9&sCU|RBcOM>?;m>U6)7wC}ymHjVmD$F~j zU9330TC2sa+$?U7q%XC+i79sja+4c-ELpspOo!s|DxGa}*>h{C4629y3Kt=*0@S|! za;535ZJu~KRmeH8-5Y6x$4Q1#2G>B?W^pfc10ij)?xJ&8r9yynQ_k zLrGw)5NJ)_O!`cxba6nraRljrE2w5<0SF2O>=D!e2YMVvDWeeK#1fLFwohvdx631( zDED{!gpAWB*ZqwQ6l@$uMm`Z0tb};-3-aKtEXaxeP!j);9>Ju0dkv@t( z*Wn)JSFGJlx~B~?JvK-z=+Yy6T~h;^@AOnj1_=bp`@E-c=|dJS%hTO-F$2Y@H4F73 z^OOh@PWEP~FyX0)%sah4l74Y)Onk*2AK2GXP8i%?$16*2V+A}gsXU2b&#sU2dOJv& zXU%MOi|oxFkl{pL)GNAYgHuqDB#{G-%@ALcU23-a5XQVGuTA+YJ9S3HpTJ#(icS9^ zl1eVJZ|c85Usj7_G_^>}KE#bjqhR$kRxkK!atPPD~! zY(@WUqch*lG-qvI1axNpaM@^_i*3bD@aj}z<0AE97U7AQi4WhguUrh&P*NIg;zL}eM8-tG1YH(3eSpC6xG=+G#QjAB$)iE`A+F=;t3iy zQ=PR~U`BQHV7_W1sN(dBgxfCy%679_TjY0Mzb4TABY?w68*g+i(PKCol-HXUt>ZR6 zMRAGkF-2;R>`eheNzBD^finy_RLgk+M#rmg zg2UIl)cIqJjVy!}km&GdZlC;w2Z4qqldiV|{!GN`4{b-u|CX}OGP;T)$f-O7nb{dA zT%JmkGn3OT&z+WLSx?bVrbhBLg88IFs2GSWzqvw5g0HeLn_Avc?Jh`=!{gdWWOQ5+ zZM;qKy=uS@8C*mN{gS(`_j`Kv>G7&1Zadnr1ut|GYmau?Ja_PF!QY>(LvykR4$x^9 zeH`8f%0EKPy~QHWq_$bR6=dy=l{mK zh~br`5^T?%fX$i`Zr?1@umBN)H8>=m0G@fs`lz*mt)0XS(HL$LmQ4t}X?-Gn7!kyX zVpygu_Q8UuoseyhOuKk-_O%C7JK<8S?X69t8sey1zI4}VZBI+tlH3#q6vuCa-@+ZY z_=b)-XS1G-i0S-ik`*&zBTKA_{%I#dRq--7&?N;Ie0P?TU^0D5$n91cRa?vV*T-U5 z1z$8)LVAI?&hN$yuObc^5Ct}Q_%GU0rPhLT>>ZkHQgY-`%kg+#Qy;1GIr=NqhT8Xx zmlQ=#FBwuV@ijVxN$+;^eyDAv_$a@ zJ<(;hWT-O<3FjJ!WC(d>yZwwam1UYf3#Iq%JceZ_J-#!Io`=_3J-dI;Atkh9A`0lQ zi{FIH(ImyG6|Ok3`%1~Tw~a~t6M^7DXV83UZ;F~(?wuGm@}OI#vYvq)!2K;l^!~&U z8i0lJI$KMleL7>M8sA6q2RE~y)UqKy4BwlL-yH%$>N1%$`f?pQip-a{S;CaQyv(a6 z1uFts^JnmrctZa88z8BAiy9*UXCC>*9TwbOsN;u=AXYv^)+7uPw}^|uzgnuvBHA$% z$H-NoZkai&af>#oWro+hesv&@97Kd={TRltX~S~uVM2s8=aX#@DT_&e#KO2%yEUSU zyrNMYAa5`1)p}TL|H0wgclTNT2&~l zJzrWz{;js0fSVB+x)v(_2Ok1WsQ9c^{Won6dI?MnRf3RrHaSu3-XY$dB2u}bUsbRo zX|@)Zba@l8N7QfEnhaN|)_4?x$gr%R!u(VnZA?T2JsRga{rdQ;W!Ng;GhB-3RFSJn z6DZL8O5aB6qSv>w&4(t8Z~GJI@9v%cBbnDN33%#t+|G{;U;c z#BicEKbbM3HGA&QIE6{HK9=RUw>@>d+u;}OU2J>hgW5ia<(y5A!McVaIb_(k0*NCRK0UTBgVds8@6G-N6boBr9f$|@cLG>2yp=p-0lnp-=TT2_h^E6^!cHY8nn z-~6!KRPkU_u#@U)&d?~F_dyDtIranf zJPdfdA}7dsp4ldI;~YZp_?X?rtcTR^7KlUF4bKJQo5wXW&U zpl(2aEKuPCX~TxX2F$g3FM7qlbQm)W5V~cO4Ebo2B!P|W3ccGy6@_&(Q} zJuEj3x!K!`L2>7#f>iK8y*9v)bTRcS51jYw+Do|eHpS|3l$fZX7s-J(P0cV<(Ix0TnCAA!qZ$wCI#%tGSl>d1k9i>2BJa}lhzur)#d3tHWT&Vx@` z>){fu_q0PsWi40>`ZYX%)ik)Mw^X1ZAbN6*Br}{*>^FZWUxB-*+zme<``LgIh$=}v zp_DbnunhH~#y17Yfs4`K)DFr_@*bL!-|cX>OF!zWXsf3qM5bim`kzpGJ4F4EbsJ0< z+YuJ$X`PQXlUn?u-Mk$c6d zBioUb*A0n!>785S?49W#_0c=<+B!_1|NLto_T}8Je2N$i(i2Lk-<&kK#kkMrO&wOX zL$asz@HxJmRjh7V=#*2c+C!40?&EA}ws~K&mt0i^tfE4welghAuQy6 zVg1q)RC0+$!W&>p3=8I;WJkYvvJ`l{_3)NX=X(F>YhbIyk2x{6KBb80(HbJaIZm%C zTI~s{>p6KhXOg5lWHynWX!Fx_7#MR4vc@5E@CWNA^gc!1 zD(1PilRZ!R)-w_fX3srO10 zJ5+lDntX z55-)29xwOJME{ewSHbUpzpR7aac43Ml(_J(kdm~@q>Y721dCp!UOI@sSU4#~Y4V}aNX)ffYtxUI`Qt>W;xB@o^?-Rs#?}Oc$oE3eHBjL*H#r_Ty&y%h zlI|GRhQv#rTb!JDG(+ve_OS4E=cqFVO)r3><(~mo#@;`CpZMNtL701Bd8Gt4C}G3d zC{YdhtexvP{4LyIu-&fIgcuWDQSC@fXl@1j?Ke8EVw0;k>io?~FVocD!io*>e(me>q_DDC9b!(`;Yu=ttho8e7TCLkHDT&Nh6eMWztqZ6k{N8>ZgSq@R@w{x z<87z6A!6Q5aZpuF-hqP@JwbVHz3_h*X^ZcuH|u4CetqS#IYLONN-Jy2utIKc1=wu=F>8PabK-R^o7=Q5%lnXJ!iVUcsB8P<+hzLa3-!Js# zTlP%*H4Mm1jlfc)y7SOAR8(DU9l3#sLk$}ZCor9SsUI1-nj_*MJz0+1!dDLX$8TWf zmml9wU6=4ycUTK?m8bk2s(6ZCuJAh%R>BnR)da8QoE`{tw>yt3t=Qwg=Z~R=mhF9Z zE!abV4D2{G8l=$&5ApB0RppXvr(Ng?2ds}nK)LzgUaH!-YLO>fC0^CL^7hF-b9MLo zM+D2|ka?Dr8Kv}$ZKhJ=w09;*z7P5g<;9Sd zhvz?nTPxnl7|bm5<<~RXYcLmHP5@>Vy~)80X7h=oBX02))^uuJwl!~EXyBJ=dQd?8 z$P^vIUK!IGVsM*a255~5MF&?0(X{O1{~01`8}2w#RL~Qmzb2dvK0jgjKhHk^TR!C8 zfTJ7!b2;GBWTpgzw%L1hmx~%HyVL`)kznt{&|C9!aHzVBLn|4f)_rPd0$EmT;jiC#O;z6;W|@qiqqsUS-qN1V4o3q>Tn@I zJt5}e!e`z5X0A~DVyW9Rmg^*Bi7xG(t%Ze!yw9t{BTb&&xuaT@Jg7BnW~_cVcvH+v z_)MCWe-SCOAe+VrrwnvO{A~)9GL{WkU~Fs+IjOehLPHX2`h5^FwnxXXQRe>h{_(An zL*p1*1Y)w8ctsH&;yih*SeSgu+pgyC0$&n_?2ZTzvg^+s0WN9XG8bAJ@7I-PuF z`s~gQ89-!5Gsxk1EsN{%)o5D)fyhMBVF46MBJ&aWnzWAt?*9;0IxPJF89+72g}Kis zzxv{z2PwwCja!SD3pB_4_eB72f{SKwf${QDC{%_Xd${^5ixyzVXd>v>YXkVv5CGy# zcV%!{()a7a_g!#>0Dk3o+CP)h4bn0E^D^E99`5CY%aLq2@#@*p^jVbPF#RNWC%+Fk z3=2ohIz_>?RSQF!K9{BII64Y~&ms`U2!t&tA^-+4brm~peJWwFrSK|xnH${ccA+lQ zd5>MntY^J$0UjX4@{P6P87Pkj_~5TO|NnpgUjqLR5;!`5RAmb3#KiKa;oefc_uy`^ IlEt(C1MeTu7XSbN literal 0 HcmV?d00001 diff --git a/docs/proposals/federation.md b/docs/proposals/federation.md new file mode 100644 index 00000000000..6086b13fca6 --- /dev/null +++ b/docs/proposals/federation.md @@ -0,0 +1,425 @@ +#Kubernetes Cluster Federation +##(a.k.a. "Ubernetes") + +## Requirements Analysis and Product Proposal + +## _by Quinton Hoole ([quinton@google.com](mailto:quinton@google.com))_ +_Initial revision: 2015-03-05_ +_Last updated: 2015-03-09_ +This doc: [tinyurl.com/ubernetes](http://tinyurl.com/ubernetes) +Slides: [tinyurl.com/ubernetes-slides](http://tinyurl.com/ubernetes-slides) + +## Introduction + +Today, each Kubernetes cluster is a relatively self-contained unit, +which typically runs in a single "on-premise" data centre or single +availability zone of a cloud provider (Google's GCE, Amazon's AWS, +etc). + +Several current and potential Kubernetes users and customers have +expressed a keen interest in tying together ("federating") multiple +clusters in some sensible way in order to enable the following kinds +of use cases (intentionally vague): + +1. _"Preferentially run my workloads in my on-premise cluster(s), but + automatically overflow to my cloud-hosted cluster(s) if I run out + of on-premise capacity"_. +1. _"Most of my workloads should run in my preferred cloud-hosted + cluster(s), but some are privacy-sensitive, and should be + automatically diverted to run in my secure, on-premise + cluster(s)"_. +1. _"I want to avoid vendor lock-in, so I want my workloads to run + across multiple cloud providers all the time. I change my set of + such cloud providers, and my pricing contracts with them, + periodically"_. +1. _"I want to be immune to any single data centre or cloud + availability zone outage, so I want to spread my service across + multiple such zones (and ideally even across multiple cloud + providers)."_ + +The above use cases are by necessity left imprecisely defined. The +rest of this document explores these use cases and their implications +in further detail, and compares a few alternative high level +approaches to addressing them. The idea of cluster federation has +informally become known as_ "Ubernetes"_. + +## Summary/TL;DR + +TBD + +## What exactly is a Kubernetes Cluster? + +A central design concept in Kubernetes is that of a _cluster_. While +loosely speaking, a cluster can be thought of as running in a single +data center, or cloud provider availability zone, a more precise +definition is that each cluster provides: + +1. a single Kubernetes API entry point, +1. a consistent, cluster-wide resource naming scheme +1. a scheduling/container placement domain +1. a service network routing domain +1. (in future) an authentication and authorization model. +1. .... + +The above in turn imply the need for a relatively performant, reliable +and cheap network within each cluster. + +There is also assumed to be some degree of failure correlation across +a cluster, i.e. whole clusters are expected to fail, at least +occasionally (due to cluster-wide power and network failures, natural +disasters etc). Clusters are often relatively homogenous in that all +compute nodes are typically provided by a single cloud provider or +hardware vendor, and connected by a common, unified network fabric. +But these are not hard requirements of Kubernetes. + +Other classes of Kubernetes deployments than the one sketched above +are technically feasible, but come with some challenges of their own, +and are not yet common or explicitly supported. + +More specifically, having a Kubernetes cluster span multiple +well-connected availability zones within a single geographical region +(e.g. US North East, UK, Japan etc) is worthy of further +consideration, in particular because it potentially addresses + +## What use cases require Cluster Federation? + +Let's name a few concrete use cases to aid the discussion: + +## 1.Capacity Overflow + +_"I want to preferentially run my workloads in my on-premise cluster(s), but automatically "overflow" to my cloud-hosted cluster(s) when I run out of on-premise capacity."_ + +This idea is known in some circles as "[cloudbursting](http://searchcloudcomputing.techtarget.com/definition/cloud-bursting)". + +**Clarifying questions:** What is the unit of overflow? Individual + pods? Probably not always. Replication controllers and their + associated sets of pods? Groups of replication controllers + (a.k.a. distributed applications)? How are persistent disks + overflowed? Can the "overflowed" pods communicate with their + brethren and sistren pods and services in the other cluster(s)? + Presumably yes, at higher cost and latency, provided that they use + external service discovery. Is "overflow" enabled only when creating + new workloads/replication controllers, or are existing workloads + dynamically migrated between clusters based on fluctuating available + capacity? If so, what is the desired behaviour, and how is it + achieved? How, if at all, does this relate to quota enforcement + (e.g. if we run out of on-premise capacity, can all or only some + quotas transfer to other, potentially more expensive off-premise + capacity?) + +It seems that most of this boils down to: + +1. **location affinity** (pods relative to each other, and to other + stateful services like persistent storage - how is this expressed + and enforced?) +1. **cross-cluster scheduling** (given location affinity constraints + and other scheduling policy, which resources are assigned to which + clusters, and by what?) +1. **cross-cluster service discovery** (how do pods in one cluster + discover and communicate with pods in another cluster?) +1. **cross-cluster migration** (how do compute and storage resources, + and the distributed applications to which they belong, move from + one cluster to another) + +## 2. Sensitive Workloads + +_"I want most of my workloads to run in my preferred cloud-hosted +cluster(s), but some are privacy-sensitive, and should be +automatically diverted to run in my secure, on-premise cluster(s). The +list of privacy-sensitive workloads changes over time, and they're +subject to external auditing."_ + +**Clarifying questions:** What kinds of rules determine which + workloads go where? Is a static mapping from container (or more + typically, replication controller) to cluster maintained and + enforced? If so, is it only enforced on startup, or are things + migrated between clusters when the mappings change? This starts to + look quite similar to "1. Capacity Overflow", and again seems to + boil down to: + +1. location affinity +1. cross-cluster scheduling +1. cross-cluster service discovery +1. cross-cluster migration +with the possible addition of: + ++ cross-cluster monitoring and auditing (which is conveniently deemed + to be outside the scope of this document, for the time being at + least) + +## 3. Vendor lock-in avoidance + +_"My CTO wants us to avoid vendor lock-in, so she wants our workloads +to run across multiple cloud providers at all times. She changes our +set of preferred cloud providers and pricing contracts with them +periodically, and doesn't want to have to communicate and manually +enforce these policy changes across the organization every time this +happens. She wants it centrally and automatically enforced, monitored +and audited."_ + +**Clarifying questions:** Again, I think that this can potentially be + reformulated as a Capacity Overflow problem - the fundamental + principles seem to be the same or substantially similar to those + above. + +## 4. "Unavailability Zones" + +_"I want to be immune to any single data centre or cloud availability +zone outage, so I want to spread my service across multiple such zones +(and ideally even across multiple cloud providers), and have my +service remain available even if one of the availability zones or +cloud providers "goes down"_. + +It seems useful to split this into two sub use cases: + +1. Multiple availability zones within a single cloud provider (across + which feature sets like private networks, load balancing, + persistent disks, data snapshots etc are typically consistent and + explicitly designed to inter-operate). +1. Multiple cloud providers (typically with inconsistent feature sets + and more limited interoperability). + +The single cloud provider case might be easier to implement (although +the multi-cloud provider implementation should just work for a single +cloud provider). Propose high-level design catering for both, with +initial implementation targeting single cloud provider only. + +**Clarifying questions:** +**How does global external service discovery work?** In the steady + state, which external clients connect to which clusters? GeoDNS or + similar? What is the tolerable failover latency if a cluster goes + down? Maybe something like (make up some numbers, notwithstanding + some buggy DNS resolvers, TTL's, caches etc) ~3 minutes for ~90% of + clients to re-issue DNS lookups and reconnect to a new cluster when + their home cluster fails is good enough for most Kubernetes users + (or at least way better than the status quo), given that these sorts + of failure only happen a small number of times a year? + +**How does dynamic load balancing across clusters work, if at all?** + One simple starting point might be "it doesn't". i.e. if a service + in a cluster is deemed to be "up", it receives as much traffic as is + generated "nearby" (even if it overloads). If the service is deemed + to "be down" in a given cluster, "all" nearby traffic is redirected + to some other cluster within some number of seconds (failover could + be automatic or manual). Failover is essentially binary. An + improvement would be to detect when a service in a cluster reaches + maximum serving capacity, and dynamically divert additional traffic + to other clusters. But how exactly does all of this work, and how + much of it is provided by Kubernetes, as opposed to something else + bolted on top (e.g. external monitoring and manipulation of GeoDNS)? + +**How does this tie in with auto-scaling of services?** More + specifically, if I run my service across _n_ clusters globally, and + one (or more) of them fail, how do I ensure that the remaining _n-1_ + clusters have enough capacity to serve the additional, failed-over + traffic? Either: + +1. I constantly over-provision all clusters by 1/n (potentially expensive), or +1. I "manually" update my replica count configurations in the + remaining clusters by 1/n when the failure occurs, and Kubernetes + takes care of the rest for me, or +1. Auto-scaling (not yet available) in the remaining clusters takes + care of it for me automagically as the additional failed-over + traffic arrives (with some latency). + +Doing nothing (i.e. forcing users to choose between 1 and 2 on their +own) is probably an OK starting point. Kubernetes autoscaling can get +us to 3 at some later date. + +Up to this point, this use case ("Unavailability Zones") seems materially different from all the others above. It does not require dynamic cross-cluster service migration (we assume that the service is already running in more than one cluster when the failure occurs). Nor does it necessarily involve cross-cluster service discovery or location affinity. As a result, I propose that we address this use case somewhat independently of the others (although I strongly suspect that it will become substantially easier once we've solved the others). + +All of the above (regarding "Unavailibility Zones") refers primarily +to already-running user-facing services, and minimizing the impact on +end users of those services becoming unavailable in a given cluster. +What about the people and systems that deploy Kubernetes services +(devops etc)? Should they be automatically shielded from the impact +of the cluster outage? i.e. have their new resource creation requests +automatically diverted to another cluster during the outage? While +this specific requirement seems non-critical (manual fail-over seems +relatively non-arduous, ignoring the user-facing issues above), it +smells a lot like the first three use cases listed above ("Capacity +Overflow, Sensitive Services, Vendor lock-in..."), so if we address +those, we probably get this one free of charge. + +## Core Challenges of Cluster Federation + +As we saw above, a few common challenges fall out of most of the use +cases considered above, namely: + +## Location Affinity + +Can the pods comprising a single distributed application be +partitioned across more than one cluster? More generally, how far +apart, in network terms, can a given client and server within a +distributed application reasonably be? A server need not necessarily +be a pod, but could instead be a persistent disk housing data, or some +other stateful network service. What is tolerable is typically +application-dependent, primarily influenced by network bandwidth +consumption, latency requirements and cost sensitivity. + +For simplicity, lets assume that all Kubernetes distributed +applications fall into one of 3 categories with respect to relative +location affinity: + +1. **"Strictly Coupled"**: Those applications that strictly cannot be + partitioned between clusters. They simply fail if they are + partitioned. When scheduled, all pods _must_ be scheduled to the + same cluster. To move them, we need to shut the whole distributed + application down (all pods) in one cluster, possibly move some + data, and then bring the up all of the pods in another cluster. To + avoid downtime, we might bring up the replacement cluster and + divert traffic there before turning down the original, but the + principle is much the same. In some cases moving the data might be + prohibitively expensive or time-consuming, in which case these + applications may be effectively _immovable_. +1. **"Strictly Decoupled"**: Those applications that can be + indefinitely partitioned across more than one cluster, to no + disadvantage. An embarrassingly parallel YouTube porn detector, + where each pod repeatedly dequeues a video URL from a remote work + queue, downloads and chews on the video for a few hours, and + arrives at a binary verdict, might be one such example. The pods + derive no benefit from being close to each other, or anything else + (other than the source of YouTube videos, which is assumed to be + equally remote from all clusters in this example). Each pod can be + scheduled independently, in any cluster, and moved at any time. +1. **"Preferentially Coupled"**: Somewhere between Coupled and Decoupled. These applications prefer to have all of their pods located in the same cluster (e.g. for failure correlation, network latency or bandwidth cost reasons), but can tolerate being partitioned for "short" periods of time (for example while migrating the application from one cluster to another). Most small to medium sized LAMP stacks with not-very-strict latency goals probably fall into this category (provided that they use sane service discovery and reconnect-on-fail, which they need to do anyway to run effectively, even in a single Kubernetes cluster). + +And then there's what I'll call _absolute_ location affinity. Some +applications are required to run in bounded geographical or network +topology locations. The reasons for this are typically +political/legislative (data privacy laws etc), or driven by network +proximity to consumers (or data providers) of the application ("most +of our users are in Western Europe, U.S. West Coast" etc). + +**Proposal:** First tackle Strictly Decoupled applications (which can + be trivially scheduled, partitioned or moved, one pod at a time). + Then tackle Preferentially Coupled applications (which must be + scheduled in totality in a single cluster, and can be moved, but + ultimately in total, and necessarily within some bounded time). + Leave strictly coupled applications to be manually moved between + clusters as required for the foreseeable future. + +## Cross-cluster service discovery + +I propose having pods use standard discovery methods used by external clients of Kubernetes applications (i.e. use DNS). DNS might resolve to a public endpoint in the local or a remote cluster. Other than Strictly Coupled applications, software should be largely oblivious of which of the two occurs. +_Aside:_ How do we avoid "tromboning" through an external VIP when DNS +resolves to a public IP on the local cluster? Strictly speaking this +would be an optimization, and probably only matters to high bandwidth, +low latency communications. We could potentially eliminate the +trombone with some kube-proxy magic if necessary. More detail to be +added here, but feel free to shoot down the basic DNS idea in the mean +time. + +## Cross-cluster Scheduling + +This is closely related to location affinity above, and also discussed +there. The basic idea is that some controller, logically outside of +the basic kubernetes control plane of the clusters in question, needs +to be able to: + +1. Receive "global" resource creation requests. +1. Make policy-based decisions as to which cluster(s) should be used + to fulfill each given resource request. In a simple case, the + request is just redirected to one cluster. In a more complex case, + the request is "demultiplexed" into multiple sub-requests, each to + a different cluster. Knowledge of the (albeit approximate) + available capacity in each cluster will be required by the + controller to sanely split the request. Similarly, knowledge of + the properties of the application (Location Affinity class -- + Strictly Coupled, Strictly Decoupled etc, privacy class etc) will + be required. +1. Multiplex the responses from the individual clusters into an + aggregate response. + +## Cross-cluster Migration + +Again this is closely related to location affinity discussed above, +and is in some sense an extension of Cross-cluster Scheduling. When +certain events occur, it becomes necessary or desirable for the +cluster federation system to proactively move distributed applications +(either in part or in whole) from one cluster to another. Examples of +such events include: + +1. A low capacity event in a cluster (or a cluster failure). +1. A change of scheduling policy ("we no longer use cloud provider X"). +1. A change of resource pricing ("cloud provider Y dropped their prices - lets migrate there"). + +Strictly Decoupled applications can be trivially moved, in part or in whole, one pod at a time, to one or more clusters. +For Preferentially Decoupled applications, the federation system must first locate a single cluster with sufficient capacity to accommodate the entire application, then reserve that capacity, and incrementally move the application, one (or more) resources at a time, over to the new cluster, within some bounded time period (and possibly within a predefined "maintenance" window). +Strictly Coupled applications (with the exception of those deemed +completely immovable) require the federation system to: + +1. start up an entire replica application in the destination cluster +1. copy persistent data to the new application instance +1. switch traffic across +1. tear down the original application instance It is proposed that +support for automated migration of Strictly Coupled applications be +deferred to a later date. + +## Other Requirements + +These are often left implicit by customers, but are worth calling out explicitly: + +1. Software failure isolation between Kubernetes clusters should be + retained as far as is practically possible. The federation system + should not materially increase the failure correlation across + clusters. For this reason the federation system should ideally be + completely independent of the Kubernetes cluster control software, + and look just like any other Kubernetes API client, with no special + treatment. If the federation system fails catastrophically, the + underlying Kubernetes clusters should remain independently usable. +1. Unified monitoring, alerting and auditing across federated Kubernetes clusters. +1. Unified authentication, authorization and quota management across + clusters (this is in direct conflict with failure isolation above, + so there are some tough trade-offs to be made here). + +## Proposed High-Level Architecture + +TBD: All very hand-wavey still, but some initial thoughts to get the conversation going... + +![image](federation-high-level-arch.png) + +## Ubernetes API + +This looks a lot like the existing Kubernetes API but is explicitly multi-cluster. + ++ Clusters become first class objects, which can be registered, listed, described, deregistered etc via the API. ++ Compute resources can be explicitly requested in specific clusters, or automatically scheduled to the "best" cluster by Ubernetes (by a pluggable Policy Engine). ++ There is a federated equivalent of a replication controller type, which is multicluster-aware, and delegates to cluster-specific replication controllers as required (e.g. a federated RC for n replicas might simply spawn multiple replication controllers in different clusters to do the hard work). ++ These federated replication controllers (and in fact all the + services comprising the Ubernetes Control Plane) have to run + somewhere. For high availability Ubernetes deployments, these + services may run in a dedicated Kubernetes cluster, not physically + co-located with any of the federated clusters. But for simpler + deployments, they may be run in one of the federated clusters (but + when that cluster goes down, Ubernetes is down, obviously). + +## Policy Engine and Migration/Replication Controllers + +The Policy Engine decides which parts of each application go into each +cluster at any point in time, and stores this desired state in the +Desired Federation State store (an etcd or +similar). Migration/Replication Controllers reconcile this against the +desired states stored in the underlying Kubernetes clusters (by +watching both, and creating or updating the underlying Replication +Controllers and related Services accordingly). + +## Authentication and Authorization + +This should ideally be delegated to some external auth system, shared +by the underlying clusters, to avoid duplication and inconsistency. +Either that, or we end up with multilevel auth. Local readonly +eventually consistent auth slaves in each cluster and in Ubernetes +could potentially cache auth, to mitigate an SPOF auth system. + +## Proposed Next Steps + +Identify concrete applications of each use case and configure a proof +of concept service that exercises the use case. For example, cluster +failure tolerance seems popular, so set up an apache frontend with +replicas in each of 3 availability zones with either an Amazon Elastic +Load Balancer or Google Cloud Load Balancer pointing at them? What +does the zookeeper config look like for N=3 across 3 AZs -- and how +does each replica find the other replicas and how do clients find +their primary zookeeper replica? And now how do I do a shared, highly +available redis database?