From 613a743a245eaeb953a79df71a9b78bb4e0d474a Mon Sep 17 00:00:00 2001 From: Ogbonna Basil <19818425+basilcea@users.noreply.github.com> Date: Tue, 8 Feb 2022 21:17:03 +0200 Subject: [PATCH] update image --- Data_ETL_DotNet/Infrastructure/Bigquery.cs | 14 + .../Infrastructure/Infrastructure.csproj | 5 + .../Debug/net5.0/Infrastructure.assets.cache | Bin 1203 -> 52530 bytes .../Infrastructure.csproj.nuget.dgspec.json | 20 + .../Infrastructure/obj/project.assets.json | 6163 ++++++++++++++++- .../Infrastructure/obj/project.nuget.cache | 111 +- logs/scheduler/latest | 2 +- template.yaml | 435 +- 8 files changed, 6381 insertions(+), 369 deletions(-) create mode 100644 Data_ETL_DotNet/Infrastructure/Bigquery.cs diff --git a/Data_ETL_DotNet/Infrastructure/Bigquery.cs b/Data_ETL_DotNet/Infrastructure/Bigquery.cs new file mode 100644 index 0000000..a499a15 --- /dev/null +++ b/Data_ETL_DotNet/Infrastructure/Bigquery.cs @@ -0,0 +1,14 @@ +using System; +using Google.Cloud.BigQuery.V2; +namespace Infrastructure +{ + public class BigQuery + { + private readonly BigQueryClient _bqClient; + private const string projectId = "river-yew-256601"; + + public BigQuery(){ + _bqClient = BigQueryClient.Create(projectId); + } + } +} \ No newline at end of file diff --git a/Data_ETL_DotNet/Infrastructure/Infrastructure.csproj b/Data_ETL_DotNet/Infrastructure/Infrastructure.csproj index 1b55ce7..b79efed 100644 --- a/Data_ETL_DotNet/Infrastructure/Infrastructure.csproj +++ b/Data_ETL_DotNet/Infrastructure/Infrastructure.csproj @@ -6,6 +6,11 @@ + + + + + diff --git a/Data_ETL_DotNet/Infrastructure/obj/Debug/net5.0/Infrastructure.assets.cache b/Data_ETL_DotNet/Infrastructure/obj/Debug/net5.0/Infrastructure.assets.cache index f56cd74c8a0a56fdaf4a83ebf33441bbaf0e4593..f727dc4c676004e4fa1cc3841dfee0a96aae40bb 100644 GIT binary patch literal 52530 zcmcg#2Y_6~c?N?dgb*M^5nc4Ex4P4*(2-8HP|K`oids`IQ!`}O6=KsI{pWYAL zyKC#|C!KWC=k|Z|t>+#8>bKwj-`{w0=d-uH>`PC5s7LK#)am z|DjUQ9Sp)UidYiQ#TK4Ry0u2D+U@|xNTFmHM_?)aQYHN|{8BEw9B#azHK-hH%*}dj zDQ+#;Q(brk9ErOUZnmhns65{}Qc&hr1dR^V%bn?FYZis?Sgle`A-M{YC6cT0{o)gb zL^MXW;AuEQ@S$MtZ< zjS%PmVp+h>^fxHMoDXiqcerU#RY0ZQoS%xM6^4CvvC@AujsXMQ7u)U7ur$C;3dha( zB?GMM4#!lvQ${6RiH;5BmcRtZc8+mqNaBbT4lafzd_PnqQ`DP{nbv%xQESYg9q_bD z$`X4Jc036yWv^GV*#<-SE>s1?&ClfgG*llF{V}Q$l*G8bK?o^sSc%(!Uo!a!+`h1? zRD;K5e>fT$@>FvaCNA`%sBKiDHYrh?i;0T!b&e=ILvOJ}-Gc8!`C5@EYS58(LCs=u z)Q$`7ZnZ?-hVQYU$a-^T1RZ7wTOV!Vz8v?a(5ES>08+~BO3EGhrR2I3ZayewwpM91 z+s)}tG+C)f<@REuQfqXot?4qVU{qS`k%F!2bh_(4RNAYfZ9}xJ6m3jLlZft8i0;NO zA=(P}U=UHGdc4zY#E>3tHyfoPloNDa9pi_6CgeQ`DaCGs7-}#^;YjRuxR(UQwinx- z>MW9XuF`}i3!TzPMA_F;ekj^hnn;3fotr}sl7dRfJ0OW9?}Q`CyWqloG$pyxoT^SW zXCan42bNeGE-A622)DgCD`oD643bI1GLm@@T)0BgGVAp!MnSMQdnb=z4mCCif?YN*(F0u`7c)Tx%|Ea-+PP5uQh!_H*y9U+b<=()ROd2QB7;jXu4Lf zCOOq7RM%4wx)Y%%l1f@M!7sDqm+>8*Sx{voB|n;L)n;on%amVbrlB0r5v5foEmReJ zhexkZ|d%XHFSqnh}-;+fVw5k~2wmVc3&}xU8Z2~Z^LQ4ZQ zmH0XQQc^quSE%bkNzrarj$%sLm+N$uK*A@Iuu^tQ$!6oX@m;7nAlb*OlSiuMscNgz z87tqQ|7Vaz1nDYI$4-m6DHH);Nyu@XWU= z6h?AXMa}3FKg6wq5WfTAC58nH!y>+4eR7f9Xn}}o^|(VBY#eo$+D0_CC+=3pktMuG zEqIUN`<1<*tJh|ZbdFc)zdmz$!R;||CAP;cY%j-mp>{-CZlP0cwAE<9*$sE1(y=6- zS13H3kzd&Zp8N{Vg~wgby-MNX!v1P}Kh|a1br$EU815MxIs3=1a&{0gcj%bXPEROY z+)I8E--Sjim`Li>W~mQ~8J%pUV^AdFq|DbSnXkn!Rl2W(E7blenNi9J&S7Ilyj%O% zDynoT%T!zdIZbzp7|c#Ri9u6qmsc^ z%|Kf784Kn+@%`FDiSCA3ZD2NxX30GuwJqHscc|zR;E)v`g(w*SXSuo#? z?`Qk4BLEAG;Rj>)5-nFo!`M_kEo0x-J0%;7To>>r{q7pNstdvgx?ohkgVcZNu-)1-)Yk`Vn82R-(r6+>4<+5kfI6((Yg=WOvE zFQif+0W!6Rfc#PsSlSxJC9~*GG18u5C7e9ZQ8C3LHIdI!WL`-o+VXfy4=whZF9xtO z_M@)|mL>mHSmtX@#hchr0H&~?jHt`hQYZa=X>jS0LR@S@?AviL=FWRdV&h$y^*>l{ zVfSL}dy(-yTx)C?7Sm?Le2wkRd)UbHENexQXuDO8H zo_PuD?PO%FBh`AexocR0)gERwKZ$Ycd^Cg>zpqS~VTk>XgJ#%6N(hx#sj_Z%4^aL; zQ&H98K9SDRB%WW6>!(YzWb7AhV6fLh6HO5>afBF~ot^KLCzE31VquojiG_g^uXGb> zjp8boG~?xG^1RCBsn@BKMNbNMkt3X1IVNugCS{nFTncfSgMnCBr64XD6a>Rb@;+B_ zNG*tx2eVFjM5E<;l{T!$w!hpJH!?F@#Adk)Ge(_0EiJX|Mjp&6YDDV<9*G+JCIts$;gjIz|4RHFTsXo~s})*2?m-fN_2nq2hT4;*b@dggVe{#&&ZE zr|AKT%53d1JNa=m@)JX8@;Vvr(%|d?g`$77b#~ClvzEr36BM+=>=Za!J6Zzw3Aj(f zeG2Z=aG!zuUbu1)ZCM`?>;2dkqoQAlY8k#=sJ;*M@O)TBpUzVFdOps@*iKW}mgAQ! zb~>C3+xLTh!j5?{m=zF0hB^a|z^sIGf%z=x;SG$&P=h%B#rV#IT#4^2556A&J-m6# zhi?$a!Wiw@3hgTVk~PnPb6N8TK@U%X&J6Il7^7PasbtG@;RxM%a4vM813kRtEYNj% zjEu3I4;h5z0yx652F``$^Pq>TjWm|tj+Zg23n5LSy2yj-3!sNLQ#eo!;Mf`Cyx78d zi3jHwLGQ|XSo7%qI2xn6RH3>Izm%qz!@1J*hd?iMMg?apx;UoBFxG-SCE^uu1mj9L z7mObUJ-o}oBw~9Y$JZF)Rgf$ZUhP5nCD4nX=!y^76g$zyXKj3duJHi+5zxci$EC^S zL9gDAyD`RVA)AbL9ULX=^>D6a{ZY`18?A!L=7OgZ68?7K`wHmcEuhMNeH^D_ zOzRaUw%L#e(~pB*s0zc`iB7ZdF*}AatYB=wFD2awoXa#n0s4TC-!X)ZU`?sB363Dp z7{UeNCqXX&AueY{&dG@awgs#ugj+liz6$yYI01yWt4wPs6#8{tW0R@Yo;2*#%*?Y(2r^H|+|sjO9HJ$cT185+T|NM~Hq7&V}gbK@a!A z+sW?1^f5sTj{BG(W7!QEgk=nlu>1m?3(GHp9?n)8i(Z<*B^SH~p*C$<&^#vzM|CvX zEx<^B2?z-39ymh!bvPH&Uj{wgFLxlV@mdJ_0c5ZYP`#`Ikp2ef#dh4wGjZZ?J{adX z-7d0#T;VZ)o6~We5q=YJWJdTc5A5Fty^w(g(kOGb&~IRau^pKeDQVyb6jCTsnBu(`#lfZ-v_ca0jA53XN^i)tfR1U`u?E@>mPw0o?%q!+cKwc#IYGaMzIFP$7a}$3kn;j z<{x{o{R!yB3bzH$b)2R+Qn@-OP5P(>`%gWv{|xj29kVG8S}YYx%#T}`|J;N5FF+5E zoU_?rRxK@}U2@i*<^+MsI=mbyG5)2($g`4v<-z#Zpbz8-Ovb9_^oK#3Uig%FpHO&r z;+NW{zkzeLO@9k|covmys#Yx{N5U(iykIHO{T-yrCg|@y=)MK|Ko2@(tg82D zGhE*qs)YYG3;sWN;QwQv@aOdD=EBxa!*@4SPh0T+$pinNK`&J8w)t2?C0-}a1*=y@ zN8|9)wBNR{{fh_Nzk>c`AA9(lv`ejB_d(UotAQ%lxz#$d1paRp@PGFJ{}0fMWocto zDY}b1;l}RH6VF-j|I-8izd#R9_^SFsqX*z*<=ba#Jv7jG0Wpap4gB90!vFCg{9n)u zb!=z^iVd-zqY9atLT3rjlDfqDyu!-0FRe4XYTuJV@979b2A%G9nRRtB_WPh^yBB!y zoC5mGgA2?mbvyc?M0~U%eRz=$7-RtIR5k#UQx0aUm$cxiKte6VI2^TKOFW2|f?lZo z5{TtNhzw4LArr(B-G`Mu8Qn4uy3;^!b!pES-5hp2puEr))vPJK5NV%;gt;8z$%v=p zmyEc=1M>{f`#MA6x1zltL&E#0vf>J{nK!tfg}2AxO3sogF7ECXDu@aDK^>%aMCt>KgtUr zooYcFCD94x8aP6EAzb*n48J|G8=E#y=U|*#gpqy5ya?i5Fo{kuFNPzSm%xQ5JpC|_ z*BTP0o!(?eRNp>?mqM}&A<+rpWpIS>a=5+_ZrQ|!%plydDTHt>B)bq2oe*9DM+mQk zdugzOL_(zxuV5s%B^oZ}lpu~{a^qDHMP=p;II3-~#6Ho$QC7UJ9hv-KsHoKziY7oHnD2#HPzWgQu=%5xD~RVBhF zr|;JdJ(L^<5*4Kl^F|L$q7zJ6QHJ~BJ;2n4G!?mx?*Lg4!-NqQc39VaD&HpGFO zP*s1SB;M%3M|2zCAgVr-#6_yUO&)Yax6uuv>NDtyR(+d22#IbZ97NS;5aw2WTRhN+ zZbKVH)hD3kRDHL2kP+QR7G9S(d#Oc?84kjc8SaPsLc!b+y2&GG0&HoG z-#Cb`?c_Eqbumac%HtXfymFSBe>ns#Nc%l-R1+S?w@Z7XllBk5k@hcy`_hTl-qQ~B ztUosdDGNLZO=SAN2;VLX5Z$)GXNy_DfA>l-@jdDrZr!9k>uwss7h8(I#G^RTN%5D$ zQPMvI_vK=W=iL<3i~2Ut{*3T4C?iw9jBi)!6P=7O2}ee#z;$T^-|aU27=d~AXM`yz zBaKkSx624bCnHS5kr8I#x;BFQMxOqRknrr!2uGldG(rvEE+Y_~jPNiV8R010D}udw zrJLN%jFU(5)fTSV<3-zSk6!O3lRKN$RXlP)atIYTsldBK>VQTn*nN7UlM1tNq(TGk z@d2rT%6V`KGy%+|0MSW-IXF__5x7Aq;2cx+ngT5Vb16V{QlJe-3ec#k*d4qw7dc+H z;hf|f*G?qkC7F@Ll}s`-J*YAhZ$;?D#Q*`KzPXEFevJn+(QQM2qEJ=Nv*Xr7Bz()T z+OmbkgmOaNL4FcS3Z_ zX%zcW?oA%$h)&AM#`44@oHKh4B;nrd(T(V&+gso$;V4t~bLOXFUTxvr*i(w(%1v3P zycMwI80BsFcI88&+e!6?KJuYMonF!?N1L~Mv?2OH6Q`X#n2Fbo!-=E2Kt=#ov`a@AK$GbkaxG z{Dm&4Fb6xE`ljmS=w#sr*&InVmtD&=c|UMbn($%3=RGuzKK6F>c!7BR|u zObei}da)^}m&G-Cg^QSr2d?ns$?R;ifdxyP`C2j@ZN%|o2h&~2t;+VZUBt|Kwrk0T z2n*kAEt%ee2cGw$sIKR|Txo8_+(pM#5k2?{*oLqcgdpZa|KSTtZ9Md981%um)B>m| zFrku9j4e%WmAx{EC4TcS^{yD*GP@-wPJ9zBKgWfnjbF?q`PUVRh;Skkq+U5&A#R)v zXN6QW5}i_ljGBjq4WE7^ZCh>TzWHivF*-Di9O<~hhYF9%6PYL12uGTocyi5E3R8?k zSYqUOp6v#|dBr1c6xn{nZ7H6YM#XrUVt|syO2nlPcb!r!^UT}QLa1o{U3EtdF)ZcrMp;6P*e`fx~agq0YiB5hJ(~E8P?n zxF*wn5N;{n8N)pZ*E%$?o{l3}{5%f!*rCp)D?paSmr!1BLGfNfX`XSKs%~%2A~x!T zYpmFmE7cOddVPvy^P@iEWUIlC)|HfooK)46_?79YcmnG zGIHsapZduzY_1P8$Sxzg-wC@p%*I%sOzC=1Y-wCA4BZ&!BbV8@rXrmtnw6vItq1BT z5sDyiF(Ihgq`HLgH7_h)I!G z(r(Rg)UALeJN394u`lqr+AYPoTlXw-@sc3)-luHwa)Q?339W;n$%Vxjn()Ms9|02x z(^ngA&M5Cn>~tJr%+1<(GSL><0gMlY9JYZZ>7a=B)b+2@rl*O~YxFtO4(FhLQi zcjK}-P**T4RD~oaT&&p3T0iC%t&$8%hcGBreFaYh;@SO^bh|SNkt;{Z7w_7UXnuc< z(WNggG;i14nGu*Vk}qn$7?TUT7Gt7HG47=uvkaUSqZ^~Vs3fM?J9xMIP39IN7XUb2 z+#!wDSxt1h!Rj zG0nWfcU?wTZCEqOhlpIM7-D>%UZ7I3LXvcTHcxoP_s?O!PHJQ(dj6T9ZY=aLUlz+S zS4x5lg_Kw_SH}6VWP%DTn7%VN=HwL%h>5nVSk@sztdbG8@*r>!^f$1`is!*z~c1(L*Z;A3wZ}1Sr%#RRD*Bq{% ziHPSx;_Q?q@hDtQV<6M2m$hvJh+4)Sfu!&=qy7-wJ(VH!QxlkZ!t9JaB}~tdX!K@k zMaLq++(~t^r2qxhNl(1E)Jccnp@*DAFn=fSJKUN{UtPPIpzX-~ZJveT2H*I(!v(aH zQPWc21$KzcgBbUnz{Y?M$Kd#+t%dVA#0-y8jQ3=B?r^cDz0H`B?fPNC*_ zcC^VeVGgfAO536BHv%}N^TORy9IG|byq1R?mP%GIhkAVH{!m7?8DAKt9GY?h_sWD; z(tC4;p&6_hhS`xLKO`;KW&?5ei3c(IuKSEoT^ut(p#r~q?DD%a!%BC3Ie2LNa}%eC z8hVDpW<_OCr=zEfWLQ)>EO%Z_?>)W=K2S=mkd*SAjT~N_dd5rMp{_Ot-Ig(}?yv{r zk?iZ0U2_8ljOvb?HWVARNLZe+&|$$Uxkr6{MpNC~WdoCX+-|#lv$do)TwtWbwPYZO zy+|l(Ni$#oJUx_)2P?H?TPwFUCZv7AxV?}nT77>cG6#>l!=daYQYZR^1u^APFPJ7{ zuYe!j@SRNOt{={%sV=QHN4^)Fr|*6K0R>zmEmpwg7bwY&;TD%4 zHth@23`4lkStt15RyerbPix-lH@6~|&zw_?WAg?o|fHBr0b=Mr% zgi%sKL`BrtqlKneag`WkvCqD>@3Jtnl_?P!9h^7N1p%mDd{&1(?w-r)kYc8Fh^c!y z9h&r*ROlMHB6=Lq=F70_Ej_NxXSgaODytJ;-_}e1a_GeD2BwqE(CEZIyL38*E_e&3 zQG6vE+ooMYqm7q&@PFB#3NlcoCFYY{?Pju?liSJ^RTq~OwU;E>Z*oPiN~nUDsIilc zYwkPi1r{sVE*U<(Ws!@uf>}ZI$Opg zk8BS`PwUCKa<%=DI#H%N_cB-#``oD|p|a>+j_F8wqMrh~k>TvB<>_K%c%qML9eh1T zmF&`3VlO5cY@6oc6ryKK0xWpKB)#dY^P~0h^Cn{;JhA;XN#o2k!_@5dW zKGXGf+O8)~J*6f0uYhShN4=a(>e~KypJwS)vj>j6;G|PqueU{qlY`wnIP#U-tZg$a lD1}VEb3!l3g_mp0>W?|?Y}#FS%4rd<$OQ4Ouyu3e{~w~|G7KOSb3Ru{b}EL@r>-M;75w>l@w9YQj_H34PJ zKt)ghV){>h$SpfD?9AlJ(g~9%F^UMnBw;k`{D)@kyABY7eE=--g*GxtYn= 1.8.0" + "Apache.NMS.ActiveMQ >= 1.8.0", + "Confluent.Kafka >= 1.8.2", + "Confluent.SchemaRegistry >= 1.8.2", + "Confluent.SchemaRegistry.Serdes >= 1.3.0", + "Google.Cloud.BigQuery.V2 >= 2.3.0", + "Google.Cloud.Dataproc.V1 >= 3.4.0" ] }, "packageFolders": { @@ -254,6 +6185,26 @@ "Apache.NMS.ActiveMQ": { "target": "Package", "version": "[1.8.0, )" + }, + "Confluent.Kafka": { + "target": "Package", + "version": "[1.8.2, )" + }, + "Confluent.SchemaRegistry": { + "target": "Package", + "version": "[1.8.2, )" + }, + "Confluent.SchemaRegistry.Serdes": { + "target": "Package", + "version": "[1.3.0, )" + }, + "Google.Cloud.BigQuery.V2": { + "target": "Package", + "version": "[2.3.0, )" + }, + "Google.Cloud.Dataproc.V1": { + "target": "Package", + "version": "[3.4.0, )" } }, "imports": [ diff --git a/Data_ETL_DotNet/Infrastructure/obj/project.nuget.cache b/Data_ETL_DotNet/Infrastructure/obj/project.nuget.cache index c4ae016..ec9b709 100644 --- a/Data_ETL_DotNet/Infrastructure/obj/project.nuget.cache +++ b/Data_ETL_DotNet/Infrastructure/obj/project.nuget.cache @@ -1,14 +1,121 @@ { "version": 2, - "dgSpecHash": "D5yOjh3g0GuCCUs9n3fYfaZMWemfYQ0K9WyE6XEmIIFap0QR/E5LiDBR6822mEQgDJPcb9f1/t5DGXN41ecNJQ==", + "dgSpecHash": "J3+hPnY+IzixhqVHMslt87j+zKBJK/647nb9L0LHBZEKlMYbbxqIseyFhvJrdwjBwMLbo84qlCFLZY6dvoYurA==", "success": true, "projectFilePath": "/Users/mrcea/Desktop/Udacity/Data_ETL_DotNet/Infrastructure/Infrastructure.csproj", "expectedPackageFiles": [ "/Users/mrcea/.nuget/packages/apache.nms/1.8.0/apache.nms.1.8.0.nupkg.sha512", "/Users/mrcea/.nuget/packages/apache.nms.activemq/1.8.0/apache.nms.activemq.1.8.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/confluent.apache.avro/1.7.7.7/confluent.apache.avro.1.7.7.7.nupkg.sha512", + "/Users/mrcea/.nuget/packages/confluent.kafka/1.8.2/confluent.kafka.1.8.2.nupkg.sha512", + "/Users/mrcea/.nuget/packages/confluent.schemaregistry/1.8.2/confluent.schemaregistry.1.8.2.nupkg.sha512", + "/Users/mrcea/.nuget/packages/confluent.schemaregistry.serdes/1.3.0/confluent.schemaregistry.serdes.1.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/google.api.commonprotos/2.5.0/google.api.commonprotos.2.5.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/google.api.gax/3.6.0/google.api.gax.3.6.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/google.api.gax.grpc/3.6.0/google.api.gax.grpc.3.6.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/google.api.gax.grpc.grpccore/3.6.0/google.api.gax.grpc.grpccore.3.6.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/google.api.gax.rest/3.4.0/google.api.gax.rest.3.4.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/google.apis/1.55.0/google.apis.1.55.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/google.apis.auth/1.55.0/google.apis.auth.1.55.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/google.apis.bigquery.v2/1.54.0.2397/google.apis.bigquery.v2.1.54.0.2397.nupkg.sha512", + "/Users/mrcea/.nuget/packages/google.apis.core/1.55.0/google.apis.core.1.55.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/google.cloud.bigquery.v2/2.3.0/google.cloud.bigquery.v2.2.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/google.cloud.dataproc.v1/3.4.0/google.cloud.dataproc.v1.3.4.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/google.longrunning/2.3.0/google.longrunning.2.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/google.protobuf/3.18.0/google.protobuf.3.18.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/grpc.auth/2.41.0/grpc.auth.2.41.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/grpc.core/2.41.0/grpc.core.2.41.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/grpc.core.api/2.41.0/grpc.core.api.2.41.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/librdkafka.redist/1.8.2/librdkafka.redist.1.8.2.nupkg.sha512", + "/Users/mrcea/.nuget/packages/log4net/2.0.8/log4net.2.0.8.nupkg.sha512", + "/Users/mrcea/.nuget/packages/microsoft.bcl.asyncinterfaces/1.0.0/microsoft.bcl.asyncinterfaces.1.0.0.nupkg.sha512", "/Users/mrcea/.nuget/packages/microsoft.netcore.platforms/1.1.1/microsoft.netcore.platforms.1.1.1.nupkg.sha512", "/Users/mrcea/.nuget/packages/microsoft.netcore.targets/1.1.3/microsoft.netcore.targets.1.1.3.nupkg.sha512", - "/Users/mrcea/.nuget/packages/system.runtime/4.3.1/system.runtime.4.3.1.nupkg.sha512" + "/Users/mrcea/.nuget/packages/microsoft.win32.primitives/4.3.0/microsoft.win32.primitives.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/microsoft.win32.registry/4.0.0/microsoft.win32.registry.4.0.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/newtonsoft.json/13.0.1/newtonsoft.json.13.0.1.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.debian.8-x64.runtime.native.system.security.cryptography.openssl/4.3.2/runtime.debian.8-x64.runtime.native.system.security.cryptography.openssl.4.3.2.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.fedora.23-x64.runtime.native.system.security.cryptography.openssl/4.3.2/runtime.fedora.23-x64.runtime.native.system.security.cryptography.openssl.4.3.2.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.fedora.24-x64.runtime.native.system.security.cryptography.openssl/4.3.2/runtime.fedora.24-x64.runtime.native.system.security.cryptography.openssl.4.3.2.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.native.system/4.3.0/runtime.native.system.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.native.system.net.http/4.3.0/runtime.native.system.net.http.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.native.system.security.cryptography.apple/4.3.0/runtime.native.system.security.cryptography.apple.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.native.system.security.cryptography.openssl/4.3.2/runtime.native.system.security.cryptography.openssl.4.3.2.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.opensuse.13.2-x64.runtime.native.system.security.cryptography.openssl/4.3.2/runtime.opensuse.13.2-x64.runtime.native.system.security.cryptography.openssl.4.3.2.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.opensuse.42.1-x64.runtime.native.system.security.cryptography.openssl/4.3.2/runtime.opensuse.42.1-x64.runtime.native.system.security.cryptography.openssl.4.3.2.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.osx.10.10-x64.runtime.native.system.security.cryptography.apple/4.3.0/runtime.osx.10.10-x64.runtime.native.system.security.cryptography.apple.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.osx.10.10-x64.runtime.native.system.security.cryptography.openssl/4.3.2/runtime.osx.10.10-x64.runtime.native.system.security.cryptography.openssl.4.3.2.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.rhel.7-x64.runtime.native.system.security.cryptography.openssl/4.3.2/runtime.rhel.7-x64.runtime.native.system.security.cryptography.openssl.4.3.2.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.ubuntu.14.04-x64.runtime.native.system.security.cryptography.openssl/4.3.2/runtime.ubuntu.14.04-x64.runtime.native.system.security.cryptography.openssl.4.3.2.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.ubuntu.16.04-x64.runtime.native.system.security.cryptography.openssl/4.3.2/runtime.ubuntu.16.04-x64.runtime.native.system.security.cryptography.openssl.4.3.2.nupkg.sha512", + "/Users/mrcea/.nuget/packages/runtime.ubuntu.16.10-x64.runtime.native.system.security.cryptography.openssl/4.3.2/runtime.ubuntu.16.10-x64.runtime.native.system.security.cryptography.openssl.4.3.2.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.appcontext/4.1.0/system.appcontext.4.1.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.codedom/4.4.0/system.codedom.4.4.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.collections/4.3.0/system.collections.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.collections.concurrent/4.3.0/system.collections.concurrent.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.collections.immutable/1.2.0/system.collections.immutable.1.2.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.collections.nongeneric/4.0.1/system.collections.nongeneric.4.0.1.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.console/4.0.0/system.console.4.0.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.diagnostics.debug/4.3.0/system.diagnostics.debug.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.diagnostics.diagnosticsource/4.3.0/system.diagnostics.diagnosticsource.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.diagnostics.process/4.1.0/system.diagnostics.process.4.1.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.diagnostics.stacktrace/4.0.1/system.diagnostics.stacktrace.4.0.1.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.diagnostics.tracesource/4.0.0/system.diagnostics.tracesource.4.0.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.diagnostics.tracing/4.3.0/system.diagnostics.tracing.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.globalization/4.3.0/system.globalization.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.globalization.calendars/4.3.0/system.globalization.calendars.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.globalization.extensions/4.3.0/system.globalization.extensions.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.io/4.3.0/system.io.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.io.filesystem/4.3.0/system.io.filesystem.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.io.filesystem.primitives/4.3.0/system.io.filesystem.primitives.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.io.filesystem.watcher/4.0.0/system.io.filesystem.watcher.4.0.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.linq/4.3.0/system.linq.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.memory/4.5.3/system.memory.4.5.3.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.net.http/4.3.4/system.net.http.4.3.4.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.net.nameresolution/4.3.0/system.net.nameresolution.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.net.primitives/4.3.0/system.net.primitives.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.net.requests/4.0.11/system.net.requests.4.0.11.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.net.sockets/4.3.0/system.net.sockets.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.net.webheadercollection/4.0.1/system.net.webheadercollection.4.0.1.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.reflection/4.3.0/system.reflection.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.reflection.emit/4.3.0/system.reflection.emit.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.reflection.emit.ilgeneration/4.3.0/system.reflection.emit.ilgeneration.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.reflection.emit.lightweight/4.3.0/system.reflection.emit.lightweight.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.reflection.extensions/4.0.1/system.reflection.extensions.4.0.1.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.reflection.metadata/1.3.0/system.reflection.metadata.1.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.reflection.primitives/4.3.0/system.reflection.primitives.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.reflection.typeextensions/4.1.0/system.reflection.typeextensions.4.1.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.resources.resourcemanager/4.3.0/system.resources.resourcemanager.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.runtime/4.3.1/system.runtime.4.3.1.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.runtime.extensions/4.3.0/system.runtime.extensions.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.runtime.handles/4.3.0/system.runtime.handles.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.runtime.interopservices/4.3.0/system.runtime.interopservices.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.runtime.interopservices.runtimeinformation/4.0.0/system.runtime.interopservices.runtimeinformation.4.0.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.runtime.numerics/4.3.0/system.runtime.numerics.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.runtime.serialization.formatters/4.3.0/system.runtime.serialization.formatters.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.runtime.serialization.primitives/4.3.0/system.runtime.serialization.primitives.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.security.claims/4.3.0/system.security.claims.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.security.cryptography.algorithms/4.3.0/system.security.cryptography.algorithms.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.security.cryptography.cng/4.3.0/system.security.cryptography.cng.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.security.cryptography.csp/4.3.0/system.security.cryptography.csp.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.security.cryptography.encoding/4.3.0/system.security.cryptography.encoding.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.security.cryptography.openssl/4.3.0/system.security.cryptography.openssl.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.security.cryptography.primitives/4.3.0/system.security.cryptography.primitives.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.security.cryptography.x509certificates/4.3.0/system.security.cryptography.x509certificates.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.security.principal/4.3.0/system.security.principal.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.security.principal.windows/4.3.0/system.security.principal.windows.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.text.encoding/4.3.0/system.text.encoding.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.text.encoding.extensions/4.0.11/system.text.encoding.extensions.4.0.11.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.text.regularexpressions/4.1.0/system.text.regularexpressions.4.1.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.threading/4.3.0/system.threading.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.threading.overlapped/4.0.1/system.threading.overlapped.4.0.1.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.threading.tasks/4.3.0/system.threading.tasks.4.3.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.threading.tasks.extensions/4.0.0/system.threading.tasks.extensions.4.0.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.threading.thread/4.0.0/system.threading.thread.4.0.0.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.threading.threadpool/4.0.10/system.threading.threadpool.4.0.10.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.threading.timer/4.0.1/system.threading.timer.4.0.1.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.xml.readerwriter/4.0.11/system.xml.readerwriter.4.0.11.nupkg.sha512", + "/Users/mrcea/.nuget/packages/system.xml.xmldocument/4.0.1/system.xml.xmldocument.4.0.1.nupkg.sha512" ], "logs": [] } \ No newline at end of file diff --git a/logs/scheduler/latest b/logs/scheduler/latest index eebaf45..6e52709 120000 --- a/logs/scheduler/latest +++ b/logs/scheduler/latest @@ -1 +1 @@ -/opt/airflow/logs/scheduler/2022-02-03 \ No newline at end of file +/opt/airflow/logs/scheduler/2022-02-08 \ No newline at end of file diff --git a/template.yaml b/template.yaml index d64dd55..8b1a553 100644 --- a/template.yaml +++ b/template.yaml @@ -1,116 +1,194 @@ version: '3.7' -x-airflow-common: - &airflow-common - # In order to add custom dependencies or upgrade provider packages you can use your extended image. - # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml - # and uncomment the "build" line below, Then run `docker-compose build` to build the images. - image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.2.3} - # build: . - environment: - &airflow-common-env - AIRFLOW__CORE__EXECUTOR: CeleryExecutor - AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow - AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow - AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 - AIRFLOW__CORE__FERNET_KEY: '' - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' - AIRFLOW__CORE__LOAD_EXAMPLES: 'true' - AIRFLOW__API__AUTH_BACKEND: 'airflow.api.auth.backend.basic_auth' - _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} - volumes: - - ./dags:/opt/airflow/dags - - ./logs:/opt/airflow/logs - - ./plugins:/opt/airflow/plugins - user: "${AIRFLOW_UID:-50000}:0" - depends_on: - &airflow-common-depends-on - redis: - condition: service_healthy - postgres: - condition: service_healthy - services: - zoo1: + zookeeper: image: confluentinc/cp-zookeeper:latest restart: always - hostname: zoo1 + hostname: zookeeper ports: - 2181:2181 environment: - ZOOKEEPER_SERVER_ID: 1 ZOOKEEPER_CLIENT_PORT: "2181" + ZOOKEEPER_TICK_TIME: 2000 - zoo2: - image: confluentinc/cp-zookeeper:latest - restart: always - hostname: zoo2 - ports: - - 2182:2181 - environment: - ZOOKEEPER_SERVER_ID: 2 - ZOOKEEPER_CLIENT_PORT: "2181" - zoo3: - image: confluentinc/cp-zookeeper:latest - restart: always - hostname: zoo3 - ports: - - 2183:2181 - environment: - ZOOKEEPER_SERVER_ID: 3 - ZOOKEEPER_CLIENT_PORT: "2181" + # zoo2: + # image: confluentinc/cp-zookeeper:latest + # restart: always + # hostname: zoo2 + # ports: + # - 2182:2181 + # environment: + # ZOOKEEPER_SERVER_ID: 2 + # ZOOKEEPER_CLIENT_PORT: "2181" + # zoo3: + # image: confluentinc/cp-zookeeper:latest + # restart: always + # hostname: zoo3 + # ports: + # - 2183:2181 + # environment: + # ZOOKEEPER_SERVER_ID: 3 + # ZOOKEEPER_CLIENT_PORT: "2181" - kafka1: + kafka: image: confluentinc/cp-kafka:latest ports: - "9092:9092" environment: KAFKA_BROKER_ID: 0 - KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2181,zoo3:2181" - KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092" - KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "3" - KAFKA_LOG_DIR: "" - KAFKA_NUM_PARTITION: 2 + KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1" + KAFKA_LOG_DIR: "~/kafka/logs/kafka-logs" + KAFKA_NUM_PARTITION: 1 depends_on: - - "zoo1" - - "zoo2" - - "zoo3" + - "zookeeper" + # - "zoo2" + # - "zoo3" - kafka2: - image: confluentinc/cp-kafka:latest + # kafka2: + # image: confluentinc/cp-kafka:latest + # ports: + # - "9093:9092" + # environment: + # KAFKA_BROKER_ID: 1 + # KAFKA_ZOOKEEPER_CONNECT: "zoo2:2181,zoo2:2181,zoo3:2181" + # KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka2:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092" + # KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" + # KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT" + # KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1" + # KAFKA_LOG_DIR: "~/kafka/logs/kafka-logs" + # KAFKA_NUM_PARTITION: 2 + # depends_on: + # - "zoo1" + # - "zoo2" + # - "zoo3" + + # kafka3: + # image: confluentinc/cp-kafka:latest + # ports: + # - "9094:9092" + # environment: + # KAFKA_BROKER_ID: 2 + # KAFKA_ZOOKEEPER_CONNECT: "zoo3:2181,zoo2:2181,zoo3:2181" + # KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka3:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092" + # KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" + # KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT" + # KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1" + # KAFKA_LOG_DIR: "~/kafka/logs/kafka-logs" + # KAFKA_NUM_PARTITION: 2 + # depends_on: + # - "zoo1" + # - "zoo2" + # - "zoo3" + + schema-registry: + image: confluentinc/cp-schema-registry:5.2.2 ports: - - "9093:9092" + - "8081:8081" environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: "zoo2:2181,zoo2:2181,zoo3:2181" - KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka2:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092" - KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "3" - KAFKA_NUM_PARTITION: 2 + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "PLAINTEXT://localhost:9092" + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181" + SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8081" + SCHEMA_REGISTRY_HOST_NAME: "schema-registry" + SCHEMA_REGISTRY_KAFKASTORE_TOPIC_REPLICATION_FACTOR: "1" depends_on: - - "zoo1" - - "zoo2" - - "zoo3" + - "kafka" + - "zookeeper" + # - "kafka2" + # - "kafka3" - kafka3: - image: confluentinc/cp-kafka:latest + rest-proxy: + image: confluentinc/cp-kafka-rest:5.2.2 + ports: + - "8082:8082" + environment: + KAFKA_REST_BOOTSTRAP_SERVERS: "PLAINTEXT://localhost:9092" + KAFKA_REST_LISTENERS: "http://0.0.0.0:8082/" + KAFKA_REST_HOST_NAME: "rest-proxy" + KAFKA_REST_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_REST_SCHEMA_REGISTRY_URL: "http://schema-registry:8081/" + depends_on: + - "kafka" + # - "kafka2" + # - "kafka3" + - "schema-registry" + + connect: + image: confluentinc/cp-kafka-connect:5.2.2 + ports: + - "8083:8083" + environment: + CONNECT_BOOTSTRAP_SERVERS: "PLAINTEXT://localhost:9092" + CONNECT_GROUP_ID: "connect" + CONNECT_REST_ADVERTISED_HOST_NAME: "connect" + CONNECT_PLUGIN_PATH: "~/kafka/plugins" + CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" + CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" + CONNECT_CONFIG_STORAGE_TOPIC: "connect-config" + CONNECT_OFFSET_STORAGE_TOPIC: "connect-offset" + CONNECT_STATUS_STORAGE_TOPIC: "connect-status" + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1" + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1" + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1" + depends_on: + - "kafka" + # - "kafka2" + # - "kafka3" + - "schema-registry" + + ksql: + image: confluentinc/cp-ksql-server:5.2.2 + ports: + - "8088:8088" + environment: + KSQL_BOOTSTRAP_SERVERS: "PLAINTEXT://localhost:9092" + KSQL_LISTENERS: "http://0.0.0.0:8088" + KSQL_KSQL_SERVICE_ID: "ksql_service_docker" + KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081/" + depends_on: + - "kafka" + # - "kafka2" + # - "kafka3" + - "schema-registry" + + connect-ui: + image: landoop/kafka-connect-ui:0.9.7 + ports: + - "8084:8084" + environment: + PORT: "8084" + PROXY: "true" + CONNECT_URL: "http://connect:8083" + depends_on: + - "connect" + + topics-ui: + image: landoop/kafka-topics-ui:0.9.4 ports: - - "9094:9092" + - "8085:8085" environment: - KAFKA_BROKER_ID: 2 - KAFKA_ZOOKEEPER_CONNECT: "zoo3:2181,zoo2:2181,zoo3:2181" - KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka3:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092" - KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "3" - KAFKA_NUM_PARTITION: 2 + PORT: "8085" + PROXY: "true" + KAFKA_REST_PROXY_URL: "http://rest-proxy:8082" depends_on: - - "zoo1" - - "zoo2" - - "zoo3" + - "rest-proxy" + + schema-registry-ui: + image: landoop/schema-registry-ui:0.9.5 + ports: + - "8086:8086" + environment: + PORT: "8086" + PROXY: "true" + SCHEMAREGISTRY_URL: "http://schema-registry:8081/" + depends_on: + - "schema-registry" spark: image: docker.io/bitnami/spark:3 @@ -124,8 +202,8 @@ services: - "9090:8080" - "7077:7077" volumes: - - ./apps:/opt/spark-apps - - ./data:/opt/spark-data + - bitnami-spark:/bitnami/spark + spark-worker: image: docker.io/bitnami/spark:3 depends_on: @@ -139,181 +217,18 @@ services: - SPARK_RPC_ENCRYPTION_ENABLED=no - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no - SPARK_SSL_ENABLED=no - volumes: - - ./apps:/opt/spark-apps - - ./data:/opt/spark-data - - airflow-webserver: - <<: *airflow-common - command: webserver - ports: - - 8080:8080 - healthcheck: - test: ["CMD", "curl", "--fail", "http://localhost:8080/health"] - interval: 10s - timeout: 10s - retries: 5 - restart: always - depends_on: - <<: *airflow-common-depends-on - airflow-init: - condition: service_completed_successfully - - airflow-scheduler: - <<: *airflow-common - command: scheduler - healthcheck: - test: ["CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"'] - interval: 10s - timeout: 10s - retries: 5 - restart: always - depends_on: - <<: *airflow-common-depends-on - airflow-init: - condition: service_completed_successfully - - airflow-worker: - <<: *airflow-common - command: celery worker - healthcheck: - test: - - "CMD-SHELL" - - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"' - interval: 10s - timeout: 10s - retries: 5 - environment: - <<: *airflow-common-env - # Required to handle warm shutdown of the celery workers properly - # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation - DUMB_INIT_SETSID: "0" - restart: always - depends_on: - <<: *airflow-common-depends-on - airflow-init: - condition: service_completed_successfully - - flower: - <<: *airflow-common - command: celery flower - ports: - - 5555:5555 - healthcheck: - test: ["CMD", "curl", "--fail", "http://localhost:5555/"] - interval: 10s - timeout: 10s - retries: 5 - restart: always - depends_on: - <<: *airflow-common-depends-on - airflow-init: - condition: service_completed_successfully - - airflow-init: - <<: *airflow-common - entrypoint: /bin/bash - # yamllint disable rule:line-length - command: - - -c - - | - function ver() { - printf "%04d%04d%04d%04d" $${1//./ } - } - airflow_version=$$(gosu airflow airflow version) - airflow_version_comparable=$$(ver $${airflow_version}) - min_airflow_version=2.2.0 - min_airflow_version_comparable=$$(ver $${min_airflow_version}) - if (( airflow_version_comparable < min_airflow_version_comparable )); then - echo - echo -e "\033[1;31mERROR!!!: Too old Airflow version $${airflow_version}!\e[0m" - echo "The minimum Airflow version supported: $${min_airflow_version}. Only use this or higher!" - echo - exit 1 - fi - if [[ -z "${AIRFLOW_UID}" ]]; then - echo - echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m" - echo "If you are on Linux, you SHOULD follow the instructions below to set " - echo "AIRFLOW_UID environment variable, otherwise files will be owned by root." - echo "For other operating systems you can get rid of the warning with manually created .env file:" - echo " See: https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#setting-the-right-airflow-user" - echo - fi - one_meg=1048576 - mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg)) - cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat) - disk_available=$$(df / | tail -1 | awk '{print $$4}') - warning_resources="false" - if (( mem_available < 4000 )) ; then - echo - echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m" - echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))" - echo - warning_resources="true" - fi - if (( cpus_available < 2 )); then - echo - echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m" - echo "At least 2 CPUs recommended. You have $${cpus_available}" - echo - warning_resources="true" - fi - if (( disk_available < one_meg * 10 )); then - echo - echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m" - echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))" - echo - warning_resources="true" - fi - if [[ $${warning_resources} == "true" ]]; then - echo - echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m" - echo "Please follow the instructions to increase amount of resources available:" - echo " https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#before-you-begin" - echo - fi - mkdir -p /sources/logs /sources/dags /sources/plugins - chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins} - exec /entrypoint airflow version - # yamllint enable rule:line-length - environment: - <<: *airflow-common-env - _AIRFLOW_DB_UPGRADE: 'true' - _AIRFLOW_WWW_USER_CREATE: 'true' - _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow} - _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow} - user: "0:0" - volumes: - - .:/sources - postgres: image: postgres:13 environment: - POSTGRES_USER: airflow - POSTGRES_PASSWORD: airflow - POSTGRES_DB: airflow - volumes: - - postgres-db-volume:/var/lib/postgresql/data - healthcheck: - test: ["CMD", "pg_isready", "-U", "airflow"] - interval: 5s - retries: 5 - restart: always - - redis: - image: redis:latest - expose: - - 6379 - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 5s - timeout: 30s - retries: 50 - restart: always - + - POSTGRES_USER=admin + - POSTGRES_PASSWORD=&Covport2022 + - POSTGRES_DB=railsdb + volumes: postgres-db-volume: + bitnami-spark: + + # always specify the version of an image in docker # kakfa internal is for communication between brokers # standalone