From 190456236a49ffc95aaffa26b6c24b5d6cdcbeab Mon Sep 17 00:00:00 2001 From: retoor Date: Sat, 23 Nov 2024 19:56:52 +0100 Subject: [PATCH] Update --- Makefile | 18 ++- README.md | 8 +- drstats.db | Bin 1339392 -> 1347584 bytes merge_images.py | 44 ++++++ setup.cfg | 2 +- src/drstats.egg-info/PKG-INFO | 8 +- src/drstats.egg-info/entry_points.txt | 2 +- src/drstats/__pycache__/db.cpython-312.pyc | Bin 11684 -> 11675 bytes .../__pycache__/statistics.cpython-312.pyc | Bin 17706 -> 16093 bytes src/drstats/__pycache__/sync.cpython-312.pyc | Bin 3732 -> 3619 bytes src/drstats/dataset.py | 42 ++++++ src/drstats/db.py | 131 ++++++++---------- src/drstats/duration.py | 5 +- src/drstats/sync.py | 42 +++--- 14 files changed, 191 insertions(+), 111 deletions(-) create mode 100644 merge_images.py create mode 100644 src/drstats/dataset.py diff --git a/Makefile b/Makefile index 76adc47..48686ca 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,28 @@ -all: build sync export_stats export_dataset +all: build sync_excempt export_dataset export_stats merge_images build: time pip install build time python -m build . time pip install -e . + + sync: - @echo "Synchronizing with devrant.com." time dr.sync + +sync_excempt: + @echo "Sync is not executed because it's a lengthy process ending with timeout error." + export_stats: + @echo "Make sure you have ran 'make sync' first. Results will be in ./export/" @echo "Exporting statisticts." time dr.stats_all + export_dataset: - @echo "Exporting dataset to be used for LLM embedding." + @echo "Make sure you have ran 'make sync' first." + @echo "Exporting dataset to be used for LLM embedding. Result will be ./export/0_dataset.txt" time dr.dataset > export/dataset.txt + +merge_images: + @echo "Merging images to one big image. Result will be ./export/1_graphs_compliation.png." + python merge_images.py diff --git a/README.md b/README.md index 03e9762..295cbbf 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ If you type `dr.` in terminal and press tab you'll see all available apps auto c ``` 1. `dr.sync` synchronizes all data from last two weeks from devrant. Only two weeks because it's rate limited. 2. `dr.dataset` exports all data to be used for LLM embedding., don't forget to execute `dr.sync` first. -3. `dr.rant_stats_all` exports all graphs to export folder, don't forget to execute `dr.sync` first. -4. dr.rant_stats_per_day` exports graphs to export folder. don't forget to execute `dr.sync` first. -5.dr.rant_stats_per_hour` exports graphs to export folder. don't forget to execute `dr.sync` first. -6. dr.rant_stats_per_weekday` exports graphs to export folder. don't forget to execute `dr.sync` first. +3. `dr.stats_all` exports all graphs to export folder, don't forget to execute `dr.sync` first. +4. `dr.rant_stats_per_day` exports graphs to export folder. don't forget to execute `dr.sync` first. +5. `dr.rant_stats_per_hour` exports graphs to export folder. don't forget to execute `dr.sync` first. +6. `dr.rant_stats_per_weekday` exports graphs to export folder. don't forget to execute `dr.sync` first. diff --git a/drstats.db b/drstats.db index b5c07567cef7bbbb940c89d9a9c9e93e13dd79e6..db8569d659c4100384295aaa35f8a83509ffd1e5 100644 GIT binary patch delta 10361 zcma)B34B!5xu5OMawo}TML zjHW7z(t5R3pKaAX^%ZJcJ+?1T(cgP5YTdOit(E$!wk~xm6mWb0@7%dFgWBKkg(Y*g z?|kQ5{@?$c*}V6x&9|-IKC!-juE*o;@wh+UE04aerQKIQ_ulol#A3svX9^fuTKzNp z@9o)9RlD6^{Xn%nr8d~bq*XP(2$Ll5v;G?Il993fS{s zr9ST&_PVt1)S%~+k+I(`wr@-O&J;c!8GG(h_CeY==V-5DZ|SSM+TMGeFTn1<#@Bv~ z_bpp~F0_H2f1fWkzBtTQJn8%MvF_Xt{)D-^5BP2p><KCNL=1h3Vzo*~nG3?1l1;T9I@qy;! z-JZ>71#r-F0zYuhzl}{iH}GP(w6)4M%YhrB<+hD>klm9F>^h?;f<19x;Di;$<@W9~ zgfO!M!mJKgVDE`St-nIR$0rF}+%exNLQ}Y6sle`DEc`lfSkFI~3awtZr?y8B*aORi zo!$zG*i&GKRtOLKDmrJaMd!-ZLT}lRs?9s+ohwLzFVB2uov^#fRcpQ}G}pO}zg#95 zzEa}~;dXbL-E||jIPzAZ<2ct85AP8k^SY+}{4rs@JMhfo!ZQ_nvbJZ0pSy=*8=k`n z%L=j97lhv&QEuDU9vc+c=zGE^CDFX+llO&az}@VN4+O#f_CJLGs^}2e1N($pUqz3= zx(^6fOmwYDi`e-M!5Ni?&Qj9SCSsxbRp58gk?ozs0)Fvwg^;mH5%S+iSPX=d=b2Y!%7ZTXbZv>yIuk5-1U@%$X z6`RjGbSOBX(pbFnC!YmBJ))>S8#gX=eAqn;_tUQ?hbA|=k~g0a66{MlLZihFhK?_e za7VkBLX-OCp@`u2Tzmm~cCHS^$^p^OR$LO=SJtARJ)aF}O|Am>4Ts)yUBH^I4rzXO zWdF6HM0Lqv8TRfip=p&tdFzitZ&bOex84&9vQKw~t|&r|!S+8Cx}?GT76X88aV^Di zZ-(9|ZjiShe>-GV$baUY(5$kW8Fu~OLxD2VW!Q|W@ZM4q+)xv~I96KR&n}-FzF=CZ zt+RoJVShQ5HCDYOJg+QVXD=@eFL2z<0NGbfEG^Bj3)S!jcOQFQe|THP*f%rb&1Icb zcIB1fKb7!@oW(x5HvCwnm3qGy_BK{Xc^!Eqk=I1G@yq)orsMNUIC%zbTr|pl3Qz(2$fL+@p2H5rq;#q`Z z<(~786t8b~dIniyzPNyr2g8G`d7-#Ngvw}aD0P1SZ1F`ZujtIPw^HIyPDVo>i(1To zhj`$LXIkbxaYAX@p}&h^@1$l=*gL~hv%KmL!e503|8((x z!ykBO&>zpVU(`Jq^2UZYZ5`eO#u94k>Iz>Gvfn=>j{Dw2(a?RP(cgQ8;neWx5_asz z;@`af;n6efrhVeG_;5OVd!N|h^*#p~PhpleJi3(aJ0N~QvwGNH4~Y|mW^tS+neqaj-hL_58r zZ3@viDsfb{RUukqe^ZD)ZZ8c+C)(*?^w_G=?%~lDp{6NQ!rNEcw*;f@F$q6w7B5?T zTCYeS?f!k@WWo8kFBp~Vx={2AKMkxDqes{~L(!TEG;fBpMmlZf*(-Wy%s$px@o(Yi zH2cADw99@X5*=@Mg`+F~zq2nC)vRK+IC9oSqDR_?3F{k2yPm#!<#H*mXS0fCna{Zd|lRn=XRwsE^`h+c2!=tP1CI09&7(B~<*&n?! zY7URC3N_82AO4QOZmN!+?emK4?O62cRZigK*}Lnbb16^}Yu#F0rh-ORFsQ+)-6a2$OkZdrQ?iia_3wTdP_qhtb%`f*Y$uRCI>M zsZ_S{d(i&vx2wjApV=pcs%@Ll00gV{_G245?d@-eX6!icrCOHwq-uc2^;d`P|E%hU zGUBYaTh6SG+R~%-$JyI&3H4mDDqPK0g=-#5eyw2p@9);35a>0}oL2;`=2(VZ{8-In zE@)BvKcA?%qo!i~qd(Tnb3u}ie^~Rq38it3{UsVx%Ih`uYEA4EXFcm~j{V35dRl5t zi+$`8P--8X6HvnKPyAkfh03MUrzlMK(-H)g(i)blsS4 zN?AFj#^sC@Ps@g7wo1Ke6%$O!)TMMjt7hh@vZZRNR%xNp*CM5KO_QZQLm9*bNK~}} z)l~b`jA{)@Io(oq&1?;(y7tVp7rf}Z?uK2lKxlU7oD(|R+t{8T#m0N>=&sm4w)e-e zqrIobN5&o(>OFxToXbDOzxBxSxz?r?1JX@tsv)NoDWMN)1G=S5l}t;vx`b5My<@h# zGj^NUCbf5Tww>77F_*oPY&<41wc2;Hr`o&5A9$+fSJlr{E%vU7*7-m1{f9U)5)clC zUJHH*v$nykr;Us~xsc2{?W<<(UZiC8jB2LynzUBQWME<`Ee|M?Mb_cE^=0IEKa7yi zq?Bw6nJcFlX*p*~gNhWFHE3$6{jj-|P~=Q21jI3yyaG#WR;x6m=M5=`k658vX^2)} zUOge1Y1L|>VX0r!2V4EAt~ckDtXb&joZa5pJ|7WKTLfSJYwM{aV^6C%&X#X|)tcSQ zRHe_5wYUuBG8yPHh<*7MmhNg))Fr7K+_k!B~nBx^&G zo^;MDr3^WjMiMR0W^?IuI@LYtCi~uFqN9`t;#|su4ie8ky(6~CCHKuS)!ut&Okm=_ z$GoBYcgGgjmtaC=&p#WJ%5bEz;g@263092ne=YXvDW!IXDU)lX4W+iqzG$m$U+5sK zVb9uBi(8XbJJZF!72zU)T{2i(D6T5?eQRUwSrvV>*zOyuJ-Wa?-JGiN~uCBa- z-E?*B3+^dN>Sx!~UQ#i<|JvGlWuC0EvAb)hc9+NP?ssd0w)sx&sfYKiIJ_BT)84E7 zbLG1CAFUJEjjeTc>BF>}V%Nz;4N<%dw(pj@>uQUF$N^2e>Q0!f_sFd@1Xbk>lQS#`MVo_9rby7 zK(H70Z0Q>r+uM)5WU|-<-`47~#k$w1iQcr*(|bmKNRrbn4#XwZjSe8G;GYT9XC*U~ z>i(m><&^Mu?aePW1SZV}&UDUc+j=}i^P;e!EZg3)y0J;^nGURsrxnyNH7PAXbpocw z^OmIJAHjPZfJabIv;jz}()_%MnhfumfaDZvR=!V?2V_e&TBRj zPp76}Pg&V``j`7YS=ktwROLC}6aHK1tOkBMkVjt$AMj?`k7Df;ZmAeG#w6 z{xBu(Y}slscvOq8>(%Qo5FzYwpCwu5-gMksZ5%w{7O zG+r7w()XT6^iA{lUliU5JXO^nn(vzy`E_`w_>-EAY|_tSCsrJzrd?`l>%`yo`BCP1 z8&UJ79k?TQfIaog*y--&n~s0`1F;K35k!_Z9Poq#6rG1cg=+`sNqU^;)|fX?9X>i7 zj$lgQXz#pv9UerBv`QvAp-ZckxIUm5L!HtReNb90=hTF>NQL7-_uMgjFfC3Omk&t` zRV%KerY%TX;E}SlTs7rPMn@qBpDOp%jFJ27}}#We^{9yye~m=QZ=xSssPlF>zV~i zV<%WA+gc1eTAj?3LrEnCPEOJB;5<{c^2KPt0^BzM^f_`=LP1<*RdOl@3b0k)aN?89 z4hENZKrC*6xM7lvX*%qI+mWH9Wqy6(f+d_#qdO2umsO2ii~T7U4A6Z?zLZ5)@iOMb zR56ZPz!kL|(x*W8B~k4BoP6vX9koOTqg(AR7qz9I4(PY2}RNrtH9S*gfUL1nWPg= zTDt2%QC^evGpgS-Ud6CIhA`j1KybWl$eN(8n@Ob`ubU_#*ZjdTk0vL`H5N#lk zgsbwJnU?#x^%ERNDF`UOoM9d*bQTnXH~|JA0lzi(l1V`Vd*hBgoo=p4Ktrpv2B$^} zQ3qSLYs5~{z@m#mgY%q_&ujuT&wi)RP2<}mX$L*rCGCp z$w*r%!4VgxcVa?98+(CrgC+h$i&F&AF<2%KuZBow|oeq`*Q zshIOZ*UR0%5RSx&rA10y$&nE|rRC*<-J=a4U^phNkyA1c zkJTtX2m+pVSO$Pc0`r%KHF9QvEVlrKuV8@zI5a^N$&TbmgM%coJc3v}@L(lgdfYAa zTuwJElwSnmhP)c@Cw$EFnhGG$u}o7_X)8-*fMCE?nXA=j==~_7Jg5la}?B1fQJh5?BEp?;WM$EwlkG@ z9<+sLB(gUNAm`))x|1iN9C}MUq(E&d^Q1D-XAOSBS&^7XBf7`HCGm`mipfs}4khEv zxxB@b%C!;hLPRQ%9m_zxkkeqx83Mc83 zM8IP|9xZ(e@Owzc8yIRnPQrl?h6Zgw)uQvtMm)`Zg6AcfAEW|Wfnyo?n*$2&d*T8j zGr;o{n2CYZL72j8AqjJ2ZQ=)DjQD|)=74(b6fq*+jI@-4T&sps&lh$}27Cnm^d3)vTp z8k<4AWJW;>5tinOtMS_I*a`1$U`dGc?2rj~Y?ap1OQIWfcsA4Uh6y3$GdvKJG3d<^ za}ZpE$hnpRypdEgmV{cY7)a!t4(7&d6D@L_uOwPXXaW|`@)W@18jMJ(Sc-PwJW0T2 z@G1hClPBWLy)cv;S3DGxI_?yMWJj2_fM`ZrG2%L>IFy8-MC4S&T4fyOsWO1WXcj1O zk_=0v$-Nq+GISY+Ij}yh149O#O9JhYz|)~aGj*vF`z!=O!NER9O?S{?9rTh5FPT(R z2vcw_n3-splLWxV6z2t|f*J|l0^Jyhpn0+WZ-P4e>h)&BIz7T*B zsd(ZAU+$-BSs)Vp`sN)Jy$)q@ioOG(AiWNT7PHue8ww3>9N2`mf+^xeXP{Cs^?Aua z%H>oFT|!X$fK$;O11|-n6ZI9aj+SvL`AwSuT{vw9#zx@t`v&2FLx-Ipr-JY739Qe7 zNYQe_MARFHX}~F^%EtqjUslSH77dFpA+g_-OS1V9>eliXT zbap@iF1YmgRLmXbqhUc<;UUyOI0=oUD7B(N4dO;26aS1)UZfc9P rZt9NpmyVwZ?hn)@0=gj>-F@}!w delta 3939 zcmZu!3s_ZE7Cx_ip7-8!U!W)#LC6$QLNh|mN1AU@)JPE(#J7UVLkt;19^zQyCCY6x z>es{SD@rFDU5-PFHB*R_hOZneUo+~cXqZ8~)Y<2rb8axxg^#=bf3LOn+H0@9_NL&? zX$6-PyxpAEUKB--rOXe_my|~hV6449N!h8Yg~fS@jEu8ZlfQJo3X9zf4zG2%r6Tn3 ze~P~Flz!o%;R_FgCoohyPF}(co0&Hd9fs+;2r5@jf$LJr;?8ITSrmcQfN@WM@sw z`$NzMoo1dv0ixdm!H3VGLtF>^%LLS)*0Fs&LU3>r+C!Ue$Q%e-+p zhOs?m*vG`qM4?`ay7=#6l*t%kDcY?o;Njbe)9$;_z({kQM{7_m zZRYD?ElM#5>W-s2b0UbTL;LjDVfAUU3?l&Oe-?e&#f)ZaA0EM`JE+a?&&Ra=h!oD8 z;ZXxZc>T}lvkrs|2OE){>A)Zu(TqyF>B-VeAj!q|_NT_j1P1c7e;0giD}(rrrW0=( ziRFWCd?l+p@!=aG{6WFgC@3KsT^{EzMR#H{Uf=_as>{U0N`!|i`2)eG&WM+J9{O$J z-Gj^&eDgWq(;vr?<2=F}zvP?DoM;5tYxpRAFR%}vtmSh%nZSh;d~~25={!b2u;UtE z=lm5PXyF%`ImbSQ)>b~qpAR4N{sX?fixDOS1`2&eGiKi|{3S#PaqAuno)i#XHc;5) zgFxV;F8aub31n{PWI;m4+_MSZGfhy9J@!KUGNI9!*9<g^U1m%Kb&c9W%KQ@QRSZ znj=kHg#}hUA>~lNQwaB$)vmXNYZf!i{*?j`?R$k%qYY$2(-C2ri@pw|iEYrc0jsVH z*9^xw`1ma$+w@y^TZkrs0cm8f8?B}sdRxTHM%iw&iStyVeDu6n4y7StQn+cE^{mJn zXRI8QabldYzH&G>K^$%DQ#pJTFZSqW+RRxht~Snc@^W#DNv~TW78sM4!RF24MdSQy z8Hcv5Vy%DWGhP>Iw`q3c1M#?Qg3Tyv`xfM6 zN`0Lq;e_wnOE(@{DP59v@y`dOF-9K=Jt-mB@(-!0C-L&hBsm+Jf0TYeCYaD9J@A97 zcIibw_>hsmtH!Gn(ZlLAD;unR0*jTYt!U>WR2&heN=RC|@ zN~Gpd)I4RPypb>G=3BS0XV_8N!$Hok2%9<*73CEcuY-Q~@XH`g4<>8#M!`H_AO|8CP;jC_c$4 zMK}>D5jcfc0`MF`>46IbWe9FT#N+>PM+9XSzchVeR+Nl~2}<`4>SSoHZOpVOlS9*1 zrl&8=$j*}SKv9XrsiG1_()c@EA}XH~Asy|$C>~QtI_J;C-mS}bQXaY#6 zW%$jtD#Dku)xUQnAa0EsXwmh(_;R7Tu7k`$Ly_tZ?rEiOQtV|_VQQ>@vwW{im4oS< z(qp2^-$Lz(&=TxqpFs z2h=?6<|Gl8LF+-47x4La)W%>dbBB_ca0*@HPFj`=aZI>$NZcd8Z(EBJAUMVSw2`MW z+knuRn8D+;aq=tJTp?LamM=&-Zhu)DSI6K2UOA zy~>-Q`I7qGGrDCtEDW(LE|P7cR{gBLiJ3S$&rV)0+4g=~1i$8TS+4!1PQ>rl+7nDd zv%&LUv?q0NGOdG*pe4`la2V?F#w+%-{^!JlGW#-fplPc;*0|%!pz1?=&k^PfBkJuu z&bnqU$h2i}>M8m-cXeYpyE?&dhRspgmDEfbY+7eN4Ink<;j|h)txp>q@4) zoK2L{*>>hn^c8A~v{Ac62D6Zu=BkWxkyg+1S1H>AC12hujS{o@tJ?Sp9Qf#NSHDe?emfH+Buv4)2W@2S0{ F{{my_ok9Qr diff --git a/merge_images.py b/merge_images.py new file mode 100644 index 0000000..ae1c7b9 --- /dev/null +++ b/merge_images.py @@ -0,0 +1,44 @@ +from PIL import Image +from pathlib import Path +import functools +import sys + +printr = functools.partial(print,file=sys.stderr) + +per_image_width = 480 +per_image_height = 320 +cols = 2 + +images = list(Path("./export/").glob("*.png")) +image_count = len(images) +total_image_height = (image_count / cols * per_image_height) +if(image_count / cols * per_image_height > total_image_height): + total_image_height += per_image_height +total_image_width = image_count / cols * per_image_width + +resized_images = [] + +for path in images: + image = Image.open(path) + image = image.resize((per_image_width, per_image_height)) + resized_images.append((path,image)) + +new_image = Image.new("RGB",(per_image_width * cols, int(per_image_height * image_count / cols)), (250,250,250)) + +current_col = 0 +current_row = 0 +current_image_number = 0 +for path, image in resized_images: + printr("Merging image {}".format(path)) + current_row = int(current_image_number / cols) + left = int((current_col) * per_image_width) + top = int(per_image_height * current_row ) + new_image.paste(image,(left,top)) + new_image.save("export/1_graphs_compliation.png") + + current_col += 1 + current_image_number += 1 + if current_col == cols: + current_col = 0 + +new_image.show() \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index b3c786b..a500e15 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,5 +27,5 @@ console_scripts = dr.rant_stats_per_day = drstats.statistics:rant_stats_per_day dr.rant_stats_per_weekday = drstats.statistics:rant_stats_per_weekday dr.rant_stats_per_hour = drstats.statistics:rant_stats_per_hour - dr.rant_stats_all = drstats.statistics:rant_stats_all + dr.stats_all = drstats.statistics:rant_stats_all dr.dataset = drstats.dataset:dump \ No newline at end of file diff --git a/src/drstats.egg-info/PKG-INFO b/src/drstats.egg-info/PKG-INFO index f5b3833..0108d86 100644 --- a/src/drstats.egg-info/PKG-INFO +++ b/src/drstats.egg-info/PKG-INFO @@ -41,7 +41,7 @@ If you type `dr.` in terminal and press tab you'll see all available apps auto c ``` 1. `dr.sync` synchronizes all data from last two weeks from devrant. Only two weeks because it's rate limited. 2. `dr.dataset` exports all data to be used for LLM embedding., don't forget to execute `dr.sync` first. -3. `dr.rant_stats_all` exports all graphs to export folder, don't forget to execute `dr.sync` first. -4. dr.rant_stats_per_day` exports graphs to export folder. don't forget to execute `dr.sync` first. -5.dr.rant_stats_per_hour` exports graphs to export folder. don't forget to execute `dr.sync` first. -6. dr.rant_stats_per_weekday` exports graphs to export folder. don't forget to execute `dr.sync` first. +3. `dr.stats_all` exports all graphs to export folder, don't forget to execute `dr.sync` first. +4. `dr.rant_stats_per_day` exports graphs to export folder. don't forget to execute `dr.sync` first. +5. `dr.rant_stats_per_hour` exports graphs to export folder. don't forget to execute `dr.sync` first. +6. `dr.rant_stats_per_weekday` exports graphs to export folder. don't forget to execute `dr.sync` first. diff --git a/src/drstats.egg-info/entry_points.txt b/src/drstats.egg-info/entry_points.txt index a7e6769..f4206ab 100644 --- a/src/drstats.egg-info/entry_points.txt +++ b/src/drstats.egg-info/entry_points.txt @@ -1,7 +1,7 @@ [console_scripts] dr.dataset = drstats.dataset:dump -dr.rant_stats_all = drstats.statistics:rant_stats_all dr.rant_stats_per_day = drstats.statistics:rant_stats_per_day dr.rant_stats_per_hour = drstats.statistics:rant_stats_per_hour dr.rant_stats_per_weekday = drstats.statistics:rant_stats_per_weekday +dr.stats_all = drstats.statistics:rant_stats_all dr.sync = drstats.sync:sync diff --git a/src/drstats/__pycache__/db.cpython-312.pyc b/src/drstats/__pycache__/db.cpython-312.pyc index db03e4b57fa084985f226dcb3b9091e3149e2c0f..02d46685749dffcfa0792c94bb4973780889240e 100644 GIT binary patch delta 909 zcmZ1yJv*BBG%qg~0}xDk9`5iyMaiR_j0|OUG$T`T>F~n6N z%+obop|m))D89HPv7}hRF&Hcx?CRs{90F!eVwGbBGbXzWC`?WkXlDi)xKLPpk|5vY zJA#svKME>O77((ZEH5lEd7_{{npiAXW^QV+4#b3YLefaelf*y{aSQVIRY=ay%}vcK zDTXNP7uJF}d#A9M8aopf7boZEl@w(rm6l}Y=YbtP`JG7e2kWkOxf%vrphD-wg+l6l1X6UDn)Q`Ens8WJY8ZUV?6^CAIBF4CRWZ5 z91N`N?H-LD9~d}6LYzP$F{sc5UXv>4Y%GPFB~IRgDJ2pXYjNm>sT zi65XhQC>X>qM#t^CG3At1(6`+utG)9f(lBZMjNOgsJkXvDTK~pV4mMRGtV>3-SLOx zLS7JfqK)&`1808bgx8z#CUv;xGKuu5%xr8n!+R?uwpd39kM-6@)3L$Bg98!#Lp{Lb z39A<`Sg#^u^VtGr;vE=0J`xFZw({5_ZO0D#K73@0Z%7@OwGZNw#N&#z1^tqz;?nhD z@!-+XBi{J*)Kns+CBhH(HWVDgRAdOjNkU95fk;ObE)KX>R)QS=!oN!h8dJ) zH+IS)oRGU{Z`tEgaPyPYrY=794<1PF4b#N zV?`Aj=aR+)2H&I7zFIt6zOx|osdR6x4OQN+()}Cp zb^aOwdsNzM`2X{yQJUc_<#(_clC1Y|YCS?mk1myls>cvkw3G8%Wz6l!!HN>!NdbP~*}xcl$LE0pl$6`8DX^?*|2vb=rc30L zBDOyRE~+hl>nfVtlT7*~zUqiT6y3pQn85y^+ceLBxHgZI!Cit0C0+-Qi=7t|QgL=qCC=giiY7KoedZ_k{$ z=gj$^J2U(K>*R-p(3ioWPlC^6>%{CAU6(_V#rI(JwnQaL@=}55q#)~ZLD3b0J~^)z zG+h(6l6UDY@Tqxs!J~VGPs@7?KHaxG@7Mj*wIt~QD(OM$1}j88z#8fW)>0p^j{1T1 zGyrU%!KJ;D9;P7}G}0Pi6Ria{(>h>;)&pB;1F&^oiHC1t<#EMv&6?#5opI#UjN?VS zVwudkd~<^M_(pOd9E8VErKJI0mcCOx%+FsX*GLWDD<7&4fYMwpGujtu!-%*>V;wPbL`Vobz^#M(yL%rK)kYqBVxQjZfq|42QajEYd! z3O{TEU`=5C+Q5ueW~`HsYJ;$FQTs%xbz@irp(5J&h&M($`7!U%7z(8we;Vql-GnJNBL)x&{#$6*a1za} zh*7}$jE&h+p7KOxuEu7I!8TSsxU{L%N5Uvgtt~Y z>Zp5Q3e=tH=zmsSl zpdE-2#FL1f2qCN!t%tw1;dnPO>|^mcHF*@DdI6y$jv-cs9f#rh!1HrwOJy^bEmewT z_9Fj#b2KU5TLvFqFB5!DodrkO%;XKLoY{t_!D7@tIM7R6GccVcJkb=kFAeV2+EAKyJ2(8LK6n;76aBTEr6K)otv!PL)`}DBpLIMQ=n$ zRvXj?5!Ed&Ho-$$6?ZYwkd;Hvc45pIq61*L0OH`sWESB4!x2)l)5E{Y0X$6#eE)n$ zmiU`mvpuzg$b4Tc^$W5%@?z|lB2Lnk&UAtj~bgoq!IfO(+C4W5qSHu5+F_-Rb0^o2LIio_uW%Q zKF`j=R2|GK)(we&HXc=cT9W@X{z^LB&L{tnh&OpL^?jh>t;yFWzlm(RGIS%7_%^WR zjsg>R8{pZy65tNvo`o0nyQ!wMtXz||wBH@yz<=M}v?vO4+?0vikzxOs1#oJYT?4ej zjB>72tg2Saxq>OW;COR|xe_bqb2DNaJHgEI-$!*9L@*WJ{;U%*ju34TT@t;Sk`91# zU5BtaLOJuH7%Q6vsXG+FpOe`PzInO=2K4b(_YCn~$cSp-G`t2`y=>7SrOcy&Gp6dpDY0 zw@un&kO~M6pfU$eNENrz6Y_ye5dsNu08X|98u4*%k)Ws&mCB*Kw@zI*+k&P2dGmgxn&;rdAsX^?$Lz9A8I!adg#0+k2nwQPZ^DNWDN4tB%6mjp}q-CN19 z1ij$c4NiSj%@|ZM-5^Dy)E*bHCzLHUMP)s!(q5JpPLd$|QaG9DwTIF!_@nCpj|^;i zbYRM~D2=jNZyYAR)O z86%%p9ZqLTOO3~g@EnXN?P?cEsoeZG*@Sb@0RZ=AsjBKUhC*#Z?mw=oix|@|yVIH& z#+k1*(zU7Mn5~F0#5f{~m|%ycp7ppIkBzjQ%IvGIw%`FUI1QKiEQ|NImA;mWLVqnV z`aBMQG%Wf8YST>Z+P2PKjQx1Ob}`uJU47(W>Bp{%!T`GC!h%ttTv*cdMcd&`drc8s z*biNhG&Axah zQO)mVym#am=wDBcl{WMb65$9KFr&M@)VTSwo82Bb7utkFsva5Lcy`18WW_ycnN}az zx+o5PY*uVvte>1Norx_8iQ{N>0`UqWiFla7t8lv#KDKn$u+(VADCibF#lk~<3Hy1y zhQ@320gX4Cu!Gf9PByL71a1bCW=f}qHWDJ2-j1L2_S9xrX&Nx2wsjQtO%!W1&hC#M zl`s$<*0MS}s?L0pWyg*bapqXHc)4%hpn2J<+rJYS(ssl)#3Z5{VHdXj%y;1JPDB)e zD-`oldmU<9o$ki2J%}EFiCe|q?$Pem4PP)}n^)6(u;KO6(edBiVYKzYb?3Iv#Xk0S zl+u&L&Hmip+kk=CKSj7s*zopvo#J&UK8NCWD566NIFz762|1LoLuqg*5r@*~P?{V{ zvqNcdD6I}fawu&MrM;@OJzUj}>!iBA&|`GAv+sBGkWO}M$G#%IR4l`wJTNEgnFT{% zKF|HKwV-OI#Y58z>bbP6%Ng0^p(7cpEm_lsqAJ{*mRIr$56r6tl`g1sWhFGTWi)G8 zBg0#UM|j|jYFdh6S<5^)qvnmAW-b(T9?Hn`_zJUR#o%JDpl6GUOf4=fYKs~db=A@q zDNN<{>qwW-O}ds!C%K z?p=bWMg5SrebPZv=CYpA4DO)?olc<+=kg&Jox|n~;vnJ};xrop2jA}L)%hG5?9IJEW9JNxG5*2Z4 zoz2;@G*l7eb;?7V;)Cp+nK>tOI(oe_`bQr^mHYi$}@@|U}qj=&1l}26)icdZ3#U@@hgd$WMjs9GBujE`c*z_ zLNOQ>R4!Xmu?>tund-|H+11%^)_u9EnibCzPhup=B|=+=x>6;nk$3yI4)uPpWvcpA zU#v9Rmvgix>ymfVZT^hojOy!;RYcF_>@^84W3N|z-)*IfJ^3P9HE#uiJvNp$)u>{= z448|4D+b!(g6xqy;1E^+2Mgb$^CyDDX)MtRfV3MAK#p88zGO$sp^2$b1lO^wdWHrHg`;rNfs3ioL}ey4bQK3e5e2u`xQCT$Pteuf{9C?P;-tR>W&Y`FDJS;ZSK$LXx>cA?xZW_ojii< zRpruFm}QU0Txq`fT|y&QaiZy7zTWN5^4XNH}flRkTn!C{^ zd7*xzllOA#1Hyd>o5&Z@51UD~xz=!mR>W@NH`g1OLHHP%G&k`cDo?DxFfYS)HSJ(DZmJgxPMvuU7!unJ$>gu@u}%*WF}73i~_Q4o{)!!ePyXhbO0o6>9WB7H?*- zF_bV=Fce;uJ;Uw=tP(y&oR)I2i%RxalwRVscq%05+;&)?7j8Qxcme)&>R^4Lk-xw* B@wxy2 delta 987 zcmYk5O=ufO6vt=wLt3qN^^Lxg*sWyCk|oEnNu8DJ zRK!@Mg{&b5nMtt|TB1@2&Y{L7m!zlIrCg}mN$zE#NQb6IAL6H*{eu}Eg}C8~&1 ziz#?d7^eO+$nR*sf*N+EGS-kCZO&oywtW7TspY9!aBX;Fw(ip&_L#nmG2T#IRc&d) z7;6NN?IEjrq$)JLf$GSvFVyOc-Y(xPuf;aT){Z~!ow$GgVeG-B$Gz|T^nShb-KtQ% zu&lHqr#7bVzhR!vEY>5Fuy%3T*$PB!WA#9yDm?XvZ^&&YWNo{U8dy!3O4P`-L&)yj z|1*qS!5wGBbVhcZgQjzE4{=bX;@yVJJkQDve8VZvhh)M`eF!*zf9FC>boHWOpIA$h7ZrqpT34N0xKmt;e11&*wgm&>)T+GpRU>VbYk zFfJ@9ElD5A==Zg2l}~47}OTX#@9mq-z-W(~so0@dsFl=RR?axjbQB!j9oCB3{OBwnt!F$SB|>-B*g* zo6I9J4_7RbE@nlToM7T%;$sqG!tU@v#>3n!3DM}f8kElh!vBf%4Bb}G1l`!X7qjEk R-cc2QfS= DATE('now', '-7 days') GROUP BY username ORDER BY userscore DESC - """ - ) + """) db.query("DROP VIEW IF EXISTS score_last_7_days") - db.query( - """ + db.query(""" CREATE VIEW score_last_7_days AS SELECT user_username AS username, SUM(score) AS userscore FROM comments GROUP BY user_username ORDER BY userscore DESC - """ - ) + """) db.query("DROP VIEW IF EXISTS rant_stats_per_day") - db.query( - """ + db.query(""" CREATE VIEW rant_stats_per_day AS SELECT COUNT(0) AS count, DATE(created) AS created_date, @@ -53,12 +47,10 @@ CREATE VIEW rant_stats_per_day AS SELECT FROM rants GROUP BY created_date ORDER BY created_date - """ - ) + """) db.query("DROP VIEW IF EXISTS comment_stats_per_day") - db.query( - """ + db.query(""" CREATE VIEW comment_stats_per_day AS SELECT COUNT(0) AS count, DATE(created) AS created_date, @@ -74,12 +66,10 @@ CREATE VIEW comment_stats_per_day AS SELECT FROM comments GROUP BY created_date ORDER BY created_date - """ - ) + """) db.query("DROP VIEW IF EXISTS rant_stats_per_weekday") - db.query( - """ + db.query(""" CREATE VIEW rant_stats_per_weekday AS SELECT COUNT(0) AS count, DATE(created) AS created_date, @@ -95,12 +85,10 @@ CREATE VIEW rant_stats_per_weekday AS SELECT FROM rants GROUP BY weekday ORDER BY created_date - """ - ) + """) db.query("DROP VIEW IF EXISTS comment_stats_per_weekday") - db.query( - """ + db.query(""" CREATE VIEW comment_stats_per_weekday AS SELECT COUNT(0) AS count, DATE(created) AS created_date, @@ -116,73 +104,69 @@ CREATE VIEW comment_stats_per_weekday AS SELECT FROM comments GROUP BY weekday ORDER BY created_date - """ - ) + """) db.query("DROP VIEW IF EXISTS comment_stats_per_hour") - db.query( - """ + db.query(""" CREATE VIEW comment_stats_per_hour AS SELECT COUNT(0) AS count, strftime('%H', created) AS hour FROM comments GROUP BY hour ORDER BY hour - """ - ) + """) db.query("DROP VIEW IF EXISTS rant_stats_per_hour") - db.query( - """ + db.query(""" CREATE VIEW rant_stats_per_hour AS SELECT COUNT(0) AS count, strftime('%H', created) AS hour FROM rants GROUP BY hour ORDER BY hour - """ - ) + """) - db.query( - """ -DROP VIEW IF EXISTS user_stats - """ - ) + db.query(""" + DROP VIEW IF EXISTS user_stats + """) - db.query( - """ -CREATE VIEW user_stats AS -SELECT - user_username AS username, - COUNT(0) AS post_count, - (select count(0) from rants where rants.id = comments.rant_id and date(rants.created) = date(comments.created)) as rant_count, - DATE(comments.created) AS date, - (SELECT COUNT(0) - FROM comments AS comments2 - WHERE comments2.user_username = comments.user_username - AND comments2.score = 0 and date(comments2.created) = date(comments.created)) AS ignore_count, - (SELECT COUNT(0) - FROM comments AS comments2 - WHERE comments2.user_username = comments.user_username - AND comments2.score > 0 and date(comments2.created) = date(comments.created)) AS upvote_times, - (SELECT SUM(score) - FROM comments AS comments2 - WHERE comments2.user_username = comments.user_username - AND comments2.score > 0 and date(comments2.created) = date(comments.created)) AS upvote_total -FROM comments -GROUP BY username, DATE(comments.created) -ORDER BY username ASC, date ASC; - """ - ) + db.query(""" + CREATE VIEW user_stats AS + SELECT + user_username AS username, + COUNT(0) AS post_count, + (select count(0) from rants where rants.id = comments.rant_id and date(rants.created) = date(comments.created)) as rant_count, + DATE(comments.created) AS date, + (SELECT COUNT(0) + FROM comments AS comments2 + WHERE comments2.user_username = comments.user_username + AND comments2.score = 0 and date(comments2.created) = date(comments.created)) AS ignore_count, + (SELECT COUNT(0) + FROM comments AS comments2 + WHERE comments2.user_username = comments.user_username + AND comments2.score > 0 and date(comments2.created) = date(comments.created)) AS upvote_times, + (SELECT SUM(score) + FROM comments AS comments2 + WHERE comments2.user_username = comments.user_username + AND comments2.score > 0 and date(comments2.created) = date(comments.created)) AS upvote_total + FROM comments + GROUP BY username, DATE(comments.created) + ORDER BY username ASC, date ASC; + """) db.query("DROP VIEW IF EXISTS contributions") - db.query("""CREATE VIEW contributions AS select distinct user_username as username, count(0) as contributions,sum(score) as upvotes,avg(length(text)) as post_length_average, sum(length(text)) as content_length from rants - union - select distinct user_username as username, count(0) as contributions,sum(score) as upvotes, sum(length(body)) / count(0) as post_length_average, sum(length(body)) as content_length from comments - group by username - order by contributions desc, username asc - """); + db.query(""" + CREATE VIEW contributions AS + select distinct user_username as username, count(0) as contributions,sum(score) as upvotes,avg(length(text)) as post_length_average, sum(length(text)) as content_length from rants + union + select distinct user_username as username, count(0) as contributions,sum(score) as upvotes, sum(length(body)) / count(0) as post_length_average, sum(length(body)) as content_length from comments + group by username + order by contributions desc, username asc + """) db.query("DROP VIEW IF EXISTS contributions_extended") - db.query("CREATE VIEW contributions_extended as SELECT username, contributions,ROUND(CAST(contributions AS REAL) / CAST((select contributions from contributions) AS REAL),2) as ownership, upvotes, ROUND(CAST(upvotes AS REAL) / CAST((SELECT SUM(upvotes) from contributions) AS REAL),2) upvotes_ownership, ROUND(CAST(upvotes AS REAL) / CAST(contributions AS REAL),2) upvote_ratio,content_length as post_length_total, ROUND(CAST(content_length AS REAL) / CAST((SELECT SUM(content_length) from contributions) AS REAL)) as ownership_content,post_length_average FROM contributions") + db.query(""" + CREATE VIEW contributions_extended as SELECT username, contributions,ROUND(CAST(contributions AS REAL) / CAST((select contributions from contributions) AS REAL),2) as ownership, upvotes, ROUND(CAST(upvotes AS REAL) / CAST((SELECT SUM(upvotes) from contributions) AS REAL),2) upvotes_ownership, ROUND(CAST(upvotes AS REAL) / CAST(contributions AS REAL),2) upvote_ratio,content_length as post_length_total, ROUND(CAST(content_length AS REAL) / CAST((SELECT SUM(content_length) from contributions) AS REAL)) as ownership_content,post_length_average + FROM contributions + """) db.query("DROP VIEW IF EXISTS rants_of_user") db.query("CREATE VIEW rants_of_user as SELECT user_username as username, GROUP_CONCAT(text) as text FROM rants") db.query("DROP VIEW IF EXISTS posts_of_user") @@ -245,7 +229,6 @@ def get_contribution_average_per_user(): return round(get_contribution_count() / get_user_count(),2) def get_all_rants_of_user(username): - with Db() as db: try: return db.db['rants_of_user'].find_one(username=username)['text'] diff --git a/src/drstats/duration.py b/src/drstats/duration.py index 7f12603..02ef7f6 100644 --- a/src/drstats/duration.py +++ b/src/drstats/duration.py @@ -1,4 +1,5 @@ import time +import sys class Duration: @@ -12,5 +13,5 @@ class Duration: def __exit__(self, exc_type, exc_val, exc_tb): self.end = time.time() self.duration = self.end - self.start - print(self.description,end=" ") - print("took {} seconds.".format(self.duration)) \ No newline at end of file + print(self.description,end=" ",file=sys.stderr) + print("took {} seconds.".format(self.duration),file=sys.stderr) \ No newline at end of file diff --git a/src/drstats/sync.py b/src/drstats/sync.py index 73bf4e4..b6b7f38 100644 --- a/src/drstats/sync.py +++ b/src/drstats/sync.py @@ -26,35 +26,33 @@ def timestamp_to_string(timestamp): async def get_recent_rants(start_from=1, page_size=10): page = 0 while True: - try: - rants = dr.get_rants("recent", page_size, start_from)["rants"] - page += 1 - for rant in rants: - if rant is None: - break - rant["tags"] = json.dumps("tags" in rant and rant["tags"] or "") - rant["created"] = timestamp_to_string(rant["created_time"]) - rant = plain_object(rant) - - yield rant - start_from += page_size - except requests.exceptions.ConnectionError: - print("Rate limit of server exceeded.") - return + rants = dr.get_rants("recent", page_size, start_from)["rants"] + page += 1 + for rant in rants: + if rant is None: + break + rant["tags"] = json.dumps("tags" in rant and rant["tags"] or "") + rant["created"] = timestamp_to_string(rant["created_time"]) + rant = plain_object(rant) + yield rant + start_from += page_size + async def sync_rants(): count = 0 start_from = 0 page_size = 20 - async for rant in get_recent_rants(start_from, page_size): - start_from += page_size - count += 1 - rant["tags"] = json.dumps(rant["tags"]) - db["rants"].upsert(rant, ["id"]) - print(f"Upserted {count} rant(s).") - + try: + async for rant in get_recent_rants(start_from, page_size): + start_from += page_size + count += 1 + rant["tags"] = json.dumps(rant["tags"]) + db["rants"].upsert(rant, ["id"]) + print(f"Upserted {count} rant(s).") + except: + print("Rate limit of server exceeded. That's normal.s") async def sync_comments(): comments_synced = 0