From 2043c45d48c43e20d1a557f552b617bb71c87bc1 Mon Sep 17 00:00:00 2001 From: aeon0 Date: Fri, 10 Dec 2021 09:17:08 +0100 Subject: [PATCH] Improved clustering & cropping of Items (#247) --- assets/hud_mask.png | Bin 0 -> 13469 bytes assets/items/magic_gg_club.png | Bin 1357 -> 3009 bytes game.ini | 2 +- src/item/item_cropper.py | 115 +++++++++++++++++++++++++++++++++ src/item/item_finder.py | 61 +++++------------ src/utils/item_extractor.py | 62 ++++++++++++++++++ 6 files changed, 193 insertions(+), 47 deletions(-) create mode 100644 assets/hud_mask.png create mode 100644 src/item/item_cropper.py create mode 100644 src/utils/item_extractor.py diff --git a/assets/hud_mask.png b/assets/hud_mask.png new file mode 100644 index 0000000000000000000000000000000000000000..08e28a38a8e39407cda4ba107f4b2dac22bb06c4 GIT binary patch literal 13469 zcmeHucU)81*7hNyf((LVSA>Y8ID;X9^k@V`MWv{8MZk~(NFb3=gOpM4;I$xQMX4%+ zzz7xuL7F;>6%eD+!2)P#QUW3%-#S6L-tV2c&hNc5_r8C~4@o#@pS{;!>silw)(-xC zgQcnJq&bri1W{ePW~CK^jEA?FawQDB7-_9jfEQ)qnoS}E`PLWxr%=z+3PcdKY!4d; zUk3|w23z2z$KnWFxq5+KLKux8%M1gBEVd`t7wgJ(^YCVBcU~*f#(Hp=+V*4%f`!n8 z>+Z2e?8CJdTiUS2o@_ct+i*E{Ss()@@Z$Qiuz_BDZxJJqsV$$E0q@bvcx|kFh_5G8 z+ZbIC>tL|~Ya;OBVySxiI5vSmz*6aYEE<)l?@C{SH6R%1;|X;5#t{e%B9%d;VLyIp z!*V_x9>Z$o>W_=TZ%l1>Utb{uj}Hh4&V1PF;z`+QdD9GEF6^Qc| zE%-FUO0J0Q<016*5O`zJ8Ck9ZKVPOcm^!)#FX7m<-lC6w0*~PXSwg(N9s%8xd?1HC zHcsg0!jx?5>6k!i8cf(gG^!Q6L17FgFqM^Y9Ziw@Pa-cYJek9ZS*M&A{oB_X(+&$!}4W) zaWIF?;0b)ZSP%mbFP0k@FZ6cP#*RgnVItrQd|+X)jwrXn!h*5ZTja~~W^>oBWNO0} z^gKK`3<8}+qS7f89F;~Q<48P`K8{XurQ$d|BAd=5(TPM??x*Kh3fO)q*wE)k6OSWc z!yNxum1;mTFyN5MI4YgX!?DRE1DvY?n}Q>;^j$eDSA9B*J?ay%X1$LG+_lULY5B; zSexr3Vruhz1YX!N9AkY2Se8)8_h8Gziud>Cd?dy(q-p=|IwF?;zjhIU#vyTuTsDqI z)2HC{StJ&YMRaB1T)Au>flVY)cxd2$(?uj3v`)|%B-*E({olCg zPyg_1P4*{+BlM z50Uj>N8F#oIe#2+f6jVe4ALLdk^hnw{)!;s(UxmeQsMu9wKe+t%F$}E8dV_ju7vTM zy6O*N5ZV+5h~hDs{7KOM(S$^w7yP@z7}ZAdjepjAe!2t=E&6t>m4;u&nsKf-s1YA% zwI^u&{TYIMcVO*GW1GO(_PSvHQ(4_eS9a{a@N?Ms;#nnIFT5_N$4*qzEVn3sZE*1F zsaY)&?nXD#uY{S$JuoL2v8#{&m@?ZqZl~zH#P;Jw4rw3Fi-q3s=%!E>k3v$>W`_`CBm0EX)!#ESD+5qley9 zU-2}Ss>0jyZxspfGGA#c{9K`43_o{c!{KoyV`LnHsP9B3AP81b;Tt&CP{PbYkZ|=s z?SWGvQuLfeV{H{5_My?F_~%al*Kgqe>MSZ8PLOh_0@#mJlQJ zJ`4}0#Dn9F)(xPZhX5lqY3;bbBE6)~{~;uhpur;QI=`ui*Pn zs>81t_ci0bX57D>aVrKT;~K-wV`*K7*DqgNu+zle!Qox4sBHR-8OfR$qjeS*7FVuZ zNwtrfI!Q%ksqgD3@%ZPj+%8v1<~FuE1wQ|q%8Vsz3U=PSBx)&*X9g!bD;TXK@*Z53 zJ$1Cs#OmbtR+vHiX<4vS7>hh+^uBqlBU7Agc2JW!X;Q67LEQl^8Gg)eL*mLc#X0k!X{})RNsovzQS$^uZFF`H9c<4L zR9=`bQ$MXOL?Ws3tJ7~Dr{Ep4m&hiqa8D2;8c`E91GkR2#0qaHA3b{1 zqeQ5*$UJ(3gG01<14UOOY6m}(`vtBY48vDbKrLC=6VMATtTPGhmdh~2P z2GLOP^Ye>i`2DbW@mt}_!ZfqvdZ{|Yt1b!De)#5uN#WOq{_P+6{9s|^}(HWzK<;+?5Pn5Vrq5Gzdf`kreXgavNSHB z{sh_c>3dnLr>AG!go(!!62iJ}%}jANR>DMYA&HtFXbrzMJ#gSaYJ^ZYY0{*=uww~# zIhU&1?88oo=OFZ?3m1g2O_PYp1?|~2d-tJVW@cu(y1FqZPg<>CaNs_^XlSR39w1h| zo4G4~i?yAde<{7i#>OT|^Tv$h~50)gvw04C3t*hYO063du5KmMe30idvN*t z*Edw~=CGn7f!@-6AD=+>w3ANPh_Z}h$XcHX03Y^Sw%^+17U$P?vBdv%`6)0zC@5(B z`0?%4S87IT3{E^ST)#`9v4LxFq9C~c*s)`J4i^%Wk~mHGjA6Qc@WtPC zGua%nVGNm3X$!91Gq z{HM9zYnZzomMQMj@EIBIPj9e01#Z0=p%(V``oxLf%qlMo36`l8|7^QqLwTl4(bCTR zo~th-gS+qQKn$H!>W=kx)CIRyUMw1F!VmVoIV@IruC&$X7O`v@(W^R3qD#Yu4ED{V zde@$Ci%$(zpo#PR+r=*rPyvI6eU)s_r@w5kbDb;ft8#OJG<|vdz|y88)>eY~ixtc* z9g0GS7aUx6NIZUG5>VOApH$}>aMrL`r{0fmJSUZ;K9)Umaj~@}dsHM@#8BuHy`3VN zeU*>b8y2_}g{H*$S2teY74gk%ADFx|%wbWIRg@Tc)zs9)>@7C|fUhgqIj#aI#!N3a z$*nA2Ix~Dmd9J8s`^($I5GZEXuT!lB&o_|Be!I?UYcIAe>PFrSS)c= zy0@@gBM5R0_@E950O%^9z53`w-a*3;?}(+emu;8JI;t+W6pegXdw9{6p*52&LPJ9{ zoeS~;Iy>IY3_s>O)KzkDF4ep1#jS-47g|nn%mmK)#o^)$ppATKTKetCc}oFYKsia8 ztCnD*7w}-D769?kLbG-2*4?{z&%d*N_(|Ai2Zun&w6?G|doGCk?Dm=*Et;_L&d&?%&u?y0zj_7s7PCC$Jrobr!$7e6XX(Gl?i9?FhU=3>hO!j$oXCJ=soy2;*1v4W1fyo8h5;!Og zs1IsU3w^um^y$;%QXBH~^T(^qs0Aw9Hxpla|Ned6Wt0l<&7~i@RK*9I{QRzzl_Kt} zc?@4#&jTVr_T|*dY`3aQTri;mQe+vw@`9;m5a)5SMe58)JmsyE>-J99UA=!+*HBNH z7GLGOV@`*g;b2vLL9pkxZR^a;4puCS{0S(>*|~7rleG8m-%A{kJqryDi#(rY6o_R% zEPgwcA9KaOy}Cp*xDASU+_Ioc>ucz9hFW%+*Z|eFUwIb#{!L0mUuGD*WES*Qm&Pqk zPfLqzBSdD$Ee$A*p$ac)k@KuG8_((D@kRah!BC0{0I(8AcTYt^#OeI} z9g=e!U3PTcdcNald*>i-eSQ##1=DTQac2q(LrY`nXV0ERsX7Nyzh4+KD3wYzzM0U~ zaji2x?8Cri-otgw-IP^`XmJNb+=VyJu{yaYZzmqcl7~=3%hR5jTrGuOG|4mjjJgy7rt}Z zJHyV=(Q(U`A-J74byoO{#}Y}_j)88kn(VWhEB2*jWN0$Qy;&ZQt<+8(P%qYqt*EGo z&c8RwmA*@zM|45Z&u8rS;bOf+!01+}g>!f*&A#b&0$ z1LLcmg@ebIYy#rRJq@KMDd`oe!d6Q(eVbGy6>EII$AqwaLWfW^iW#Fhrij>&Mb3=|CPYo12>fe(o`_-(xt(4}CdBFfyfnMaH{a znxeTvcR~2T9FUWNfL4%`0Z$~79q<-(K-X$rB=RSypGBZFof*YNV1K_}-s{s)-!Vt%{^s?3Ht2YKq^p0 zQgDvhZKwC?om?8=EuEi*!9l+D?d(3{v6Pkt2Z; zZ5Pk7c;+*gu;Db@bWoqql(&A^r8<`aH0N5Kr^N=wh!d{xMTg%?%vvoBrheq!$u(t18; znW{?o%^WD#XwHI!(ERY=N5{e)19brnapLiG3#4ac;O$Md2sN!je$^$3iF*jB13(Hh zsGEa-h-rKaYS$E0zdN6A&fIOs16~HM1$|SyFnl1axh!GirPj#Rw6q@p;OVz--#&Ec z5L6ahP+0XCBV9WemyzcYTH?Ox5zX-0hn@RjuEK+n_b>M0Gp=;)XT*3)X*>3apE(CFxpZN308szZtHlnfyO z1CljYr1uw(Q@a`!A2vVyZp^Zv?vA=^)|rXtQ_r9G;`4oFbC8LicMmNDeth`wA%s_o zb158plac^!QmuCQnVL)7`%rzL2k^3>#MA+ILID_0Kqj8e%?;|P%>_)uL#u~A^cVHF z|7x$?_!=dfyJZ^NPnh$7H8mTRi?45nY`*a~mDbi)YoNEv{GgtMk)amY{Co^83&L{T zS^T|hQgt#or|PoXHPu<+yR2+%vVa0WL;>7ux}@ou?;~F%C<@(yqB>A$W+r-Vb9WB} z?FNEu9%h^{{OK>7h(LVz?yUIOaRQ{wC^|?y2k4~N-1fs}MMGgmaR9xr z<+=0c3&4F~Q33e%00^mu!h-DV?A%HO~&g}o8lo&s$*}kCXf1BxDAbUD_9wSL}#X1Zv6Kj|7GWW}&-M8|YM_f09l|4+5TMIK_xDuu38TU0!zJ5An0I{92!$jW;sD z(9VGxK+Bdhn2kGs>5_p=@z9iF<6MPVNO&|z7Jwy8dBha8+Bzg8M5_GVg795wsi|wx zRllvw_KHbQy!mah@p*@f+NgbNwxS!c&jiYk|4%-C^ZVve1HQ0^b>vT39vOtcbz$g z7JGu|23RM{;6E6w;XTW68a<*3M<%zR!9h*4O3Ta?07Vz;rjNS+v8EkNBgYmz97ta3vhacq5{1&9FWjOoW;vbIyzaBY~gnH*q6WYofVKB5P zsORmK7Z=D^f1oUMfJ@L6B%z-fznMO58ciW`Ei&=Ndp88RBf@AzBax@m(zZcckZKPj zz<*~PK&%2WqbDfgz1=-MPMCRzkw~Zkg=kelt*j8Kq4VywO}9IX&PzvIMd*jkNE0j6fwAS+0y@!_#Mv0^-3!`9G8G)C6DyXccA(b5 zpDtXC1u2}Wg9hV9OrZj@waB?Z44{~U{uRTXv%&oxA8%SrMvx0Gs1{z2RYl>fXax-_ zWUfSajR|53xg}q-Hiy?EnQ#W@H9YS41|z$;+zWM|_!V98EQ*i#62 z!0p*?)8K#&E}mXI37r!F+ma9#io$E}>DRTna99N6px=l)E{E@8G>gUHnD6}Ilwk5K z0#-H&fV#0)`b1X_JAPyFkHC3wf(Q9CEhQmTES~{A^kMk7jb_?nOM250xhF2(*Soev zrU)Fc5!$L0U}iKnMg}H7eKULcjzH=3r0>Gdynd2y54>fKMv3Ent(}6Os9LHOHVukx zEwyOip&XP4;6z~fm5w1S{3|RBrxv>=sIHTT@M=qZFzOWWlQ}Z6JFasF9R2}d8;@99 z<~AL3gqq4!(6|#&G*|}`3g{A2u>Fvk-u5B0cw}-D)LD&Xfx^)|*2{f$NHX<~GKO~S zaw$W;W%rGB*8=1RG1FfuDrnsAFB%y}HDBnLgvguKv+^R@c4#TgZFQu*kNDM;!J_2P>faQ+39ZGmoO^Bp*qS--R$4jJxD z2NR$-P?LuM_-MYplbI+OGHE6=EAVFbB zDKtAZDeDtZV;axRUFy$=M`yrwWt~_8=T4OT`s+M6HZ?sk=O{DCQZ&WRobbwGEFju{#EuCQNKQt<{LjUa1RS+2}j Iv32MF16Fk>CjbBd literal 0 HcmV?d00001 diff --git a/assets/items/magic_gg_club.png b/assets/items/magic_gg_club.png index c9ae8dd4a434c03e37c6ff6a54ae8c6846663503..a2241e76c98bfcb5c8ef17aed7b3b4d4fa9063e6 100644 GIT binary patch delta 2989 zcmV;e3sUsW3c(j4iBL{Q4GJ0x0000DNk~Le0001V0000J2nGNE06}Xh#gQR2e+yPg zL_t(&L$z58aFxXsKKH*j2_YnqkS76oCWaso5*!M26p-3+3bn;DDoNz+1F99KlsIJ) zGy~WvcCexpXq^G2(DD$50xIDlN)e$zkYWh|B6&d|33)>v_wPApcmIFyBM=ZiX0y9z z&z?PxeVqT|a2ih2?KS}%4sgJIe*_6bu?)T44op)pIGs*zt3q?{q?rTUsJ?0O;Y_9y z9N0D-2DnL{q*BfDxlaoa7ZH8evGp>zZgaPBEr7F?K;NnN7jg}ykAl-Mz`)Yn(t^4; z0JVHTn&|7Qb~{V)Fnfk7sxdeg@16A%;lQwB#eu`xHXKJ~oBqURm0xBi=bDpAHmqMi(t)~(NJ1)REl^of3U)U7P?$6>zy(? z8c;lgWSAS}5HDS;;_60-NU0SBsDT9CIrgZ1X{qtpm|-w=+8ziAiQ>LOZEXpxd~XQc zYOawK32iaj5SA`K6ka-$SjrFdfa0U-!c@R8qoaqIxw)p{R zyDpe;`K9|Q4l1Zee}F5{q(qGh^@;9}UoTFKm(y@?ZBSd*qU|tUI7WrqJcfQzWT`(@_LFU;-{3Aj#x>2!87S<>oH@tfFRMmCdHH|D4ln}nf0VRvTLB#mpj6D$lXTy2N51^}t>?a6F!>pH?zw+JZf;j_ zqXYTDprC$fF~TXvm4;x!6k6uK(?d`P(ZV?uELHT zFT;rwYb6zJOPJu{^Qx)~@bSkJp}MMo*H;0x9t^j;8UFOn0H~}iglW@0g%_rN20@*= zS;p(AsX;{%xw&DGlM@Jkes>Uz8uc6K*LOVAe-&zOT;{sAu9RyX^!^ZT-nxOiEC^=5 zco+%_K84-8f9-kNHD@ea77aPsE?ENL`nB)S2r))gos{{9x1->#94EZJG#m=ge+I9< zb`~PKCvia3n?p0$G|?2xP%Zm}e*MN`^7CJ4X|CtKTI+UbX=&iPwY7AJe^8h$|_7j*s9S@zRFM*yDZjh19&9(5s`)RN*F9-Vfe*zXR zEalVD9#{_U_{z$29OTHTM5eb3Kbs>2ujQ9sI)Vyrg9G~)ORCM52);w(GctD}pKUpfe_K7AYzNEc2?>ctr+Yfq8$KzUrHG_^4{JBr zy&yVzFvP?R<;`~d_+Nbt$AF3zRrX|>t#Ihj5_osTK<-bOvW;thh>1<%GIrdXn9}!! z^LQF`1AJ8XGLG=dD@UQBz5>@=CMg7wtrB9;fcf(ax&6i)74YU;tybyQt+!tXf9cE6 z=*g4oSU&@=I-T8W@&q-@X1I89KRRv<4rvl~;n>l$PKGt^MVHX8XZ6;5LX7jLq#w+g zbDT?$9%)ckc0v%frSu#fx`f8_^+%|xIuC~rufXpveB|Xx@W$-fhhXN+{g?=(@X4;O zLdTPE52GcB=Ep|**6lxFJPaK8e@jX61bs$r+57^x*RP)dD^?_NS-CO=-wnFf=x03? zO@+B+TXtyK;NWm>w{e0;Ay988jfz5D1Cuh*7r1aCj}xzWf@^LtME*@ZpC~U=Fe!Z^gkLOIcSgf1R=g3%*B% z?41!4JqWX~YsjMierbc=(4|Wv;O`rr#*8lC7OWXh(l zP>!j%b_R-y4spA*^c&tZRIXK=!9=G3se(~IDb{U+#o-8qF=Mm&N;loDOd}ha!}dOc zHU=g<8jaTZ^2_BgY0}>ye`(+|%x;ewF#5;8gU+2JJ;mtopxbAM@b%a4!=sPBi1TA4 zh2Vg3;}+v4uHvMpJC-=ll$2?l6g`|Y0e0@3 z%XCkO$h7}{2-u@ud^&JWN%6mBQ2)t}^Moz5M_d|SvwgWlMh=9Ce~1L6s?1cKI)I+> zBOoF&84{Bw!K<&I!>lp@wr$B2eEdgfPEC=ynlhOBH7A7S?fpF*KeiU0eRdOO+Ql$& z!YY_Nc_W^ZMtJhcr8uX}!#Z`Nrlrn+xr3t$PaPdVVe{P|nd6KjFrV$-%V@D877HjtDsM9$!obH8xek z>HJO5>!Fb_eE2MAY`g|rw@!r%g`Z1S2^5l+HVfk8Q{l>$Z=v$~IY}3l0wpZNP`REx zheK^m36#liTL#KVO`9oYoMss{HJ73A!aj7|Mrgj(h@TrLf7w|iHz=qpj2itqZnO*- zIr23wBSy@D{QM0v`@;a?dL)z98!~hXJBCirkt6THj_tGXx@{I~BRGJnjdLk<^tSQ) zX4xSc6y$<}UHC1!i`xl_KgDn5cs%jjVe{s6xN(E7Mw-j)_TB{_K# zl$0EW;$q$Te*tlVMe@X(Ow9ND8($LUe{9Q_Rso>ue zaK3s{NRDCn{IH+`Diyn5kOp=H%tn?XgyX~;*U{f{iPwXt!)Ayzc>;SHo7(@0@~G5{ z{y&tO-ajmmcD92_4igz_L$YXF?A<|=Mu6 zpFTeGe~HPv5L8WII?m*RZSuGlOoDfhS!7SaBN;wOL|)%^D!At$qJ7OhK@pLZI` z`VJM`x;g^-^t*z$pu2)pl;Nq@57tMB^MxOpH;G(C>-(}2mCma8M+_lmp;+nbV z?6dak?DM!Y4j~VQBkh(jDzc?Ve2bsU!p z(0aqWP6?{2ocnff2F zJHW9yS$%y6PM-W4<>fIkFi0lnap9vjWHNut*h}~bsVS}`VUW4s!}5e+eU*NTufCxMA9jJ?J__G6 zwR|Jh5aC66IBZM}TR=%kInJJ4!tCr~dnWnafc8e?<;X zp5aPMn%%pflOnEM=@m+ud&p+fNYaiNK93wFsz6`w1lHF7lp`o-oYmTuhntlA%{LZs z@KCQzUewx#JJvNfKTndGS{^%hK8H!BWYOdqf36DTps}fwF6uN* z^Z^;thgMVF?m{plJbg2nv~bytf223DjLhHy7^&nR(&v5Zjg2*tQz>434N@di%64$D z6z>j3Ff;QxdV0PQE(q5ote{JqOfHJd;MEkJPN<48j~Oe-iZX~2)~Y;zhS?z1`gFQy zH#vpczoup%Vl}O3e{Af;)YK(K1|Ua>C-Ylk3{6ePG)3=u&D)(Yfpu_`n<2C|k*V>r z@B;XZ94ExBb{7vB%P6PLyZDNvaQgHVJ(Fl?IE2N;o0|2&wo(8KiNq*edgH|7FVn~* zP>{W#OB;!7r-6)w^+T8dt)`gA%E~W@#rA5Fe*{_QAX-`}f7I$~LVABr1|K1QhCVrl z?(Q#8T^$$RpGNNhaR2_V7$1KhU0p+H+0!M7%!kd79O*~2tXf8FFS}x)!zl$l6jrOb zo0zzO!-xA&Ru+>Y<+78{(WB?FzWx{G<^ufj{K#T{VH~+!7S&bF!sN(QtSL-b7zQz8 zW9R9$w*x0me|$z4YYDe*jnJ#^aqQdMC9k`mr#}@Z{pZ7?qy1G`MW#b;gm%)!^z^Wu z)8ynu960bA`upcdzJ%0+W$cf?NV^ip#}}WX9m$fFUIcPfe^9t@~d>OUn!5n4tS{{rZPkrCsvif1tkZAo}_y^w`7It8d}PjSuw5 zz(qS1+Sc|8y%HN~alXX0Ywsbwwk$b$64KILdG?tB+MNOMT3cJ8{NG`8^ljNGo;%y} z36S#Oa5fdZ)5 z4__&55P(WzBn2ZC$5I;OBju7vt>;XlZ$KzKLJYF^LI-M75$W-lViG np.ndarray: + img = inp_img[:, :, :] + if img.shape[0] == self._hud_mask.shape[0] and img.shape[1] == self._hud_mask.shape[1]: + img = cv2.bitwise_and(img, img, mask=self._hud_mask) + # In order to not filter out highlighted items, change their color to black + highlight_mask = color_filter(img, self._config.colors["item_highlight"])[0] + img[highlight_mask > 0] = (0, 0, 0) + # Cleanup image with erosion image as marker with morphological reconstruction + gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + thresh = cv2.threshold(gray, 15, 255, cv2.THRESH_BINARY)[1] + kernel = np.ones((3, 3), np.uint8) + marker = thresh.copy() + marker[1:-1, 1:-1] = 0 + while True: + tmp = marker.copy() + marker = cv2.dilate(marker, kernel) + marker = cv2.min(thresh, marker) + difference = cv2.subtract(marker, tmp) + if cv2.countNonZero(difference) <= 0: + break + mask_r = cv2.bitwise_not(marker) + mask_color_r = cv2.cvtColor(mask_r, cv2.COLOR_GRAY2BGR) + img = cv2.bitwise_and(img, mask_color_r) + return img + + def crop(self, inp_img: np.ndarray, padding_y: int = 5) -> list[ItemText]: + start = time.time() + cleaned_img = self.clean_img(inp_img) + debug_str = f" | clean: {time.time() - start}" + + # Cluster item names + start = time.time() + item_clusters = [] + for key in self._item_colors: + _, filtered_img = color_filter(cleaned_img, self._config.colors[key]) + filtered_img_gray = cv2.cvtColor(filtered_img, cv2.COLOR_BGR2GRAY) + blured_img = np.clip(cv2.GaussianBlur(filtered_img_gray, self._gaus_filter, cv2.BORDER_DEFAULT), 0, 255) + contours = cv2.findContours(blured_img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + contours = contours[0] if len(contours) == 2 else contours[1] + for count, cntr in enumerate(contours): + x, y, w, h = cv2.boundingRect(cntr) + expected_height = 1 if (self._expected_height_range[0] < h < self._expected_height_range[1]) else 0 + # increase height a bit to make sure we have the full item name in the cluster + y = y - padding_y if y > padding_y else 0 + h += padding_y * 2 + cropped_item = filtered_img[y:y+h, x:x+w] + # save most likely item drop contours + avg = int(np.average(filtered_img_gray[y:y+h, x:x+w])) + contains_black = True if np.min(cropped_item) < 14 else False + expected_width = True if (self._expected_width_range[0] < w < self._expected_width_range[1]) else False + mostly_dark = True if 4 < avg < 25 else False + if contains_black and mostly_dark and expected_height and expected_width: + # double-check item color + color_averages=[] + for key2 in self._item_colors: + _, extracted_img = color_filter(cropped_item, self._config.colors[key2]) + extr_avg = np.average(cv2.cvtColor(extracted_img, cv2.COLOR_BGR2GRAY)) + color_averages.append(extr_avg) + max_idx = color_averages.index(max(color_averages)) + if key == self._item_colors[max_idx]: + item_clusters.append(ItemText( + color_key=self._item_colors[max_idx], + roi=[x, y, w, h], + data=cropped_item + )) + debug_str += f" | cluster: {time.time() - start}" + # print(debug_str) + return item_clusters + + +if __name__ == "__main__": + import keyboard + import os + from screen import Screen + + keyboard.add_hotkey('f12', lambda: os._exit(1)) + cropper = ItemCropper() + screen = Screen(cropper._config.general["monitor"]) + + while 1: + img = screen.grab().copy() + res = cropper.crop(img) + for cluster in res: + x, y, w, h = cluster.roi + cv2.rectangle(img, (x, y), (x+w, y+h), (0, 255, 0), 1) + cv2.imshow("res", img) + cv2.waitKey(1) diff --git a/src/item/item_finder.py b/src/item/item_finder.py index 92887dd..3575bc1 100644 --- a/src/item/item_finder.py +++ b/src/item/item_finder.py @@ -7,6 +7,7 @@ from dataclasses import dataclass import math from config import Config from utils.misc import color_filter, cut_roi +from item.item_cropper import ItemCropper @dataclass @@ -26,6 +27,7 @@ class Item: class ItemFinder: def __init__(self): config = Config() + self._item_cropper = ItemCropper() # color range for each type of item # hsv ranges in opencv h: [0-180], s: [0-255], v: [0, 255] self._template_color_ranges = { @@ -37,16 +39,7 @@ class ItemFinder: "unique": [np.array([23, 80, 140]), np.array([23, 89, 216])], "runes": [np.array([21, 251, 190]), np.array([22, 255, 255])] } - self._game_color_ranges = { - "white": config.colors["white"], - "gray": config.colors["gray"], - "magic": config.colors["blue"], - "set": config.colors["green"], - "rare": config.colors["yellow"], - "unique": config.colors["gold"], - "runes": config.colors["orange"] - } - self._gaus_filter = (17, 5) + self._folder_name = "items" self._min_score = 0.86 # load all templates @@ -79,37 +72,12 @@ class ItemFinder: def search(self, inp_img: np.ndarray) -> List[Item]: img = inp_img[:,:,:] start = time.time() - # Pre filter black and highlight - mask1, _ = color_filter(img, self._config.colors["black"]) - mask2, _ = color_filter(img, self._config.colors["item_highlight"]) - filtered_img = cv2.bitwise_or(mask1, mask2) - contours = cv2.findContours(filtered_img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - contours = contours[0] if len(contours) == 2 else contours[1] - new_img = np.zeros(img.shape, np.uint8) - for cntr in contours: - x, y, w, h = cv2.boundingRect(cntr) - new_img[y:y+h, x:x+w] = img[y:y+h, x:x+w] - img = new_img - # Filter by item colors - filtered_img = np.zeros(img.shape, np.uint8) - for key in self._game_color_ranges: - _, extracted_img = color_filter(img, self._game_color_ranges[key]) - filtered_img = cv2.bitwise_or(filtered_img, extracted_img) - filtered_img_gray = cv2.cvtColor(filtered_img, cv2.COLOR_BGR2GRAY) - # Cluster item names - cluster_img = np.clip(cv2.GaussianBlur(filtered_img_gray, self._gaus_filter, cv2.BORDER_DEFAULT), 0, 255) - contours = cv2.findContours(cluster_img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - contours = contours[0] if len(contours) == 2 else contours[1] + item_text_clusters = self._item_cropper.crop(img, 7) item_list = [] - for cntr in contours: - x, y, w, h = cv2.boundingRect(cntr) - x -= 5 - y -= 5 - w += 10 - h += 10 + for cluster in item_text_clusters: + x, y, w, h = cluster.roi # cv2.rectangle(inp_img, (x, y), (x+w, y+h), (0, 255, 0), 1) - - cropped_input = filtered_img[y:y+h, x:x+w] + cropped_input = cluster.data best_score = None item = None for key in self._templates: @@ -132,10 +100,11 @@ class ItemFinder: if template.blacklist: item = None else: - max_loc = [max_loc[0] + x, max_loc[1] + y] # Do another color hist check with the actuall found item template - cropped_roi = [*max_loc, template.data.shape[1], template.data.shape[0]] - cropped_item = cut_roi(filtered_img, cropped_roi) + # TODO: After cropping the "cropped_input" with "cropped_item", check if "cropped_input" might need to be + # checked for other items. This would solve the issue of many items in one line being in one cluster + roi = [max_loc[0], max_loc[1], template.data.shape[1], template.data.shape[0]] + cropped_item = cut_roi(cropped_input, roi) grayscale = cv2.cvtColor(cropped_item, cv2.COLOR_BGR2GRAY) _, mask = cv2.threshold(grayscale, 0, 255, cv2.THRESH_BINARY) hist = cv2.calcHist([cropped_item], [0, 1, 2], mask, [8, 8, 8], [0, 256, 0, 256, 0, 256]) @@ -143,10 +112,10 @@ class ItemFinder: same_type = hist_result > 0.65 and hist_result is not np.inf if same_type: item = Item() - item.center = (int(max_loc[0] + int(template.data.shape[1] * 0.5)), int(max_loc[1] + int(template.data.shape[0] * 0.5))) + item.center = (int(max_loc[0] + x + int(template.data.shape[1] * 0.5)), int(max_loc[1] + y + int(template.data.shape[0] * 0.5))) item.name = key item.score = max_val - item.roi = [*max_loc, template.data.shape[1], template.data.shape[0]] + item.roi = [max_loc[0] + x, max_loc[1] + y, template.data.shape[1], template.data.shape[0]] center_abs = (item.center[0] - (inp_img.shape[1] // 2), item.center[1] - (inp_img.shape[0] // 2)) item.dist = math.dist(center_abs, (0, 0)) if item is not None and self._config.items[item.name]: @@ -168,10 +137,10 @@ if __name__ == "__main__": img = screen.grab().copy() item_list = item_finder.search(img) for item in item_list: - print(item.name + " " + str(item.score)) + # print(item.name + " " + str(item.score)) cv2.circle(img, item.center, 5, (255, 0, 255), thickness=3) cv2.rectangle(img, item.roi[:2], (item.roi[0] + item.roi[2], item.roi[1] + item.roi[3]), (0, 0, 255), 1) - cv2.putText(img, item.name, item.center, cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 255), 1, cv2.LINE_AA) + # cv2.putText(img, item.name, item.center, cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 255), 1, cv2.LINE_AA) # img = cv2.resize(img, None, fx=0.5, fy=0.5) cv2.imshow('test', img) cv2.waitKey(1) diff --git a/src/utils/item_extractor.py b/src/utils/item_extractor.py new file mode 100644 index 0000000..4a3be89 --- /dev/null +++ b/src/utils/item_extractor.py @@ -0,0 +1,62 @@ +""" +Script to autocrop items. Input image with items in the correct resolution and the script will auto crop it for you and ask for names for each of them. +""" +import argparse +import os +import cv2 +import numpy as np +from config import Config +from utils.misc import color_filter +from item.item_cropper import ItemCropper +import time + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Script to autocrop items.") + parser.add_argument("--file_path", type=str, help="Path to screenshots e.g. C:/data") + args = parser.parse_args() + + args.file_path = "C:\\Users\\aliig\\Desktop\\bot\\botty-gleed-ocr\\input_images" + gen_truth = 1 + + + item_cropper = ItemCropper() + + for filename in os.listdir(args.file_path): + if filename.endswith(".png"): + start = time.time() + inp_img = cv2.imread(f"{args.file_path}\\{filename}") + filename = filename[:-4] + img = inp_img[:,:,:] + img_clean = item_cropper.clean_img(img) + item_clusters = item_cropper.crop(img) + for count, cluster in enumerate(item_clusters): + x, y, w, h = cluster.roi + key = cluster.color_key + if gen_truth: + cv2.namedWindow("item") + cv2.moveWindow("item", 100, 100) + cv2.imshow("item", img_clean[y:y+h, x:x+w]) + cv2.waitKey(1) + print(f"{count} Input item name and press enter (converts to all caps)...") + item_name = input() + if item_name != "": + out_filename = f"{key}_{item_name.replace(' ','_')}" + if not os.path.exists(f"./ground_truth/{out_filename}.png"): + cv2.imwrite(f"./ground_truth/{out_filename}.png", img_clean[y:y+h, x:x+w]) + file1 = open(f"./ground_truth/{out_filename}.gt.txt","w") + file1.write(item_name.upper()) + file1.close() + else: + print("Skipping") + time.sleep(0.1) + cv2.destroyAllWindows() + else: + avg = int(np.average(cv2.cvtColor(cluster.data, cv2.COLOR_BGR2GRAY))) + cv2.imwrite(f"./generated/z_{filename}_{key}_{count}_{avg}.png", cluster.data) + cv2.rectangle(inp_img, (x, y), (x+w, y+h), (0, 255, 0), 1) + cv2.putText(inp_img, key, (x+5, y+5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1, cv2.LINE_AA) + + finish=time.time() + print(f"{filename} total: {finish-start}s") + cv2.imwrite(f"./generated/{filename}.png", inp_img)