
    Bh                        d dl mZmZ d dlmZmZ d dlmZ d dlZ	d dl
m
Z
 d dlmZmZ d dlmZmZmZmZmZmZmZmZmZ d dlmZ d dlZd dlZd d	lmZ d dlZ d dl!Z!d d
l"m#Z#m$Z$m%Z% d dl&m'Z' d dl(Z(d dlZd dl)Z)d dl*Z*d dl+m,Z, d dl-Z.d dl-m/Z/m0Z0 d dl1Z1d dl2Z2d dl3m4Z4 d dl5Z5d dl6Z6d dlZd dlm7Z7m8Z9 d dlm:Z: d dl;Z;d dl<Z<d dl=Z=d dl>m?Z? d dl@mAZA d dlBmCZC ddlDmEZEmFZF d dl*mGZG d dlHZHd dlIZId dl1Z1 eG       ZJ e5j                  e6j                  e5j                          eCeN      j                  dz  ZP eeP        edeQ      ZR e         eeRdddii        e5j                  de5j                         d ZTd!ZUd!ZV e/d"#      ZWi ZXdZY G d$ d%eZ      Z[ G d& d'eZ      Z\ G d( d)e]      Z^d* Z_d+ Z`d, Zad- Zbd. ZcdCd/Zdd0 Zed1 Zf eHj                  d2d3d d45      Zhd6 ZieRj                  d7d8g9      d:        Zkd; ZleRj                  d<d8g9      d=        Zmd> Znd? Zo e5j                  de5j                         eRj                  d@dAg9      dB        Zpy)D    )ImageImageOps)joinbasename)walkN)progressbar)choiceshuffle)		BlueprintFlaskrender_templaterequestredirecturl_forflashjsonifymake_response)CORS)load_dotenv)ClientErrorNoCredentialsErrorPartialCredentialsError)BytesIO)Queue)ProcessPoolExecutoras_completed)Config)connectError)pooling)get_session)TransferConfig)Path   )get_shared_pathget_shared_url)Lock)streamlevelz.envsnmo_appz/*origins*)	resourcesz	error.log)filenamer)   iL  
      max_workersc                       e Zd Zy)S3ImagesInvalidExtensionN__name__
__module____qualname__     I/var/www/fastuser/data/www/generator.snapmosaic.io/flask_app/app/views.pyr4   r4          r:   r4   c                       e Zd Zy)S3ImagesUploadFailedNr5   r9   r:   r;   r>   r>      r<   r:   r>   c                   ,    e Zd ZddZd Zd Zd Zd Zy)S3ImagesNc                 T    d| _         t        j                  | j                   d       y )N)/var/www/fastuser/data/www/shared/uploadsTexist_ok)shared_folderosmakedirs)selfaws_access_key_idaws_secret_access_keyregion_names       r;   __init__zS3Images.__init__   s    H
D&&6r:   c                     	 t         j                  j                  | j                  |      }t	        j
                  |      }||fS # t        $ r}t        d| d|       d }~ww xY w)NFailed to retrieve image  from shared folder: )rF   pathr   rE   r   open	Exceptionr>   )rH   bucketkey
local_pathimges         r;   _download_imagezS3Images._download_image   sg    	bd&8&8#>J**Z(C8O 	b&)B3%G\]^\_'`aa	bs   AA 	A$AA$c           
      4   |j                  d      rg }	 t        j                  j                  | j                  |      }g }t        j
                  |      D ]  \  }}}|D ]  }	|	j                         j                  d      s#t        j                  j                  ||	      }
t        j                  j                  |
| j                        }|j                  |t        j                  j                  |
      d         |j                  d d       t        j                  j                  d      5 }|D 	ci c]#  }	|j                  | j                  ||	d	         |	% }}	t        j                  j!                  |      D ]!  }|j                  |j#                                # 	 d d d        |S 	 | j                  ||      gS c c}	w # 1 sw Y   |S xY w# t$        $ r}t'        d
|       d }~ww xY w# t$        $ r}t'        d| d|       d }~ww xY w)N/)z.jpgz.jpeg.png)KeyLastModifiedc                     | d   S )Nr]   r9   )xs    r;   <lambda>z"S3Images.from_s3.<locals>.<lambda>   s    !N2C r:   T)rT   reverser/   r1   r\   z.Failed to retrieve images from shared folder: rN   rO   )endswithrF   rP   r   rE   r   lowerrelpathappendgetmtimesort
concurrentfuturesThreadPoolExecutorsubmitrX   r   resultrR   r>   )rH   rS   rT   images	full_pathmatching_filesroot_filesfile	file_pathrel_pathexecutorri   futurerW   s                   r;   from_s3zS3Images.from_s3   s   <<FaGGLL););SA	!#&(ggi&8 ND!U % ::<001JK(*T4(@I')wwy$BTBT'UH*11'/020@0@0K3 	 ##(CT#R''::r:J 7h %3  !(<(<fd5kRTXXG  #-"4"4"A"A'"J 7fmmo677 Mf,,VS9::7 M  a*-[\][^+_``a  f*-FseK`ab`c+deefsh   A,G B3G 6G;(G#AG)G 4G8 GGG G 	G5"G00G58	HHHc                    	 t         j                  j                  | j                  |      }t        j                  t         j                  j                  |      d       |j                  d      d   j                         }|dvrt        d|       |j                  ||       y # t        $ r}t        d| d|       d }~ww xY w)	NTrC   .)JPGJPEGPNGDZIz Invalid image extension for key zFailed to upload image z to shared folder: )rF   rP   r   rE   rG   dirnamesplitupperr4   saverR   r>   )rH   rV   rS   rT   rn   
img_formatrW   s          r;   to_s3zS3Images.to_s3   s    
	^T%7%7=IKK	2TB3+113J!>>.1QRUQV/WXXHHY
+ 	^&)@EXYZX['\]]	^s   B$B' '	C0CCc                 X   t         j                  j                  d      5 }t        ||      D cg c]#  \  }}|j	                  | j
                  |||      % }}}t         j                  j                  |      D ]  }|j                           	 d d d        y c c}}w # 1 sw Y   y xY w)Nr/   r1   )rh   ri   rj   ziprk   r   r   rl   )	rH   rm   rS   keysrv   rV   rT   ri   rw   s	            r;   parallel_upload_imageszS3Images.parallel_upload_images   s    22r2B 	 h&)&$&79"#s  tzz3D 9G 9$,,99'B   	  	 9	  	 s   B (B7B B  B))NNN)r6   r7   r8   rL   rX   rx   r   r   r9   r:   r;   r@   r@      s    7
b!fF^ r:   r@   c                 "   | j                   \  }}||z  }||z  }||kD  r&||z  }||z
  dz  }| j                  |d||z   |f      } n*||k  r%||z  }	||	z
  dz  }
| j                  d|
||
|	z   f      } | j                  ||ft        j                        S )Nr0   r   )sizecropresizer   LANCZOS)rV   new_xnew_yold_xold_y	old_ratio	new_ratior_   dxydys              r;   crop_centerr      s    88LE5II9aiAhhArAvu-.	Y	IaiAhh2ub1f-.::uenemm44r:   c              #   ^   K   t        |       D ]  \  }}}|D ]  }t        ||         y wN)r   r   )rP   rp   dirsrr   rs   s        r;   find_imagesr      s<     !$Z #dE 	#DtT""	##s   +-c                 F    t        d | j                  d      D              S )Nc              3   2   K   | ]  }t        |        y wr   )round).0vs     r;   	<genexpr>z$get_average_color.<locals>.<genexpr>  s     9aq9s   )r   r$   )axis)tuplemean)rV   s    r;   get_average_colorr     s    93888#8999r:   c                     g }| D ]n  \  }}t        j                  t        t        j                  |      j                  d      t        t                    }t        |      }|j                  |||f       p |S )NRGB)
npasarrayr   r   exif_transposeconvertTILE_X_SIZETILE_Y_SIZEr   re   )tilestile_imagesrV   nametilecolors         r;   
load_tilesr     sp    K 0	Tzz+h&=&=c&B&J&J5&QS^`klm!$'E4./	0
 r:   c                     g }| d   d   }| D ]-  \  }}}}||k(  r||vr|j                  ||||f       '||kD  s- n |s| }t        j                  |      S Nr   )re   randomr	   )rankedrecent_tilesbest	best_distdist	avg_colorr   r   s           r;   get_bestr     st    Dq	!I'- #it9\!9KKy$56I	 ==r:   c                 :   | D cg c]H  \  }}}t        |d   |d   z
        t        |d   |d   z
        z   t        |d   |d   z
        z   |||fJ }}}}|j                  d        |rj|rh|D 	cg c]  }	|	d   |D cg c]  }|d   	 c}v s|	 }
}	}|
r*t        |
|      \  }}}}|D cg c]  }|d   |k7  s| }}n#t        ||      \  }}}}nt        ||      \  }}}}t        j                  |d   |d   z
  |d   |d   z
  |d   |d   z
  ft        j
                        }t        j                  |      j                  dd      j                  t        j                        }t        j                  |      ||fS c c}}}w c c}w c c}}	w c c}w )	Nr   r$   r0   c                     | d   S r   r9   )r   s    r;   r`   zget_tile.<locals>.<lambda>0  s
    ad r:   rT      )dtype   )absrg   r   r   arrayint16addclipastypeuint8r   	fromarray)r   r   r   unused_source_3_tilesprioritize_source_3r   r   r   r   itemsource_3_rankedr   diffs                r;   get_tiler   %  s    &+  "ItT a9Q<'(3uQx)A,/F+GG#eTUhYbcdYeNeJff		
F  KKNK# 4,2mDd1gVkAld$q'Al6l4mm*2?L*Q'D)T46K$_dtTUwZ^T$_!$_*26<*H'D)T4&.v|&D#it 88U1X	!,eAh1.EuQxR[\]R^G^_gigogopD66$""1c*11"((;D??4 $(===7 Bmm %`s0   AF4FFFF0F>FFc                    t        j                  d|t        z  |t        z  f      }t	        |      D cg c]  }t	        |      D ]  }||f  }}}g }g }	d}
|D cg c]  }|
|d   v s| }}|D cg c]  }|
|d   vs| }}|D ]  \  }}|t        z  }|t        z  }|r&t        || j                  ||f      ||d      \  }}}n%t        || j                  ||f      ||d      \  }}}t        ||       |j                  |||f       |j                  |       |j                  d      d   }|	j                  |||f       t        |      t        kD  s|j                  d	        ||	fS c c}}w c c}w c c}w )
Nr   3r$   T)r   FrZ   r{   r   )r   newr   r   ranger   getpixelprintpastere   r   lenRECENT_TILE_WINDOWpop)imager   mosaic_widthmosaic_heightmosaicr_   r   coordsr   tile_positionssource_3_identifierr   r   non_source_3_tilesx_posy_posr   r.   s                     r;   generate_mosaicr   {  s   YYu|k9=;;VWXF#L1P5;OPaq!fPfPFPLN  /4Vd7JdSTg7UTVV+0W44GtTUw4V$WW  1KK !08PQSTv@VXdf{  RV  1W-D$-089KU^^]^`a\bMceq  tI  _d  1e-D$-dDTE5>*D!::c?2&xA./|11Q% ( >!!A Q WWs   E1E7*E74E<E<c           
      h   	 t        j                  d      }t        j                  d      }t        j                  d      }t        j                  d      }t        t        j                  d            }t        |||||      }|j	                         }d}	t        |d 	      }
|
D cg c]  \  }}}|||| f }}}}t        j                  d
t        |       d|d d         d}d}d}t        dt        |      |      D ]M  }||||z    }t        |      D ]5  }	 |j                  |	|       |j                          ||j                  z  } M O t        j$                  d| d|         	 dt+               v r|j-                          dt+               v r|j-                          y y c c}}}w # t        $ rZ}dt        |      v rG||dz
  k  r?t        j                  d||z  dz    d|dz    d       t!        j"                  d       Y d }~ d }~ww xY w# t        $ r'}t        j&                  dt        |               d }~wt(        $ r'}t        j&                  dt        |               d }~ww xY w# dt+               v rj-                          dt+               v rj-                          w w xY w)NdbHostdbUserdbPassdbNamedbPorthostportuserpassworddatabasez
            UPDATE `tiles_image_fixed_info`
            SET x = %s, y = %s
            WHERE tile_name = %s AND project_name = %s;
        c                     | d   | d   fS )Nr   r$   r9   )ts    r;   r`   z'insert_tile_positions.<locals>.<lambda>  s    adAaD\ r:   r   z	Prepared z update entries. Sample: r0   i  r   r   zDeadlock foundr$   z"Deadlock detected. Retrying batch z
 (Attempt )zSuccessfully updated z tile positions for project: z)MySQL Error during tile position update: z,General error during insert_tile_positions: cursordb_connection)rF   getenvintr   r   sortedloggingdebugr   r   executemanycommitrowcount
MySQLErrorstrwarningtimesleepinfo	exceptionrR   localsclose)project_namer   db_hostdb_userdb_passdb_namedb_portr   r   update_querysorted_tile_positions	tile_namer_   r   update_data
batch_sizetotal_updatedmax_retriesibatchattemptrW   s                         r;   insert_tile_positionsr    s   @"))H%))H%))H%))H%bii)*
 %%' !'~;Q R Oddd?9a1i6dd	#k"2!33L[Y[Z[_L]^_
q#k*J7 	A!j.1E - &&|U;!((*!V__4M	  	,]O;XYeXfgh vxLLNfh&! 'K e  " '3q61ga6O*LQR\__`M`Laaklsvwlwkxxy(z{

1   Ec!fXNO HQQR
 vxLLNfh&! 'st   B(H *F(;A H 1F/ H (H /	H8AHH HHH 	I3"I  I3"I..I33I6 6;J1	localhosti  T)r   r   dbdecode_responsesc                    d}|j                  d      d   }t        j                  j                  ||      }t        j                  |d       | d}| d}	t        j                  j                  ||      }
t
        j                  j                  | d	      }t
        j                  j                  ||
|	d
dd      }t        j                  |        y)NrB   rZ   r$   TrC   z_output.dzi_output
sequential)accessr[   r0   `   )r   suffixoverlap	tile_sizeSuccess)
r   rF   rP   r   rG   pyvipsr   new_from_filedzsaveremove)input_image_pathoutput_folderaws_credentialsbucket_name	s3_prefixrE   r
  base_output_diroutput_dzi_nameoutput_basenameoutput_dzi_pathr   dzs                r;   run_vips_commandr5    s    ?M??3'*Lggll=)<OKK$/%k2O%g.Oggll?ODOLL&&'7&ME			UO$3"(#$%'	 
 
)B IIr:   z/task_statusGET)methodsc                     dd l } i }	 t        j                  d      }|D ]C  }	 t        j                  |      }|r) | j                  |      }|j                  dd      }|||<   E 	 t        |      S # t
        $ r%}t        j                  d| d|        Y d }~{d }~ww xY w# t
        $ r,}t        j                  d|        Y d }~t        |      S d }~ww xY w)Nr   r,   statusUnknownzError reading Redis key z: zError fetching task statuses: )	jsonredis_clientr   getloadsrR   r   errorr   )r;  task_statusesr   rT   valuedecoded_valuer9  rW   s           r;   task_statusrC    s    M<  % 		C$((-$.DJJu$5M*..xCF)/M#&		 =!!   8RsCD  <6qc:;;=!!<sA   B" A A1#B" 1	B:BB" BB" "	C+CCc                  &   	 t         j                  d      } | D ]/  }t         j                  |       t        j                  d|        1 dt        |       iS # t        $ r/}t        j                  d|        dt        |      icY d }~S d }~ww xY w)Nzsnmo-project-*zDeleted Redis key: deleted_keyszError clearing Redis keys: r?  )	r<  r   deleter   r  r   rR   r?  r  )r   rT   rW   s      r;   clear_task_statusesrG    s    !  !12 	6C$LL.se45	6 D	** !3A378Q  !s   AA 	B!$BBBz/clear_task_statusesc                  ,    t               } t        |       S r   )rG  r   )rl   s    r;   clear_task_statuses_routerI    s     "F6?r:   c                    	 t        j                  d      }t        j                  d      }t        j                  d      }t        j                  d      }t        t        j                  d            }t        |||||      }|j	                         }d}d| d	}	t        d
|	        |j                  |	| f       |j                         |j                          |j                          S # t        $ r:}
t        d|
        g cY d }
~
j                          j                          S d }
~
ww xY w# j                          j                          w xY w)Nr   r   r   r   r   r   tiles_image_fixed_infozSELECT * FROM `z` WHERE project_name = %s AND (queued IS NULL OR queued = '' OR queued = '0' OR queued = 0) AND tile_terms_and_condition = '1';zExecuting query: An error occurred: )
rF   r   r   r   r   r   executefetchallr	  r  )r
  r  r  r  r  r  r   r   tiles_tablequeryrW   s              r;   get_tiles_from_dbrQ    s.   !))H%))H%))H%))H%bii)*
 %%' / "+  /n  o!%)*u|o.  	  #A3'(	 	s0   C
C- -	D06D+D0D3 +D00D3 3"Ec                    t        |      }d}|D cg c]  }t        |d         |k(  s|d   |d   f  }}|D cg c]  }t        |d         |k7  s|d   |d   f  }}d| d| d}|j                  | |      }	g }
|D ]0  \  }}|	D ]&  \  }}t        |      |v s|
j                  ||f       ( 2 |
s5|D ]0  \  }}|	D ]&  \  }}t        |      |v s|
j                  ||f       ( 2 |
S c c}w c c}w )	Nr      r$   r0   	projects/z/tiles-pos-rZ   )rS   rT   )rQ  r  rx   re   )rS   r
  	s3_imagestiles_from_dbr   r   source_3_tilesother_tilestile_key_prefixall_tiles_s3r   r  rq   rV   rT   s                  r;   load_tiles_from_s3_and_dbr[  A  sD   %l3M 5BjTc$q'lViFitAwQ(jNj2?g$3tAw<SfCfDGT!W%gKg ",{<.JO$$F$HLK ' /	1$ 	/HC9~$""C:.	// ' 	3LIq( 3Sy>S(&&Sz23	3
 - kgs   C,C,C1C1z/generate_mosaicPOSTc                  	  ! dd l !dd l} 	  | j                         }t        j                  d       t	               }t
        j                  j                  d      }t
        j                  j                  d      }t        j                  j                  |      }t        j                  |      }|j                  d      }t        ||f      }t        j!                  | !j"                  ddi             t%        t'        j(                  dd	            }t*        j,                  j/                  t'        j(                  d
      |t'        j(                  d      t'        j(                  d      t'        j(                  d            }	|	j1                  d      }
|
j3                  d||f       |
j5                         }|st7        ddi      dfS |d   d   }t%        t'        j(                  d            }|xaa|j=                  dd| d|       }|d   d   }t?        d||      }|j@                  |jB                  z  }tD        t:        z  }t%        tD        |z  t8        z        }tG        |||      }tI        |      }tK        ||||      \  }}d| d| d}|jM                  |d|       d| d}d}t&        jN                  jQ                  ||      }|jS                  |       tT        jW                  tX        || dddd| d      }||ftZ        |<   !fd }|j]                  |       t        j!                  | !j"                  dd!i             t_        ||       t&        jN                  ja                  |      rt'        jb                  |        | j                         }te        ||z
  d"      }t        j                  d#| d$| d%te        |d"              tg        t7        dd&d'      d(      }d)|jh                  d*<   |S # tj        $ r} t        jl                  d+t        |        d,       d-to               v r4t&        jN                  ja                        rt'        jb                  |       t7        dt        |       i      d.fcY d } ~ S d } ~ ww xY w)/Nr   zStarting mosaic generationr
  user_idzutf-8r9  Startedr   3306r   r   r   r   r   T)
dictionaryzESELECT * FROM snmo_projects WHERE user_id = %s AND project_slug = %s;r?  zProject not foundi  
main_imageTILE_IMAGE_DENSITYignoredrT  rZ   z_mosaic.pngtemp_mosaic_r[   z/tmpz_output/)rd  rd  rd  c                    	 t         j                  | d      \  }}t        ||f      }	 | j                          t        j                  | j                  ddi             | t         v rt         | = y y # t        $ rV}t        j                  d|        t        j                  | j                  ddt        |       i             Y d }~kd }~ww xY w# | t         v rt         | = w w xY w)N)NNr9  	CompletedzTask failed: zFailed: )
future_to_taskr=  r  rl   r<  setdumpsrR   r   r?  )fut	proj_nameusr_idtask_keyrW   r;  s        r;   update_statusz,generate_mosaic_route.<locals>.update_status  s    ,$2$6$6sL$I!	6	623\JJL $$Xztzz8[:Q/RS
 .(&s+ )	 ! \MMM!"56 $$Xztzz8xPSTUPVxEX:Y/Z[[\ .(&s+ )s/   &C 8A3 3	C<ACC CC C'zIn Progressr0   [z] Mosaic generated in z sec with aspect ratio: zMosaic generation started)successmessage   z
keep-alive
ConnectionrL  )exc_infomosaic_output_image_pathi  )8r;  r  r   r  r@   r   formr=  urllibparseunquotebase64	b64decodedecoder  r<  ri  rj  r   rF   r   mysql	connectorr   r   rM  rN  r   r   r   rx   r[  widthheightOUTPUT_HEIGHTr   r   r   r   rP   r   r   rv   rk   r5  rh  add_done_callbackr  existsr*  r   r   headersrR   r?  r  )"r  
start_timerm   r
  r^  url_decoded_stringdecoded_bytestask_idr  r   r   resultsimage_filenamer%  
image_datar   r   aspect_ratior   r   r   r   r   
mosaic_keyoutput_filename
output_dirrv  rw   ro  end_timedurationresponserW   r;  s"                                    @r;   generate_mosaic_router  a  s   m/TYY[
12||''7,,""9-#\\11,?(();<$++G4|W-.*$**h	-B"CDbii&12//8$8$YYx(YYx( 0 
 %%%6Sl#	
 //#G%89:C?? L1		"678	$--k^^I<..IY/Z[
1a ))\6J{{U\\1%4ML8[HIE<? '!0\S`!a a~[I
VY
3(d;
#%77<<
O#L ,-$nH%-~Q'
 #/!8v	, 	  /*$**h-F"GHlN;77>>23II./499;J.2q&<XJF^_deqst_u^vwx TFa)b!cehi)5& /+CF84tD%1bggnnE]6^II./Q()3..	/s&   F=O< 	H2O< <	R
A:R?R
R
)F)qPILr   r   os.pathr   r   rF   r   numpyr   r   r   r	   r
   flaskr   r   r   r   r   r   r   r   r   
flask_corsr   r{  urllib.parserx  dotenvr   mysql.connectorr~  boto3botocore.exceptionsr   r   r   ior   tempfile
subprocess	threadingqueuer   concurrent.futuresrh   r   r   r  r'  botocore.configr   r   sysr   r   r  r    mathasyncioaiobotocoreaiobotocore.sessionr!   boto3.s3.transferr"   pathlibr#   utils.pathsr%   r&   r'   redisr;  future_lockbasicConfigstdoutDEBUG__file__parentenv_pathr6   bpERRORr  N_TILESr   rv   rh  rw   rR   r4   r>   objectr@   r   r   r   r   r   r   r   r  Redisr<  r5  routerC  rG  rI  rQ  r[  r  r9   r:   r;   <module>r     s2    "   # " n n n       X X   	     @   "  
  8 #    + ,  8     f   3::W]] ;>  6) H  z8$ 
 REIs+, -   [ > 
 1-	j	y 		9 	C v C L5#
:
&>l""`A"R u{{$1tT. .5'*" +",	! 
 5'2 3X"D<   [ >
vh/p/ 0p/r:   