
    xh(                         S SK r S SKrS SKJr  S SKJr  S SKJr  S SKJ	r
  S SKJrJr  S SKJr  SS	KJr  S
 r " S S\5      r " S S\5      r	 " S S\R(                  5      r " S S\5      rg)    N)ThreadPoolExecutor)sha1)sync_to_async)File)modelstransaction)CreatedModel   )AssembleChecksumMismatchc                     Sn[        5       n U R                  S5      nU(       d  O!U[        U5      -  nUR                  U5        M:  XR	                  5       4$ )Nr   i   )r   readlenupdate	hexdigest)fileobjsizechecksumchunks       /code/apps/files/models.py_get_size_and_checksumr      sX    DvH
U#E
  ##%%%    c                       \ rS rSrSr\R                  " SS9r\R                  " SS9r	\R                  " SSS9r\SS
 j5       r\S 5       rSrg	)FileBlob   a8  
Port of sentry.models.file.FileBlob with simplifications

OSS Sentry stores files in file blob chunks. Where one file gets saved as many blobs.
GlitchTip uses Django FileField and does not split files into chunks.
The FileBlob's provide file deduplication. Multiple File objects may refer to the same
FileBlob.
zuploads/file_blobs)	upload_toT)null(   )
max_lengthuniqueNc                    #    UR                  S5        / nU H>  n[        U[        5      (       a  UR                  U5        M+  UR                  US 45        M@     U H  nU " 5       nUS   nUS   R                  Ul        US   Ul        [        R                  R                  UR
                  S9R                  5       I S h  vN (       a  Mq  [        UR                  R                  5      " UR                  U5      I S h  vN   M     g  NH N7f)NzFileBlob.from_files.startr   r
   )r   )debug
isinstancetupleappendr   r   r   objectsfilteraexistsr   blobsavename)	clsfilesorganizationloggerfiles_with_checksumsr   file_with_checksumr(   	blob_files	            r   
from_filesFileBlob.from_files)   s     01!G'5))$++G4$++WdO<	  #75D*1-I*1-22DI.q1DM!))00$--0HPPRRR#DIINN3INNINNN #7
 SNs*   B?DD
D3D D
DDc                    [        5       nUR                  S5       nUR                  5       (       a)  UR                  5        H  nUR	                  U5        M     OUR	                  UR                  5       5        U R                  R                  UR                  5       XR                  S.S9u  pVSSS5        U$ ! , (       d  f       W$ = f)z:
Retrieve a single FileBlob instances for the given file.
rbr(   r   )r   defaultsN)
r   openmultiple_chunkschunksr   r   r%   get_or_creater   r   )r+   r   r   fr   	file_blob_s          r   	from_fileFileBlob.from_file<   s    
 6\\$1  ""XXZEOOE* ( );;44!++-")<<@ 5 LI      s   BB::
C	 )NN)__name__
__module____qualname____firstlineno____doc__r   	FileFieldr(   PositiveIntegerFieldr   	CharFieldr   classmethodr2   r?   __static_attributes__rA   r   r   r   r      sd     &:;D&&D1D2d;HO O$  r   r   c                       \ rS rSrSr\R                  " 5       r\R                  " SS9r	\R                  " SSS9r\R                  " \\R                  SS9r\R                   " SS	9r\R                  " S
SSS9rS rS r SS jrS rSS jrSrg)r   P   z!
Port of sentry.models.file.File
@   )r   T)blankr   )	on_deleter   r   )defaultr   )r   r   db_indexc                     UR                   U l         [        R                  U5      nUR                  U l        U R	                  5         g)z(Save a Django File object as a File BlobN)r   r   r?   r   r)   )selfr   r=   s      r   put_django_fileFile.put_django_filec   s3    LL	&&w/	!**		r   c                     [        U5      u  p#UR                  S5        [        R                  R	                  S[        XS90UUS9u  pEX0l        X@l        U R                  5         g)z&Save a file-like object as a File Blobr   r(   )r*   )r7   r   r   N)	r   seekr   r%   r;   FileObjr   r(   r)   )rT   r   r   r   r=   r>   s         r   putfileFile.putfilej   s^    /8Q''55gg=> 6 
	
 !			r   Nc                     [        [        R                  R                  U S9R	                  S5      R                  S5      UUUUS9$ )N)filer(   offset)modeprefetchprefetch_todelete)ChunkedFileBlobIndexWrapperFileBlobIndexr%   r&   select_relatedorder_by)rT   r_   r`   ra   rb   s        r   _get_chunked_blobFile._get_chunked_blobw   sH     +!!((d(3^F#Xh#
 	
r   c                 L    U R                  5       n[        XR                  5      $ N)rg   rY   r*   )rT   impls     r   getfileFile.getfile   s    %%'tYY''r   c                 >   [         R                  " 5       n[        R                  " 5          [        R
                  R                  US9R                  5       nU Vs0 s H  ofR                  U_M     nnU Vs/ s H  oU   PM	     nn[        S5      n	Sn
U Hr  n[        R
                  R                  XU
S9  UR                  R                  5        H%  nU	R                  U5        UR                  U5        M'     XR                   -  n
Mt     Xl        U	R#                  5       U l        X R$                  :w  a  ['        S5      e SSS5        U(       a  U R)                  5         UR+                  5         UR-                  S5        U$ s  snf s  snf ! , (       d  f       NR= f)zQ
This creates a file, from file blobs and returns a temp file with the
contents.
)id__inr   r   r]   r(   r^   zChecksum mismatchN)tempfileNamedTemporaryFiler   atomicr   r%   r&   allidr   rd   creater(   r:   r   writer   r   r   r   r)   flushrX   )rT   file_blob_idsr   committf
file_blobsr(   blobs_by_idblob_idnew_checksumr^   r   s               r   assemble_from_file_blob_ids File.assemble_from_file_blob_ids   sS   
 ((*!!))000FJJLJ 6@@ZT77D=ZK@>KLm7g.mJL9LF"%%,,$&,Q!YY--/E ''.HHUO 0 ))# # I(224DM==(./BCC )' ", IIK



	- AL "!s*   0FF1F7F	B<F
F
F)r(   r   r   NFNT)T)rB   rC   rD   rE   rF   r   	TextFieldr*   rI   type	JSONFieldheaders
ForeignKeyr   CASCADEr(   rH   r   r   rU   rZ   rg   rl   r   rK   rA   r   r   r   r   P   s     D r*DT5GXdKD&&q1D2D4HH CG
( r   r   c                       \ rS rSrSr\R                  " \\R                  S9r	\R                  " \
\R                  S9r\R                  " 5       r " S S5      rSrg)rd      z_
Ported from OSS Sentry. Should be removed as GlitchTip does not
split file blobs into chunks.
)rP   c                       \ rS rSrSrSrg)FileBlobIndex.Meta   )rp   rA   N)rB   rC   rD   rE   unique_togetherrK   rA   r   r   Metar      s    7r   r   rA   N)rB   rC   rD   rE   rF   r   r   r   r   r]   r   r(   rH   r^   r   rK   rA   r   r   rd   rd      sM    
 TV^^<DX@D((*F8 8r   rd   c                   t    \ rS rSr SS jrS rS rS rS r\	S 5       r
S	 rSS
 jrS rS rS rSS jrSrg)rc      Nc                     [        U5      U l        S U l        S U l        U(       a  SU l        U R                  XE5        OSU l        X l        U R                  5         g )NTF)list_indexes_curfile_curidx
prefetched	_prefetchr_   r8   )rT   indexesr_   r`   ra   rb   s         r   __init__$ChunkedFileBlobIndexWrapper.__init__   sH     W"DONN;/#DO			r   c                     U $ rj   rA   rT   s    r   	__enter__%ChunkedFileBlobIndexWrapper.__enter__   s    r   c                 $    U R                  5         g rj   )close)rT   exc_type	exc_valuetbs       r   __exit__$ChunkedFileBlobIndexWrapper.__exit__   s    

r   c                     U R                   (       d  [        S5      eU R                  nS U l        U R                  5         UR	                  S5        U$ )Nz*Can only detech tempfiles in prefetch moder   )r   	TypeErrorr   r   rX   )rT   rvs     r   detach_tempfile+ChunkedFileBlobIndexWrapper.detach_tempfile   s>    HII]]



	r   c                 l   U R                   (       a   S5       eU R                  n  [        U R                  5      U l        U R                  R
                  R                  5       U l        Ub  UR                  5         g g ! [         a    S U l        S U l         N/f = f! Ub  UR                  5         f f = f)Nzthis makes no sense)	r   r   next_idxiterr   r(   rl   StopIterationr   )rT   old_files     r   _nextidx$ChunkedFileBlobIndexWrapper._nextidx   s    ??9$99"==		!%#DMM2 $ 1 1 9 9 ;
 #  $	 ! %# $% #  $s$   AA? ?BB BB B3c                 :    [        S U R                   5       5      $ )Nc              3   L   #    U  H  oR                   R                  v   M     g 7frj   r6   ).0is     r   	<genexpr>3ChunkedFileBlobIndexWrapper.size.<locals>.<genexpr>   s     6166;;s   "$)sumr   r   s    r   r    ChunkedFileBlobIndexWrapper.size   s    6666r   c                 4    SU l         U R                  S5        g )NFr   )closedrX   r   s    r   r8    ChunkedFileBlobIndexWrapper.open   s    		!r   c                   ^ U R                   n[        R                  " SXS9nUS:X  a  X@l        g UR	                  US-
  5        UR                  S5        UR                  5         [        R                  " UR                  5       U5      mU4S jn[        SS9 nU R                   H3  nUR                  XWR                  UR                  R                  5        M5     S S S 5        TR                  5         X@l        g ! , (       d  f       N%= f)	Nz._prefetch-)prefixdirrb   r   r
    c                    > U" 5        n UR                  S5      nU(       d  O UTX [        U5      -   & U [        U5      -  n M9  S S S 5        g ! , (       d  f       g = f)Ni  )r   r   )r^   rl   sfr   mems       r   
fetch_file9ChunkedFileBlobIndexWrapper._prefetch.<locals>.fetch_file   sP    bGGENE 8=C#e*!45c%j(F  s   ;A
A   )max_workers)r   rq   rr   r   rX   rw   rx   mmapfilenor   r   submitr^   r(   rl   )	rT   ra   rb   r   r<   r   exeidxr   s	           @r   r   %ChunkedFileBlobIndexWrapper._prefetch   s    yy'' k
 19M 	
tax			ii
D)	)  A.#}}

:zz3883C3CD % / 			 /.s   AC<<
D
c                     U R                   (       a  U R                   R                  5         S U l         S U l        SU l        g NT)r   r   r   r   r   s    r   r   !ChunkedFileBlobIndexWrapper.close  s.    ==MM!r   c                    U R                   (       a  [        S5      eU R                  (       a  U R                  R	                  U5      $ US:  a  [        S5      e[        U R                  S S S2   5       HW  u  p#UR                  U::  d  M  X0R                  :w  a1  [        U R                  US-   * S  5      U l        U R                  5           O   [        S5      eU R                  R	                  XR                  R                  -
  5        g )NI/O operation on closed filer   zInvalid argumentr
   zCannot seek to pos)r   
ValueErrorr   r   rX   IOError	enumerater   r^   r   iterr   r   )rT   posnr   s       r   rX    ChunkedFileBlobIndexWrapper.seek  s    ;;;<<??==%%c**7,--dd 34FAzzS ,,&$(Qxz)B$CDMMMO 5 1223!4!445r   c                 $   U R                   (       a  [        S5      eU R                  (       a  U R                  R	                  5       $ U R                  c  U R
                  $ U R                  R                  U R                  R	                  5       -   $ )Nr   )r   r   r   r   tellr   r   r^   r   s    r   r    ChunkedFileBlobIndexWrapper.tell(  se    ;;;<<??==%%''== 99||""T]]%7%7%999r   c                    U R                   (       a  [        S5      eU R                  (       a  U R                  R	                  U5      $ [        5       nUS:  aa  U R                  bS  U R                  R	                  S5      nU(       d  U R                  5         OUR                  U5        U R                  b  MS  OUS:  a~  U R                  bq  U R                  R	                  [        US5      5      nU(       d  U R                  5         OU[        U5      -  nUR                  U5        US:  a  U R                  b  Mq  [        U5      $ )Nr   r   i   )r   r   r   r   r   	bytearrayr   extendminr   bytes)rT   r   resultblob_results       r   r    ChunkedFileBlobIndexWrapper.read1  s    ;;;<<??==%%a(( q5--+"mm007"MMOMM+. --+ a%DMM5"mm00Q?"MMO[))AMM+. a%DMM5 V}r   )r   r   r   r   r   r_   r   r   r   )r   )rB   rC   rD   rE   r   r   r   r   r   propertyr   r8   r   r   rX   r   r   rK   rA   r   r   rc   rc      sQ    KO! 7 7@6&:r   rc   )r   rq   concurrent.futuresr   hashlibr   asgiref.syncr   django.core.files.baser   rY   	django.dbr   r   glitchtip.base_modelsr	   
exceptionsr   r   r   Modelrd   objectrc   rA   r   r   <module>r      s_      1  & 2 ) . 0	&2| 2jX< Xv8FLL 8T& Tr   