
    Miu                    	   S SK Jr  S SKrS SKrS SKrS SKrS SKrS SKrS SKJ	r	  S SK
JrJrJrJrJrJr  S SKrS SKrS SKJrJrJrJr  S SKJr  S SKJr  S SKJr  S S	K J!r!  S S
K"J#r#J$r$  S SK%J&r&  S SK'J(r(J)r)  S SK*J+r+J,r,  S SK-J.r.J/r/J0r0J1r1JrJ2r2J3r3J4r4J5r5J6r6J7r7J8r8  S SK9J:r:J;r;  S SK<J=r=  S SK>J?r?J@r@  S SKAJBrB  S SKCJDrDJErEJFrFJGrGJHrHJIrI  S SKJJKrKJLrL  S SKMJNrNJOrO  S SKPJQrQJRrRJSrSJTrT  S SKUJVrVJWrWJXrXJYrY  S SKZJ[r[J\r\  S SK]J^r^  S SK_J`r`  S SKaJbrbJcrcJdrdJereJfrfJgrg  S SKhJiriJjrj  S SKkJlrl  S SKmJnrnJoro  \(       a  S SKpJqrq  S S KrJsrs  \l" 5         \R                  " S!\R                  S"9rv\R                  " S#\R                  5      rx\R                  " S$\R                  5      ryS%rzS&r{S'\l|        \o" S(5      SIS) j5       r}\N" \=R                  5      \&SSS*\gR                  S*SSS*SSSS*SSS*S+S*S,SSSSS-4                                                 SJS. jj5       5       rS/SS*S*\gR                  S*SS*SSS*SSS*S+S*S,SSSSS-\b\b4                                                   SKS0 jjr       SL                 SMS1 jjr    SN                 SOS2 jjr\o" S35      S\`GR                  S+SS4                 SPS4 jj5       r  SQ     SRS5 jjr   SS       STS6 jjrS/SS7S8\z\{4             SUS9 jjr    SVS: jr\o" S(5      S/SS*S*S*SS\fGR                  GR                  SSSSS*S*SSS*S*SS+S*S,SSS\b\b4                                                         SWS; jj5       rS*\e4     SXS< jjr         SY                   SZS= jjrSSS+S*S\e4                 S[S> jjrS\S? jrS]S@ jr\GR$                  " SASB5      SC 5       r      S^SD jr      S_SE jr        S`SF jr  Sa         SbSG jjrS*S*SS,SS\eSS+S4
                         ScSH jjrg)d    )annotationsN)Path)IOTYPE_CHECKINGAnyOptionalUnioncast)LTContainerLTImageLTItem	LTTextBox)open_filename)register_heif_opener)Image)	PdfReader)ArrayObjectIndirectObject)add_chunking_strategy)%clean_extra_whitespace_with_index_run-index_adjustment_after_clean_extra_whitespace)
PixelSpace
PointSpace)CoordinatesMetadataElementElementMetadataElementTyper   LinkListItem	PageBreakTable
TableChunkTextTitle)PageCountExceededErrorUnprocessableEntityError)FileType)loggertrace_logger)PARAGRAPH_PATTERN)add_element_metadataexactly_oneget_page_image_metadatanormalize_layout_elementocr_data_to_elementsspooled_to_bytes_io_if_needed)check_language_argsprepare_languages_for_tesseract)apply_metadataget_last_modified_date) check_annotations_within_elementget_urisget_words_from_objmap_bbox_and_index)PDFMinerConfigget_text_with_deduplicationopen_pdfminer_pages_generatorrect_to_bbox)determine_pdf_or_image_strategyvalidate_strategy)element_from_text)
env_config)OCR_AGENT_TESSERACTSORT_MODE_BASICSORT_MODE_DONTSORT_MODE_XY_CUTOCRModePartitionStrategy)coord_has_valid_pointssort_page_elements)patch_psparser)firstrequires_dependencies)DocumentLayout)LayoutElementz\s+)patternflagsst   (?:^|(?<=\s))(?:m|l|c|v|y|h|re|S|s|f|F|f\*|B|B\*|b|b\*|n|W|W\*|cm|q|Q|Do|g|G|rg|RG|k|K|cs|CS|w|J|j|M|d|i|gs)(?=\s|$)s9   (?:^|(?<=\s))(?:Tj|TJ|'|\"|Tf|Td|TD|Tm|T\*|BT|ET)(?=\s|$)i   i g    eͽAunstructured_inferencec                 N    SSK Jn   [        R                  R	                  SU 5      $ )Nr   DEFAULT_MODELUNSTRUCTURED_HI_RES_MODEL_NAME)"unstructured_inference.models.baserS   osenvirongetrR   s    t/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/unstructured/partition/pdf.pydefault_hi_res_modelrZ   q   s     A::>>:MJJ    F   TgGz?c                    [        XS9  [        U=(       d    / U5      n[        S0 SU _SU_SU_SU_SU_SU_SU_S	U_S
U
_SU_SU_SU_SU_SU_SU_SU_SU_SU_SU_SU_SU_UD6$ )ae  Parses a pdf document into a list of interpreted elements.
Parameters
----------
filename
    A string defining the target filename path.
file
    A file-like object as bytes --> open(filename, "rb").
strategy
    The strategy to use for partitioning the PDF. Valid strategies are "hi_res",
    "ocr_only", and "fast". When using the "hi_res" strategy, the function uses
    a layout detection model to identify document elements. When using the
    "ocr_only" strategy, partition_pdf simply extracts the text from the
    document using OCR and processes it. If the "fast" strategy is used, the text
    is extracted directly from the PDF. The default strategy `auto` will determine
    when a page can be extracted using `fast` mode, otherwise it will fall back to `hi_res`.
infer_table_structure
    Only applicable if `strategy=hi_res`.
    If True, any Table elements that are extracted will also have a metadata field
    named "text_as_html" where the table's text content is rendered into an html string.
    I.e., rows and cells are preserved.
    Whether True or False, the "text" field is always present in any Table element
    and is the text content of the table (no structure).
languages
    The languages present in the document, for use in partitioning and/or OCR. To use a language
    with Tesseract, you'll first need to install the appropriate Tesseract language pack.
metadata_last_modified
    The last modified date for the document.
hi_res_model_name
    The layout detection model used when partitioning strategy is set to `hi_res`.
extract_images_in_pdf
    Only applicable if `strategy=hi_res`.
    If True, any detected images will be saved in the path specified by
    'extract_image_block_output_dir' or stored as base64 encoded data within metadata fields.
    Deprecation Note: This parameter is marked for deprecation. Future versions will use
    'extract_image_block_types' for broader extraction capabilities.
extract_image_block_types
    Only applicable if `strategy=hi_res`.
    Images of the element type(s) specified in this list (e.g., ["Image", "Table"]) will be
    saved in the path specified by 'extract_image_block_output_dir' or stored as base64
    encoded data within metadata fields.
extract_image_block_to_payload
    Only applicable if `strategy=hi_res`.
    If True, images of the element type(s) defined in 'extract_image_block_types' will be
    encoded as base64 data and stored in two metadata fields: 'image_base64' and
    'image_mime_type'.
    This parameter facilitates the inclusion of element data directly within the payload,
    especially for web-based applications or APIs.
extract_image_block_output_dir
    Only applicable if `strategy=hi_res` and `extract_image_block_to_payload=False`.
    The filesystem path for saving images of the element type(s)
    specified in 'extract_image_block_types'.
extract_forms
    Whether the form extraction logic should be run
    (results in adding FormKeysValues elements to output).
form_extraction_skip_tables
    Whether the form extraction logic should ignore regions designated as Tables.
pdfminer_line_margin
    If two lines are close together they are considered to be part of the same paragraph.
    The margin is specified relative to the height of a line.
pdfminer_char_margin
    If two characters are closer together than this margin they are considered part of
    the same line. The margin is specified relative to the width of the character.
pdfminer_line_overlap
    If two characters have more overlap than this they are considered to be on the same line.
    The overlap is specified relative to the minimum height of both characters.
pdfminer_word_margin
    If two characters on the same line are further apart than this margin then they are
    considered to be two separate words, and an intermediate space will be added for
    readability. The margin is specified relative to the width of the character.
filenamefiler_   r`   include_page_breaksstrategyinfer_table_structure	languagesdetect_language_per_elementmetadata_last_modifiedhi_res_model_nameextract_images_in_pdfextract_image_block_typesextract_image_block_output_dirextract_image_block_to_payloadstarting_page_numberextract_formsform_extraction_skip_tablespasswordpdfminer_line_marginpdfminer_char_marginpdfminer_line_overlappdfminer_word_margin )r,   r1   partition_pdf_or_image)r_   r`   ra   rb   rc   ocr_languagesrd   re   rf   chunking_strategyrg   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   kwargss                           rY   partition_pdfry   |   s    F -#IO]CI!  0 	
 4  %@  6 , 4 #< (F (F 2 $  %@!" #$ 2%& 2'( 4)* 2
- r[    c                   [        5         [        XB5        U (       a  [        U 5      OSn[        UUUUS9n/ nSnU(       d^   [	        XS9(       a  [
        R                  " S5        O8[        S&U [        U5      UU=(       d    UUUUS.UD6n[        S U 5       5      n [        UUUUU
US	9nUb  UR                  S
5        Uc  [
        R                  " S5        S/n[        U5      nU[         R"                  :X  a  [$        R&                  " 5          [$        R(                  " S5        [+        S&0 SU _S[        U5      _SU_SU_SU_SU_SU_SU=(       d    U_SU	_SU_SU
_SU_SU_SU_SU_SU_SU_SU_S U_S!U_S"U_UD6sSSS5        $ U[         R,                  :X  a  [/        S&UUS#.UD6$ U[         R0                  :X  aH  [$        R&                  " 5          [3        S&U UUUUUU=(       d    UUUS$.	UD6n[5        U5      sSSS5        $ [7        S%U 35      e! [         a7  n[
        R                  " U5        [
        R                  " S5         SnAGNSnAff = f! , (       d  f       N`= f! , (       d  f       Nq= f)'zCParses a pdf or image document into a list of interpreted elements.N)line_marginchar_marginline_overlapword_marginFr^   zzPDF is too complex for text extraction based on heuristic checks. Falling back to hi_res strategy without text extraction.r_   r`   rd   rf   rl   ro   pdfminer_configc              3     #    U  H@  nU  H6  n[        U[        5      =(       a    UR                  R                  5       v   M8     MB     g 7fN)
isinstancer#   textstrip).0page_elementsels      rY   	<genexpr>)partition_pdf_or_image.<locals>.<genexpr>@  s=      +);+ r4(<RWW]]_<+ =);s   AA
z3PDF text extraction failed, skip text extraction...)is_imagepdf_text_extractablerc   rh   ri   r   z.No languages specified, defaulting to English.engignorer_   r`   r   rc   ra   rd   rv   rf   rg   r   rh   ri   rj   rk   rl   rm   rn   ro   r   	ocr_agenttable_ocr_agent)extracted_elementsra   )	r_   r`   ra   rd   rv   r   rf   rl   ro   z#Unsupported partitioning strategy: rt   )r   r>   r4   r9   is_pdf_too_complexr(   infoextractable_elementsr0   any	Exceptiondebugr=   seekwarningr2   rF   HI_RESwarningscatch_warningssimplefilter_partition_pdf_or_image_localFAST_partition_pdf_with_pdfparserOCR_ONLY _partition_pdf_or_image_with_ocr$_process_uncategorized_text_elements
ValueError) r_   r`   r   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   r   r   rx   last_modifiedr   r   r   erv   elementss                                    rY   ru   ru      s   D h)8@*84dM$((*(	O /1 	O!8?O &: 	&%6t<'+A+R])=%$3	& 	&" (+ +);+ ($ /133";H 		!GHG	3I>M$+++$$&!!(+0 !248 " '<	
 %8 $ , (>'N #4 &: '< +D 0N 0N &:  ,!" -H#$ "%& !0'( $)* !0- '&< 
&++	+, 
1 3
 
 	
 
&//	/$$&7 !$7#+!'='N%9! H 8A '& :8*E
FF[  	OLLOKKMNN	O. '&N '&s7   $H '7H A2I)I.
I#,II
I+.
I<c                ~    [        U[        5      (       a  [        R                  " U5      n[	        SU UUUUUUS.UD6$ )Nr   rt   )r   bytesioBytesIO_partition_pdf_with_pdfminer)r_   r`   rd   rf   rl   ro   r   rx   s           rY   r   r     sP     $zz$' 	51'	 	 	r[   c                    [        XS9  U (       aA  [        U S5       n[        [        [           U5      n[        SUU UUUUUS.UD6n	SSS5        U	$ U(       a  [        SUU UUUUUS.UD6n	W	$ ! , (       d  f       W	$ = f)an  Partitions a PDF using PDFMiner instead of using a layoutmodel. Used for faster
processing or detectron2 is not available.

Implementation is based on the `extract_text` implementation in pdfminer.six, but
modified to support tracking page numbers and working with file-like objects.

ref: https://github.com/pdfminer/pdfminer.six/blob/master/pdfminer/high_level.py
r^   rb)fpr_   rd   rf   rl   ro   r   Nrt   )r,   r   r
   r   r   _process_pdfminer_pages)
r_   r`   rf   rd   rl   ro   r   rx   r   r   s
             rY   r   r     s    & -8T*bbi$B. 	!#'=%9! /	 	H +2 O 
* 	
#9!5+	
 	
 O3 +*2 Os   +A..
A=pdfminerc                   / n	[        [        XUS9US9 GH  u  n
u  pUR                  UR                  p/ n/ n[	        UUS9nUR
                  (       a  [        UR
                  UUU
5      nU GHq  n[        UR                  U5      u  nnnnUUUU4n/ n[        U5      S:  aV  [        U[        5      (       aA  [        UUU
U5      n[        UU5      u  nnU H  nUR                  [        UU5      5        M      [!        US5      (       a  [#        U[$        R&                  5      /nO&[)        U5      n[*        R,                  " [.        U5      nU H  n[1        U5      u  nnUR3                  5       (       d  M(  UU4UU4UU4UU44n [5        UU US9n![7        U US9n"[9        UU5      n#[;        UU
U"UU#US9U!l        S	U!R<                  l        UR                  U!5        M     GMt     [A        UU5      nU	R                  U5        GM     U	$ )
z>Uses PDFMiner to split a document into pages and process them.)ro   r   startwidthheightr   get_textcoordinatescoordinate_systempointssystem)r_   page_numberr   r   linksrd   r   )!	enumerater;   r   r   r   annotsr6   r<   bboxlenr   r   r5   r7   appendr8   hasattrr:   r@   PDF_CHAR_DUPLICATE_THRESHOLD_extract_textresplitr*   r   r   r?   r   _get_links_from_urls_metadatar   metadatadetection_origin_combine_list_elements)$r   r_   rf   rd   annotation_thresholdrl   ro   r   rx   r   r   pagepage_layoutr   r   r   annotation_listr   objx1y1x2y2r   urls_metadataannotations_within_element_wordsannot_text_snippets_textmoved_indicesr   elementcoordinates_metadatar   s$                                       rY   r   r     s0    H,5%b_]"-((d $));+=+=v')&
 ;;&t{{F<M{[OC)#((F;NBBB#D24M?#a'JsI,F,F-M#(	.* .c6:57E!(();E5)IJ 8 sJ'' 0Z5\5\]- &c*!#*;U!C''LU'S$};;==!2hR2r(RHEF/$**;G
 ,?%0,( :-WE'6!)$/$8&<#"+(G$ 9CG$$5!((11 (5 h /}>OP&K-N Or[   c                    U(       a,  [        U5      R                  5       nUR                  S5        U$ U (       a  [        U 5      R                  5       nU$ [        S5      e)Nr   z-Either 'file' or 'filename' must be provided.)r   get_num_pagesr   r   )r_   r`   number_of_pagess      rY   _get_pdf_page_numberr   8  sZ     #D/779		!
 	 
#H-;;=  HIIr[   c                B    U(       a  [        XS9nX2:  a	  [        X2S9egg)z6Checks whether PDF exceeds pdf_hi_res_max_pages limit.r^   )document_pagespdf_hi_res_max_pagesN)r   r%   )r_   r`   r   r   s       rY   #check_pdf_hi_res_max_pages_exceededr   F  s1     -xK0(-  1 r[   i'  g      4@c                	   Sn Ub6  [        U[        5      (       d!  [        US5      (       a  UR                  5       nUb^  [        U[        5      (       a  [	        U5      nOUR                  SS5        UR                  5       nUR                  U=(       d    S5        OjU (       a   [        R                  R                  U 5      nOC Ub>  [        U[        5      (       d(  [        US5      (       a  Ub  UR                  U5        gggggXt:  aC   Ub>  [        U[        5      (       d(  [        US5      (       a  Ub  UR                  U5        gggggUbR  [        U[        5      (       a   [        [        R                  " U5      5      nO(UR                  S5        [        U5      nO[        U 5      nUR                  (       dC   Ub>  [        U[        5      (       d(  [        US5      (       a  Ub  UR                  U5        ggggg[        UR                  5       GH  u  pU
R                  S5      nUc  M  Sn [        U[        5      (       aV  U HO  n[        U[         5      (       a  UR#                  5       OUn[        US	5      (       d  M=  XR%                  5       -  nMQ     OH[        U[         5      (       a  UR#                  5       OUn[        US	5      (       a  UR%                  5       n[	        U5      U:  a  M  [	        [(        R+                  U5      5      nX::  a  GM	  [	        [,        R+                  U5      5      nU[/        US
5      -  nUU:  d  GM?  [0        R2                  " SU	S
-    SU SU SUS SU SU S35          Ub>  [        U[        5      (       d(  [        US5      (       a  Ub  UR                  U5        ggggg    Ub:  [        U[        5      (       d%  [        US5      (       a  Ub  UR                  U5        g! [&         a     GM  f = f! [&         ad  n[0        R4                  " SU 35         SnAUb>  [        U[        5      (       d(  [        US5      (       a  Ub  UR                  U5        gggggSnAff = f! Ub>  [        U[        5      (       d(  [        US5      (       a  Ub  UR                  U5        f f f f f = f)aG  Check if a PDF is likely a complex vector drawing (e.g., CAD/engineering docs)
that would be extremely slow or produce garbage results with PDFMiner text extraction.

Try to minimize overhead with early exits:
1. Avoid overhead by skipping files smaller than min_file_size_bytes.
2. For each page, decode the raw content stream bytes. Skip pages where the
   decoded stream is smaller than min_raw_stream_bytes.
3. For large streams, regex to count graphics without parsing the stream.

A page is flagged as too complex when it has a high number of graphics operators
AND a high ratio of graphics-to-text operators.

Parameters
----------
filename
    Path to a PDF file.
file
    A file-like object or bytes.
max_graphics_ops
    If any page exceeds this many graphics operators AND the graphics-to-text ratio
    exceeds `min_graphics_to_text_ratio`, the PDF is considered too complex.
min_graphics_to_text_ratio
    Minimum ratio of graphics ops to text ops required (in conjunction with
    `max_graphics_ops`) to flag a page as too complex.
min_file_size_bytes
    Skip the complexity check entirely for files smaller than this (default 1 MB).
min_raw_stream_bytes
    Skip operator counting for pages whose decoded content stream is smaller than
    this (default 100 KB). Small streams can't have enough operators to trigger
    the threshold.
Ntellr      r   Fz	/Contentsr[   get_datar\   zPage z has z graphics ops, z text ops (ratio: z.1fz). Exceeds thresholds (ops: z	, ratio: z3). Flagging PDF as too complex for text extraction.Tz!is_pdf_too_complex check failed: )r   r   r   r   r   r   rV   pathgetsizer   r   r   pagesr   rX   r   r   
get_objectr   r   GRAPHICS_OPS_PATTERNfindallTEXT_OPS_PATTERNmaxr(   r   r   )r_   r`   max_graphics_opsmin_graphics_to_text_ratiomin_file_size_bytesmin_raw_stream_bytesoriginal_pos	file_sizereader
page_indexr   contentsraw_dataitemr   num_graphics_opsnum_text_opsratior   s                      rY   r   r   T  s   P #'L\$JtU$;$;f@U@U99;L $&&I			!Q IIK			,+!,1IN tU++f%%(IIl# ) & , K *H tU++f%%(IIl# ) & , C $&&"2::d#34		!"4x(F||n tU++f%%(IIl# ) & , k !*&,, 7Jxx,H Hh44 (3=dN3S3Sdoo/Y]"3
33$6H !) 2<Hn1U1U++-[c  sJ//#&<<>
 8}33  ##7#?#?#IJ  3/77ABL$s<';;E11JN+51A0B/#n$6uSk B00@/A B89 :GG  tU++f%%(IIl# ) & , k !8j tU++f%%(IIl#S  6  8<=
 tU++f%%(IIl# ) & ,  tU++f%%(IIl# ) & , s   CP P A1P 3P 8AP	P	"P #AP	+A(P .P P 	
PP PP 
R	%R>R R		R ASc                |    [        S U R                   5       5      (       a  U=(       d
    [        5       nSUl        U$ )zBEnable detect_vertical in pdfminer when the PDF has rotated pages.c              3  j   #    U  H)  oR                   =(       d    0 R                  S S5      v   M+     g7f)pdf_rotationr   N)image_metadatarX   )r   ps     rY   r   5_enable_detect_vertical_if_rotated.<locals>.<genexpr>  s+     
cDbq"''::Dbs   13T)r   r   r9   detect_vertical)inferred_document_layoutr   s     rY   "_enable_detect_vertical_if_rotatedr    s7    
 
cD\DbDb
ccc)=^-=*.'r[   c                  ^^	^
^^>^? SSK JnJn  SSKJm>  SSKJnJnJn J	n!  SSK
Jn"  SSKJn#  SSKJn$Jn%  SSKJn&Jn'  SS	KJn(Jn)Jn*Jn+  T	=(       d    U=(       d
    [3        5       m	T
c  [4        R6                  m
T(       d  S[4        R8                  0O0 m?T(       d
  [;        XUS9  S
n,S
n-S
n.S
n/[4        R<                  n0U>U	UU?UU
4S jn1UGc   U1" UU 5      n2[?        U2U5      nU(       a	  U+" U T
TUS9O/ / 4u  n3n4U(       a  U(       d\  [4        R@                  (       a&  [C        [E        [4        RF                  5      S-  5      nO![C        [D        RH                  " 5       S-  5      n[J        RL                  " USS9  U0(       d3  U " U2T	S9n,U" U3 V5s/ s H  n5U5RO                  5       PM     sn5S9n-U!" 5       n.U)" U2U3T	S9n6U%" U U6U3TUUUUT
U.TUS9n7GO-U1" UU5      n2[Q        US5      (       a  URS                  S5        [?        U2U5      nU(       a	  U*" UT
TUS9O/ / 4u  n3n4U(       a  U(       d\  [4        R@                  (       a&  [C        [E        [4        RF                  5      S-  5      nO![C        [D        RH                  " 5       S-  5      nU0(       d3  U " U2T	S9n,U" U3 V5s/ s H  n5U5RO                  5       PM     sn5S9n-U!" 5       n.U)" U2U3T	S9n6[Q        US5      (       a  URS                  S5        U$" UU6U3TUUUUT
U.TUS9n7U(" U75      n7[U        U74SUUSUUU4S.UD6n8U&" U5      nU(       a  U'" U8U[V        RX                  U UTT
UUS9	  U H.  n9U(       a  U9[V        RX                  :X  a  M   U'" U8UU9U UTT
UUS9	  M0     / n:U8 GHS  n;[[        U;[\        5      (       a	  U(       d  M"  [[        U;[^        5      (       a!  U:Ra                  [c        [d        U;5      5        MX  [[        U;[f        5      (       d  Mo  [[        U;[h        [j        45      (       a?  [l        Rn                  " SSU;Rp                  =(       d    S5      Rs                  5       U;l8        OB[l        Rn                  " [t        SU;Rp                  =(       d    S5      Rs                  5       U;l8        U;Rp                  (       d  [[        U;[\        5      (       d  GM4  U:Ra                  [c        [d        U;5      5        GMV     U(       a  U#" UU T	U:US9n<U:Rw                  U<5        U(       a  U0(       d  U" U:S9n// n=U,(       a  U=Ra                  U,5        U-(       a  U=Ra                  U-5        U.(       a  U=Ra                  U.5        U/(       a  U=Ra                  U/5        U"" U=U UTU[4        Rx                  [4        R<                  [4        Rz                  [4        R|                  [4        R~                  [4        R                  S.
6  U:$ s  sn5f s  sn5f ) z)Partition using package installed locallyr   )process_data_with_modelprocess_file_with_model)PdfRenderTooLargeError)ExtractedLayoutDumperFinalLayoutDumperObjectDetectionLayoutDumperOCRLayoutDumper)save_analysis_artifiacts)run_form_extraction)process_data_with_ocrprocess_file_with_ocr)check_element_types_to_extractsave_elements)clean_pdfminer_inner_elements$merge_inferred_with_extracted_layoutprocess_data_with_pdfminerprocess_file_with_pdfminerNpdf_render_max_pixels_per_page)r_   r`   r   c                h   >  U " U4TTTTS.TD6$ ! T a  n[        [        U5      5      UeS nAff = f)N)r   
model_namepdf_image_dpiro   )r&   str)		processorsourceexcr  rg   r   model_render_kwargsro   r#  s	      rY   _run_layout_inference<_partition_pdf_or_image_local.<locals>._run_layout_inference;  sV    
	>!,+! &  & 	>*3s84#=	>s    1,1)r_   dpiro   r   	annotatedT)exist_ok)layoutr"  )r.  )r  extracted_layoutrg   )
r/  r   rc   r   rv   ocr_moder#  ocr_layout_dumperro   r   r   )r`   r+  ro   r   F)sortablera   last_modification_dateinfer_list_itemsrd   rl   layouts_links)	r   rl   element_category_to_saver_   r`   r   r#  rk   output_dir_pathz[^\S\n]+ rz   )r`   r_   r"  r   skip_table_regions)
r_   r`   r   analyzed_image_output_dir_pathskip_bboxesskip_dump_od	draw_griddraw_captionresizeformat)A'unstructured_inference.inference.layoutr  r  *unstructured_inference.inference.pdf_imager  5unstructured.partition.pdf_image.analysis.layout_dumpr  r  r  r  /unstructured.partition.pdf_image.analysis.toolsr  0unstructured.partition.pdf_image.form_extractionr  $unstructured.partition.pdf_image.ocrr  r  0unstructured.partition.pdf_image.pdf_image_utilsr  r  4unstructured.partition.pdf_image.pdfminer_processingr  r  r  r  rZ   r@   PDF_RENDER_DPIPDF_RENDER_MAX_PIXELS_PER_PAGEr   ANALYSIS_DUMP_OD_SKIPr  GLOBAL_WORKING_DIR_ENABLEDr$  r   GLOBAL_WORKING_PROCESS_DIRcwdrV   makedirsas_listr   r   document_to_element_listr   IMAGEr   r    r   r   r
   r   r#   r!   r"   r   subr   r    RE_MULTISPACE_INCLUDING_NEWLINESextendANALYSIS_BBOX_SKIPANALYSIS_BBOX_DRAW_GRIDANALYSIS_BBOX_DRAW_CAPTIONANALYSIS_BBOX_RESIZEANALYSIS_BBOX_FORMAT)@r_   r`   r   rc   ra   rd   rv   r0  r"  rg   r#  rf   r   rh   ri   rj   rk   analysisr:  rl   rm   rn   r   ro   r   r   r   rx   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  od_model_layout_dumperextracted_layout_dumperr1  final_layout_dumperskip_analysis_dumpr)  r  r/  r5  r.  merged_document_layoutfinal_document_layoutr   el_typeout_elementsr   formslayout_dumpersr  r(  s@     `      ``            `                                      @@rY   r   r     s   B R  YTa  *QZQ;O;Q"11  
*:+T+TU  +?S	
 EI?C377;#99> > |#89PRZ#[ <$
 $ '!!! /	 b 	(- 18858ZBBCkQ62 69k9Q5R2KK6F%)D30*& +@;KL;KFNN,;KL+' %4$5!!E%=-/"
 !6"-"7''/+!
 $99PRV#W 4  IIaL<$
 $ '}xQ` b 	(- 18858ZBBCkQ62 69k9Q5R2%)D30*& +@;KL;KFNN,;KL+' %4$5! "F%=-/"
 4  IIaL 5"-"7''/+!
  ::OP'/5 1# H !??X Y !5%0%6%6'+I:
	
 - W0A0A%A!5%,'+I:
	
	 -  Lb)$$-@b%  Wb 12D!!"uj122 &&c277=bAGGI&&4GGMr %'	 
 ww*R33##D"$56' * #(!:
 	E"!"3## !!!"89"!!"9:!!"34!!"56 +I"55#99 88#>>2222	
 G Mr Ms   
WWc                    / nU  HM  n[        U[        5      nU[        :w  a  [        Xb5      nXF-  nU(       d  M5  UR                  [        SS95        MO     U$ )z!Partitions a PDF using pdfparser.rz   r   )rH   rB   r   r    )r   ra   	sort_moderx   r   r   sorted_page_elementss          rY   r   r   0  s\     H+  2-Q'#56J#V (OOI2./ , Or[   c	                   SSK Jn
  / nU(       ay  / nUb  [        R                  " U5      O[        R                  " U 5      nUR	                  U5        [        XS9 H(  u  p[        SUUUUUUS.U	D6nUR                  U5        M*     U$ [        U
" XUS9US9 H(  u  p[        SUUUUUUS.U	D6nUR                  U5        M*     U$ )zgPartitions an image or PDF using OCR. For PDFs, each page is converted
to an image prior to processing.r   )convert_pdf_to_imagesr   )imagerd   rv   r   ra   rf   )ro   rt   )rG  rk  PILImageopenr   r   +_partition_pdf_or_image_with_ocr_from_imagerU  )r_   r`   ra   rd   rv   r   rf   rl   ro   rx   rk  r   imagesrl  r   r   s                   rY   r   r   I  s     WH'+'7d#X]]8=Te"+F"OKG #+'$7'= M OOM* #P4 O #,!(8DL`#
K H #+'$7'= M OOM*#
 Or[   c                b   SSK Jn  UR                  US9n	U	R                  5       (       a  [        nU	R                  U S9n
[        UU R                  UUS9n[        U
R                  5       U R                  US9nUnU[        :w  a  [        X5      nU(       a  UR                  [        SS95        U$ )	zQExtract `unstructured` elements from an image using OCR and perform partitioning.r   )OCRAgent)language)rl  )r   filetyper   rd   )
image_sizecommon_metadatarz   rg  )5unstructured.partition.utils.ocr_models.ocr_interfacerr  	get_agentis_text_sortedrC   get_layout_elements_from_imager   r@  r/   rP  sizerH   r   r    )rl  rd   rv   r   ra   rf   rh  rx   rr  r   ocr_datar   r   ri  s                 rY   ro  ro  |  s     O""M":I !!"	77e7DH,	H ):: M )N"1-K##I2$67r[   c                
   / nU  Hz  n[        US5      (       aS  UR                  [        R                  :X  a5  [	        [        [        U5      R                  5      nUR                  Ul        OUnUR                  U5        M|     U$ )zProcesses a list of elements, creating a new list where elements with the
category `UncategorizedText` are replaced with corresponding
elements created from their text content.category)
r   r~  r   UNCATEGORIZED_TEXTr?   r
   r#   r   r   r   )r   rc  r   new_els       rY   r   r     sm    
 L2z""r{{k6T6T'T&tD"~':':;F kkFOFF#  r[   c                    [        U S5      (       a  U R                  5       $ [        U [        5      (       a$  SnU  H  nU[	        U5      =(       d    S-  nM     U$ [        U [
        [        45      (       a  gg)znRecursively extracts text from PDFMiner objects to account
for scenarios where the text is in a sub-container.r   rz   
)r   r   r   r   r   r   r   )r  r   childs      rY   r   r     sl     tZ  }}	D+	&	&EM%(.B.D 	D9g.	/	/ r[   zpdfminer.pdfinterpz!PDFPageInterpreter.init_resourcesc                .    US   nSU;   a  US	 U " U5      $ )Nr   
ColorSpacert   )wrappedinstanceargsrx   	resourcess        rY   #pdfminer_interpreter_init_resourcesr    s&    QIy l#9r[   c                l   Sn/ nU  H  n[        U[        5      (       a%  UnUR                  nUR                  R                  nO[U(       aT  [        UR                  R                  WS9(       a1  W SUR                   3Ul        [        UUUS9nUR                  5         UR                  U5        M     U$ )zECombine elements that should be considered a single ListItem element.N)r   boundaryr8  )element1element2r   )	r   r   r   r   r   check_coords_within_boundary"_combine_coordinates_into_element1popr   )r   r   tmp_elementupdated_elementsr   tmp_text
tmp_coordss          rY   r   r     s     K&(gx((!K||H ))55J9((44
 #+1W\\N;K8$ "3G   "(% & r[   c           
         / nU  HN  n[         R                  " [        5         UR                  US   US   [	        US   U5      S.5        SSS5        MP     U$ ! , (       d  f       Mc  = f)z+Extracts links from a list of URL metadata.r   uristart_index)r   urlr  N)
contextlibsuppress
IndexErrorr   r   )r   r   r   r  s       rY   r   r     sk     E  ,LLKu:#PM*%$	 -,  L -,s   )A
A(	c                   [        U R                  R                  R                  S   S   UR                  R                  R                  S   S   5      n[	        U R                  R                  R                  S   S   UR                  R                  R                  S   S   5      n[        U R                  R                  R                  S   S   UR                  R                  R                  S   S   5      n[	        U R                  R                  R                  S   S   UR                  R                  R                  S   S   5      nX54X64XF4XE44n[        UUS9U R                  l        [        R                  " U 5      $ )zXCombine the coordiantes of two elements and apply the updated coordiantes to `elements1`r   r   r\   r   )minr   r   r   r   r   copydeepcopy)r  r  r   r   r   r   r   r   s           rY   r  r    sl    
%%,,Q/2%%,,Q/2
B 
%%,,Q/2%%,,Q/2
B 
%%,,Q/2%%,,Q/2
B 
%%,,Q/2%%,,Q/2
B h2(RH5F$7 %H! ==""r[   c                X   [        U 5      (       d.  [        U5      (       d  [        R                  " SU  SU S35        gUR                  S   S   nUR                  S   S   nUR                  S   S   nUR                  S   S   nXT-
  nXv-
  n	U R                  S   S   XBU-  -
  :  =(       a6    U R                  S   S   XRU-  -   :  =(       a    U R                  S   S   U:  n
U R                  S   S   XsU	-  -   :  =(       a    U R                  S   S   XcU	-  -
  :  nU
=(       a    U$ )ah  Checks if the coordinates are within boundary thresholds.
Parameters
----------
coordinates
    a CoordinatesMetadata input
boundary
    a CoordinatesMetadata to compare against
vertical_threshold
    a float ranges from [0,1] to scale the vertical (y-axis) boundary
horizontal_threshold
    a float ranges from [0,1] to scale the horizontal (x-axis) boundary
zcoordinates z and boundary z did not pass validationFr   r   r\   )rG   r)   detailr   )r   r  horizontal_thresholdvertical_thresholdboundary_x_minboundary_x_maxboundary_y_minboundary_y_max
line_widthline_heightx_within_boundaryy_within_boundarys               rY   r  r  (  sa   $ "+..7Mh7W7W;-~hZ?WX	
 __Q'*N__Q'*N__Q'*N__Q'*N0J 1K 
		A	q	!NZ6W$X	X 	9"1%Q[:[(\\	9"1%7  	1a >+5U#VV[a #n[8X&YY  2!22r[   c                
	  ^%^& SSK Jn  / n[        U R                  5      n[	        U R                  U	S9 GH9  u  nm&/ n[        T&5      nUR                  S5      nUR                  S5      nUR                  S5      n/ nU
(       a  U
S   (       a  XU	-
     OSnT&R                  R                  R                  5        VVs/ s H  u  nnUS;   d  M  UPM     nnnU(       a  [        U&4S	 jU 5       5      nOS
nT&R                  R                  5        GH  m%U(       aP  U(       aI  [        R                  " [        T%R                  S[        R                   5      5      (       d  [#        UUS9nOSn[%        T%UUU(       a  UOSS9n['        U[(        5      (       ai  U H,  nU(       a  UUR*                  l        UUR*                  l        M.     UR1                  U5        UR1                  U Vs/ s H  nT%U4PM	     sn5        M  U(       a  U" UT%R                  5      O/ UR*                  l        U(       a  UUR*                  l        [        T%SS5      UR*                  l        [        T%SS5      UR*                  l        [        T%SS5      UR*                  l        ['        U[:        5      (       a/  UR*                  R<                  c  U(       a  SUR*                  l        UR?                  U5        UR?                  T%U45        UR*                  R@                  (       a   UR*                  R@                  RB                  OSn[E        T%S5      (       a  T%RF                  OSnUR                  5        V V!s0 s H  u  n n!U S;  d  M  U U!_M     n"n n![I        U4UUUUUR*                  R<                  UUUS.U"D6  GM     U HX  u  m%n[E        T%S5      (       d  M  T%RJ                  c  M(  [M        U%4S jU 5       5      n#U#RN                  UR*                  l(        MZ     Un$U(       a  U[R        :w  a  [U        UU5      n$U(       a   XU	-   :  a  U$R?                  [W        SS95        UR1                  U$5        GM<     U$ s  snnf s  snf s  sn!n f )zDConverts a DocumentLayout object to a list of unstructured elements.r   )get_links_in_elementr   r@  r   r   N)HeadlineSubheadlinec              3  ~   >#    U  H2  n[         R                  " TR                  R                  U:H  5      v   M4     g 7fr   )npr   elements_arrayelement_class_ids)r   idxr   s     rY   r   +document_to_element_list.<locals>.<genexpr>~  s5      3C t**<<CDD3s   :=Fr   r   html)r   r4  source_formattext_as_htmltable_as_cellstable_extraction_method
image_pathr   )r   rt  r   r   category_depthr  r   rd   parentc              3  N   >#    U  H  u  pUTR                   L d  M  Uv   M     g 7fr   )r  )r   l_elr   layout_elements      rY   r   r    s$     ](;HDt~G\G\?\RR(;s   %	%rz   rg  ),rH  r  r   r   r   r-   rX   r  element_class_id_mapitemsr   iter_elementsr  isnangetattrr   nanr   r.   r   listr   r   r   rU  r   r  r  r  r$   r  r   r   r   r   r  r+   r  rJ   id	parent_idrC   rH   r    )'documentr2  ra   r3  r4  r  r   rh  rd   rl   r5  rx   r  r   	num_pagesr   r   page_image_metadataimage_formatimage_widthimage_heighttranslation_mappingr   r  
class_typehead_line_type_class_idshas_headliner   r   r   r   el_image_pathkvfiltered_kwargselement_parentri  r  r   s'                                        @@rY   rQ  rQ  T  s5    Z HHNN#I&x~~=QRT')5d;*..x8)--g6*..x8EG q!1 (<<= 	 $(#6#6#K#K#Q#Q#S$
#SZ88 #S 	! $

 $ 3 L
 !L"11??AN )<)<dBFF!KLL$.[$V!$(!."3!1/<m&	G '4((!B-4J1.9BKK+ " $$W-#**7+S7R^R,@7+ST IN(0C0CDSU   & *5KG$$207X\0]  -29.JZ\`2a  /;B"$=t<  8 w..73C3C3R3R3Z 67G$$3$$W-#**NG+DE7>7G7G7S7S  ,,33Y] 
 .5^\-R-R))X\  "(!/A1<`3`1   !'%'"3&//>>(!1# "s BL (;#NG~x00^5J5J5V!&](;]" .<->->  * (;  -	^3#5mY#O ;=Q1Q#Q ''	r(:;,-_ Sb OC$
F ,T@s   >Q4Q4*Q:/Q?
 Q?
)returnr$  )2r_   Optional[str]r`   Optional[IO[bytes]]ra   boolrb   r$  rc   r  rv   r  rd   Optional[list[str]]re   r  rf   r  rw   r  rg   r  rh   r  ri   r  rj   r  rk   r  rl   intrm   r  rn   r  ro   r  rp   Optional[float]rq   r  rr   r  rs   r  rx   r   r  list[Element])4r_   r$  r`   Optional[bytes | IO[bytes]]r   r  ra   r  rb   r$  rc   r  rd   r  re   r  rf   r  rg   r  rh   r  ri   r  rj   r  rk   r  rl   r  rm   r  rn   r  ro   r  rp   r  rq   r  rr   r  rs   r  r   r$  r   r$  rx   r   r  r  )rz   NNNr\   NN)r_   r$  r`   r  rd   r  rf   r  rl   r  ro   r  r   Optional[PDFMinerConfig]rx   r   r  list[list[Element]])Nr\   NN)r_   r$  r`   r  rf   r  rd   r  rl   r  ro   r  r   r  rx   r   r  r  )r   z	IO[bytes]r_   r$  rf   r  rd   r  r   r  rl   r  ro   r  r   r  r  r  )rz   N)r_   r$  r`   r  r  r  )rz   NN)r_   r$  r`   r  r   r  r  None)r_   r$  r`   z!Optional[Union[bytes, IO[bytes]]]r   r  r   floatr   r  r   r  r  r  )r   Optional['PDFMinerConfig']r  r  ):r_   r$  r`   r  r   r  rc   r  ra   r  rd   r  rv   r  r0  r$  r"  r  rg   r  r#  Optional[int]rf   r  r   r  rh   r  ri   r  rj   r  rk   r  r[  r  r:  r  rl   r  rm   r  rn   r  r   r  ro   r  r   r  r   r$  r   r$  rx   r   r  r  )r   r  ra   r  rh  r$  )	rz   NFNNFNr\   N)r_   r$  r`   r  ra   r  rd   r  rv   r  r   r  rf   r  rl   r  ro   r  rx   r   )rl  zPILImage.Imagerd   r  rv   r  r   r  ra   r  rf   r  rh  r$  rx   r   r  r  )r   r  )r  r   r  r$  )r   r  r   PixelSpace | PointSpacer  r  )r   zlist[dict[str, Any]]r   z
np.ndarrayr  z
list[Link])r  r   r  r   r   r  r  r   )g?g333333?)
r   r   r  r   r  r  r  r  r  r  )r  rL   r2  r  ra   r  r3  r  r4  r  r  r  r   r  rh  r$  rd   r  rl   r  r5  zOptional[list[list]]rx   r   r  r  )
__future__r   r  r  r   rV   r   r   pathlibr   typingr   r   r   r   r	   r
   numpyr  wraptpdfminer.layoutr   r   r   r   pdfminer.utilsr   pi_heifr   PILr   rm  pypdfr   pypdf.genericr   r   unstructured.chunkingr   unstructured.cleaners.corer   r   "unstructured.documents.coordinatesr   r   unstructured.documents.elementsr   r   r   r   r   r   r    r!   r"   r#   r$   unstructured.errorsr%   r&   unstructured.file_utils.modelr'   unstructured.loggerr(   r)   unstructured.nlp.patternsr*   $unstructured.partition.common.commonr+   r,   r-   r.   r/   r0   "unstructured.partition.common.langr1   r2   &unstructured.partition.common.metadatar3   r4   rH  r5   r6   r7   r8   /unstructured.partition.pdf_image.pdfminer_utilsr9   r:   r;   r<   !unstructured.partition.strategiesr=   r>   unstructured.partition.textr?   #unstructured.partition.utils.configr@   &unstructured.partition.utils.constantsrA   rB   rC   rD   rE   rF   $unstructured.partition.utils.sortingrG   rH   unstructured.patches.pdfminerrI   unstructured.utilsrJ   rK   rA  rL   .unstructured_inference.inference.layoutelementrM   compileDOTALLrT  	MULTILINEr   r   DEFAULT_MIN_FILE_SIZE_BYTESDEFAULT_MIN_RAW_STREAM_BYTESMAX_IMAGE_PIXELSrZ   PDFAUTOry   ru   r   r   PDF_ANNOTATION_THRESHOLDr   r   r   r   r  	FULL_PAGEvaluer   r   r   ro  r   r   patch_function_wrapperr  r   r   r  r  rQ  rt   r[   rY   <module>r     s
   "   	 	 	   @ @   C C ( ( !  5 7 F    Q 2 4 7  d Y   a 9 :  \ 8 ;FL   $&::fBII#N  zz LL  ::KLL  . &     /0K 1K " $ %%**"'#'%)(-,0'+'+"'5948+0 !(,",0,0-1,1/{{
{ { 	{
  { !{ #{ "&{ *{ %{ %{  {  3{ %2{ %){  !{" #{$ "&%{& '{( *){* *+{, +-{. */{0 1{2 3{  {~ (, %%**"'%)(-,0'+"'5948+0 !(,",0,0-1,1(.1VGVG
%VG VG 	VG
 VG  VG #VG "&VG *VG %VG  VG  3VG %2VG %)VG VG  !VG" "&#VG$ %VG& *'VG( *)VG* ++VG, *-VG. /VG0 1VG2 3VG4 5VGt (,%),0 !"04
% # *	
   .  8 &* !"04..
. *. #	.
 . . .. . .b z"
 &*,6,O,O !"04VVV *V #	V
 *V V V .V V #Vt (,
% 	 (, $
%  
	 .2"(,: <HH
+H H !&	H
 H H 
HV	/	  	 /0(,"' %%)#'%%++ $'+#',0!&"'5948+048 !(,*."04(.7AA
%A A  	A
 A #A !A A A %A !A *A A  A  3A  %2!A" %)#A$ %A& %2'A( )A* +A, "&-A. (/A0 1A2 .3A4 5A6 7A8 9A: ;A 1AL
 !&%+ 4 (, %%)#',0 !"00
%0 0 #	0
 !0 0 *0 0 0 0j &*#' %,0%,,", !, 	,
 , *, , , ,^". 24WX Y0G8'8B(##!(#=T##> #& #	)3$)3!)3  )3 	)3
 
)3\  %,0!#'&*%%) !*.EEE E *	E
 E !E $E E #E E (E E Er[   