֩i4 dZddlZddlmcmZddlZddlm Z ddl m Z m Z ddl mZmZmZmZmZmZ ddededed e fd Zded e fd Zd Zd ZdZdZdZdZdZdZdZdZ dZ!dZ"dZ#dZ$dZ%dZ&dZ'dZ(dZ)y) u> Story 2.07 — Extractor Integration Tests ========================================== 19 tests covering all extractor functions. All fixtures use pre-built HTML strings — no network calls. # VERIFICATION_STAMP # Story: 2.07 # Verified By: parallel-builder # Verified At: 2026-02-26 # Tests: 19/19 # Coverage: 100% N BeautifulSoup) FetchedPageExtractedContent)extract_from_htmlextract_headingsextract_code_blocksextract_tablesextract_with_readability extract_batchhtmlurl content_typereturnc$t||d|idS)Nz2026-02-26T00:00:00Z)rr status_coderheaders fetched_at)r)r rrs ?/mnt/e/genesis-system/tests/kb/test_m2_extractor_integration.py _make_pager"s!   !)  ct|dS)Nlxmlr)r s r_soupr.s v &&rcd}t|}t|}t|t}|s ddt j vst jtrt jtnddt j vst j|rt j|nddt j vst jtrt jtndt j|dz}tt j|d}|j}d}||k(}|st jd |fd ||fdt j vst j|rt j|ndt j|t j|d z}d d |iz}tt j|dx}x}}d} |j}| |v} | st jd| fd| |ft j| dt j vst j|rt j|ndt j|dz}d d |iz}tt j|dx} x} }|j}|j} || k(}|st jd |fd|| fdt j vst j|rt j|ndt j|dt j vst j|rt j|ndt j| dz} dd| iz} tt j| dx}x}} y)z6Basic HTML page produces clean text and correct title.zp Hello World

This is a test paragraph.

5assert %(py4)s {%(py4)s = %(py0)s(%(py1)s, %(py2)s) } isinstanceresultrpy0py1py2py4Nz Hello World==z-%(py2)s {%(py2)s = %(py0)s.title } == %(py5)sr!r#py5assert %(py7)spy7ztest paragraphinz,%(py1)s in %(py5)s {%(py5)s = %(py3)s.text }r"py3r))zD%(py2)s {%(py2)s = %(py0)s.url } == %(py6)s {%(py6)s = %(py4)s.url }page)r!r#r$py6assert %(py8)spy8)rrrr @py_builtinslocals @pytest_ar_should_repr_global_name _safereprAssertionError_format_explanationtitle_call_reprcomparetextr)r r1r @py_assert3 @py_format5 @py_assert1 @py_assert4 @py_format6 @py_format8 @py_assert0 @py_assert2 @py_assert5 @py_format7 @py_format9s rtest_extract_simple_htmlrJ6s D d D t $F f. // //////:///://////f///f//////.///./// /////// <<(=(<= ((((<=((((((6(((6(((<(((=((((((( *v{{* { **** {*** ******v***v***{******* ::!!: !!!!:!!!!!!6!!!6!!!:!!!!!!!!!!!!!!!!!!!rcd}t|}t|}d}|j}||v}|stjd|fd||ftj |dt jvstj|rtj |ndtj |dz}dd|iz}ttj|d x}x}}d }|j}||v}|stjd|fd||ftj |dt jvstj|rtj |ndtj |dz}dd|iz}ttj|d x}x}}d }|j}||v}|stjd |fd ||ftj |dt jvstj|rtj |ndtj |dz}dd|iz}ttj|d x}x}}y )z=HTML with

Real content here.

alertnot inz0%(py1)s not in %(py5)s {%(py5)s = %(py3)s.text }rr/r*r+Nz color: redz Real contentr,r. rrr>r7r=r9r5r6r8r:r;r r1rrErBrFrCrDs rtest_extract_strips_scriptsrREsf D d D t $F %&++%7+ %%%%7+%%%7%%%%%%&%%%&%%%+%%%%%%% *v{{*<{ ****<{***<******v***v***{******* (V[[(>[ ((((>[(((>((((((V(((V((([(((((((rctd}t|}t|t}|s ddt j vst jtrt jtnddt j vst j|rt j|nddt j vst jtrt jtndt j|dz}tt j|d}|j}d}||k(}|st jd|fd ||fdt j vst j|rt j|ndt j|t j|d z}d d |iz}tt j|dx}x}}|j}d}||k(}|st jd|fd ||fdt j vst j|rt j|ndt j|t j|d z}d d |iz}tt j|dx}x}}y)z8Empty HTML does not crash; returns empty text and title.rrrrr Nr%r'r(r*r+)z,%(py2)s {%(py2)s = %(py0)s.text } == %(py5)s)rrrrr5r6r7r8r9r:r;r<r=r>)r1rr?r@rArBrCrDs rtest_extract_empty_htmlrUYsz b>D t $F f. // //////:///://////f///f//////.///./// /////// <<2<2 <266<2 ;;";" ;"66;"rcd}t|}t|}|j}d}||k(}|stjd|fd||fdt j vstj|rtj|ndtj|tj|dz}dd|iz}ttj|d x}x}}y ) z7 tag is preferred over <h1> for the title field.zu<html> <head><title>Page Title

Different H1

Body text.

z Page Titler%r'rr(r*r+N rrr<r7r=r5r6r8r9r:r;r r1rrArBr?rCrDs rtest_title_from_title_tagrYcs D d D t $F <<'<'<< ''''<<''''''6'''6'''<'''<'''''''rcd}t|}t|}|j}d}||k(}|stjd|fd||fdt j vstj|rtj|ndtj|tj|dz}dd|iz}ttj|d x}x}}y ) z1When there is no , the first <h1> is used.z^<html> <head></head> <body><h1>Fallback Title</h1><p>Body text.</p></body> </html>zFallback Titler%���r'���r���r(���r*���r+���NrW���rX���s��� r���test_title_from_h1r[���o���s���� D� d D t $F <<+++<+ ++++<+++++++6+++6+++<+++++++++++r���c���������������������d}�t��������|�������}t��������|������}d}|j������������������}||v}|st��������j������������������d|fd||f������t��������j ������������������|������dt ��������j�������������������������v�st��������j������������������|������rt��������j ������������������|������ndt��������j ������������������|������dz��}dd|iz��}t��������t��������j������������������|������������d x}x}}d }|j������������������}||v}|st��������j������������������d|fd||f������t��������j ������������������|������dt ��������j�������������������������v�st��������j������������������|������rt��������j ������������������|������ndt��������j ������������������|������dz��}dd|iz��}t��������t��������j������������������|������������d x}x}}d }|j������������������}||v�}|st��������j������������������d |fd ||f������t��������j ������������������|������dt ��������j�������������������������v�st��������j������������������|������rt��������j ������������������|������ndt��������j ������������������|������dz��}dd|iz��}t��������t��������j������������������|������������d x}x}}y )z;<nav> and <footer> content is excluded from extracted text.z<html> <head><title>Clean Page

Actual article content.

z Menu itemrMrOrr/r*r+NzCopyright 2026zActual article contentr,r.rPrQs rtest_nav_footer_removalr]{sl D d D t $F )fkk);k ))));k)));))))))f)))f)))k))))))) .6;;. ; .... ;... ......6...6...;....... #2v{{2 #{ 2222 #{222 #222222v222v222{2222222rcPd}tt|}t|}d}||k(}|stjd|fd||fdt j vstjtrtjtnddt j vstj|rtj|ndtj|tj|dz}dd |iz}ttj|d x}x}}|d }d }||k(}|sltjd|fd ||ftj|tj|dz} dd| iz}ttj|d x}x}}|d}d}||k(}|sltjd|fd ||ftj|tj|dz} dd| iz}ttj|d x}x}}|d}d}||k(}|sltjd|fd ||ftj|tj|dz} dd| iz}ttj|d x}x}}y )u9H1→H2→H3 nesting produces correct breadcrumb entries.z_

Introduction

Setup

Linux

r%z0%(py3)s {%(py3)s = %(py0)s(%(py1)s) } == %(py6)slenheadingsr!r"r0r2r3r4NrzH1: Introduction)z%(py1)s == %(py4)sr"r$assert %(py6)sr2zH1: Introduction > H2: Setupz(H1: Introduction > H2: Setup > H3: Linux rrrar7r=r5r6r8r9r:r; r rbrFrGrBrHrIrEr?r@s rtest_heading_hierarchyrjs D  d ,H x=A=A =A33xx=A A;,,,;, ,,,,;,,,,;,,,,,,,,,,, A;888;8 8888;8888;88888888888 A;DDD;D DDDD;DDDD;DDDDDDDDDDDrc|d}tt|}g}||k(}|stjd|fd||fdt j vstj |rtj|ndtj|dz}dd|iz}ttj|dx}}y) z2A page with no heading tags returns an empty list.z2

No headings here.

r%z%(py0)s == %(py3)srbr!r0assert %(py5)sr)N) rrr7r=r5r6r8r9r:r;)r rbrFrA @py_format4rCs rtest_no_headingsrpsm ?Dd ,H8r>8r88rrcJd}tt|}t|}d}||k(}|stjd|fd||fdt j vstjtrtjtnddt j vstj|rtj|ndtj|tj|dz}dd |iz}ttj|d x}x}}d }|d }||v}|sltjd |fd||ftj|tj|dz} dd| iz}ttj|d x}x}}d}|d }||v}|sltjd|fd||ftj|tj|dz} dd| iz}ttj|d x}x}}d}|d }||v}|sltjd|fd||ftj|tj|dz} dd| iz}ttj|d x}x}}y )zAHeadings containing inner or yield clean plain text.zZ

Nested Link

rfr%r`rarbrcr3r4Nz Nested Linkrr,z%(py1)s in %(py4)srdrer2z is extracted with python fence.zc 
print("hello")
rfr%r`rablocksrcr3r4Nrz ```pythonLassert %(py7)s {%(py7)s = %(py3)s {%(py3)s = %(py1)s.startswith }(%(py5)s) }r"r0r)r+zprint("hello")r,rrrdrer2```zJassert %(py7)s {%(py7)s = %(py3)s {%(py3)s = %(py1)s.endswith }(%(py5)s) }) r rrar7r=r5r6r8r9r:r; startswithendswith) r rurFrGrBrHrIrE @py_assert6rDr?r@s rtest_code_block_extractionr|s D!t -F v;!;! ;!33vv;! !9,9  , ,  ,, ,,,9,,, ,,, ,,, ,,,,,,, (vay( y (((( y((( (((y((((((( !9$9  $e$ e $$ $$$9$$$ $$$e$$$ $$$$$$$rcd}tt|}t|}d}||k(}|stjd|fd||fdt j vstjtrtjtnddt j vstj|rtj|ndtj|tj|dz}dd |iz}ttj|d x}x}}|d }|j}d }||}|std tj|tj|tj|tj|dz} ttj| d x}x}x}}|d}|j}d}||}|std tj|tj|tj|tj|dz} ttj| d x}x}x}}|d}|j}d}||}|std tj|tj|tj|tj|dz} ttj| d x}x}x}}y )z3Three 
 blocks yield exactly three items.z
    
console.log(1)

    
echo hi

    
raw block

    r_r%r`rarurcr3r4Nrz```jsrvrwrfz```bashrgrx)r	rrar7r=r5r6r8r9r:r;ry)
r
rurFrGrBrHrIrEr{rDs
          rtest_multiple_code_blocksr~sD
!t
-Fv;!;!;!33vv;!!9(9(((((((9((((((((((((((((!9*9*	*	*****9******	**********!9&9&&&&&&&9&&&&&&&&&&&&&&&&rc|d}tt|}g}||k(}|stjd|fd||fdt	j
vstj|rtj|ndtj|dz}dd|iz}ttj|dx}}y)	zCInline  tags (outside 
) are NOT included in code_blocks.zO
    
Use print() to display.

    r%rlrurmrnr)N)
r	rr7r=r5r6r8r9r:r;)r
rurFrArorCs      rtest_inline_code_excludedrsqD!t
-F6R<6R66Rrcd}tt|}t|}d}||k(}|stjd|fd||fdtjvstjtrtjtnddtjvstj|rtj|ndtj|tj|dz}dd	|iz}ttj|d
x}x}}|dj}d}|d}	||	v}|sltjd
|fd||	ftj|tj|	dz}
dd|
iz}ttj|d
x}x}}	g}d}|d}||v}|}|rd}|d}
||
v}|}|r
d}|d}||v}|}|s_tjd
|fd||ftj|tj|dz}dd	|iz}|j||rtjd
fd
ftj|tj|
dz}dd|iz}|j||r_tjd
fdftj|tj|dz}dd|iz}|j|tj|diz}d d!|iz}ttj|d
x}x}x}x}x}x}x}x}
x}x}}d"}|d}	||	v}|sltjd
|fd||	ftj|tj|	dz}
dd|
iz}ttj|d
x}x}}	y
)#z0A 3x3 table is converted to pipe-delimited text.z
    

        
ABC

        
123

        
456

    

    rfr%r`ratablesrcr3r4Nr|r,rrrdrer2ABC)z%(py3)s in %(py6)s)r0r2z%(py8)s)z%(py11)s in %(py14)s)py11py14z%(py16)spy16)z%(py19)s in %(py22)s)py19py22z%(py24)spy24zassert %(py27)spy27---)r
rrar7r=r5r6r8r9r:r;
splitlinesappend_format_boolop)r
rrFrGrBrHrIlinesrEr?r@rA@py_assert10@py_assert13@py_assert12@py_assert18@py_assert21@py_assert20@py_format15@py_format17@py_format23@py_format25@py_format26@py_format28s                        rtest_simple_tablersHDE$K
(Fv;!;!;!33vv;!1I  "E%(3(?3(3(B3B%(B3(?BsBeAhBshB3B%(B3(?BBBB3(BBB3BBB(BBBBBBBshBBBsBBBhBBBBBBB3(BBB3BBB(BBBBBBBBBBBBBBBE!H5H5H5Hrcd}tt|}t|}d}||k(}|stjd|fd||fdtjvstjtrtjtnddtjvstj|rtj|ndtj|tj|dz}dd	|iz}ttj|d
x}x}}|d}|j}d}	|d}
|	|
v}|sltjd
|fd|	|
ftj|	tj|
dz}dd|iz}ttj|d
x}	x}}
d}	|d}
|	|
v}|sltjd
|fd|	|
ftj|	tj|
dz}dd|iz}ttj|d
x}	x}}
d}	|d}
|	|
v}|sltjd
|fd|	|
ftj|	tj|
dz}dd|iz}ttj|d
x}	x}}
d}	|	|v}|stjd
|fd|	|ftj|	dtjvstj|rtj|nddz}dd|iz}
ttj|
d
x}	}d}	|	|v}|stjd
|fd|	|ftj|	dtjvstj|rtj|nddz}dd|iz}
ttj|
d
x}	}y
)z> row is treated as header with separator line after it.a
    

        
            
NameAge

        
        
            
Alice30

            
Bob25

        
    

    rfr%r`rarrcr3r4NrNamer,rrrdrer2AgerAlicez%(py1)s in %(py3)sr>r"r0rnr)Bob)r
rrar7r=r5r6r8r9r:r;r)r
rrFrGrBrHrIr>rrEr?r@rorCs              rtest_table_with_headersrsT
DE$K
(Fv;!;!;!33vv;!!9DOOEU1X6X6X6XE!H5H5H5HE!H5H5H5H7d?7d7dd5D=5D5DDrcdjdtddD}d|d}t|}d}||v}|stjd|fd	||ftj
|d
t
jvstj|rtj
|nd
dz}dd
|iz}ttj|dx}}t|}d}||kD}|stjd|fd||fdt
jvstjtrtj
tndd
t
jvstj|rtj
|nd
tj
|tj
|dz}	dd|	iz}
ttj|
dx}x}}y)z:A long article page produces meaningful main-content text.
c3(K|]
}d|dyw)z

Paragraph zI: Genesis is an autonomous agentic system built for revenue at scale.
N).0is  r	z4test_readability_extracts_article..s"
scdsrfz
    Article
    
        
        

            
Main Article Title

            zj
        

        
Footer content that should not appear

    
    z'Genesis is an autonomous agentic systemr,rr>rrnr)Nd)>)z/%(py3)s
{%(py3)s = %(py0)s(%(py1)s)
} > %(py6)srarcr3r4)joinrangerr7r=r9r5r6r8r:r;ra)
paragraphsr
r>rErFrorCrGrBrHrIs           r!test_readability_extracts_articlers2q"J
L
D$D)D5<4<<<<4<<<4<<<<<<<<<<<<<<<<t9s9s?9s33tt9srcd}t|}g}d}||v}|}|st|}d}||k\}|}|stjd|fd||ftj|dtjvstj|rtj|nddz}	dd	|	iz}
|j|
|stjd
fdfdtjvstjtrtjtnddtjvstj|rtj|ndtj|tj|d
z}dd|iz}|j|tj|diz}
dd|
iz}ttj|dx}x}x}x}x}x}}y)zFWhen readability produces empty output, body text is returned instead.z7
Fallback text content.
zFallback text contentrr,)z%(py3)s in %(py5)sr>)r0r)z%(py7)sr+)>=)z4%(py12)s
{%(py12)s = %(py9)s(%(py10)s)
} >= %(py15)sra)py9py10py12py15z%(py17)spy17rfzassert %(py20)spy20N)rrar7r=r9r5r6r8rrr:r;)r
r>rArFrBrE@py_assert11@py_assert14rrCrD@py_format16@py_format18@py_format19@py_format21s               rtest_readability_fallbackr/sED#D)D="<"d*Three valid HTML pages produce three ExtractedContent results.rfzPage z
Body z.
zhttps://example.com/rr_r%r`raresultsrcr3r4N)startrrrrr zPage r'r(r*r+)rrrrar7r=r5r6r8r9r:r;	enumeraterrr<)rpagesrrFrGrBrHrIrr?r@rArCrDs              rtest_batch_all_goodr=s
q!

	-aS0MaSPcd-aS1	3
E

E"Gw<1<1<133ww<1wa0+	6&"233333333z333z333333&333&333333"2333"23333333333||*qc{*|{****|{******v***v***|***{*******+
sM
crtdd}tddd}tdd	}t|||g}t|}d
}||k(}|stjd|fd||fd
tjvstjtrtjtnd
dtjvstj|rtj|ndtj|tj|dz}dd|iz}ttj|dx}x}}|d}	t|	t}|sddtjvstjtrtjtndtj|	dtjvstjtrtjtndtj|dz}
ttj|
dx}	}|d}d}||u}|sltjd|fd||ftj|tj|dz}
dd|
iz}ttj|dx}x}}|d}	t|	t}|sddtjvstjtrtjtndtj|	dtjvstjtrtjtndtj|dz}
ttj|
dx}	}y)zIA page with non-HTML content_type returns None; good pages still succeed.zDGood
OK.
zhttps://example.com/goodrznot htmlzhttps://example.com/pdfzapplication/pdf)rrzNAlso Good
Also OK.
zhttps://example.com/also-goodr_r%r`rarrcr3r4Nrz5assert %(py5)s
{%(py5)s = %(py0)s(%(py2)s, %(py3)s)
}rr)r!r#r0r)rf)is)z%(py1)s is %(py4)srdrer2rg)
rrrar7r=r5r6r8r9r:r;rr)	good_page
non_html_pageanother_goodrrFrGrBrHrIrArCrEr?r@s              rtest_batch_with_failurerLs&N&I%&M
X+L
Y
|DEGw<1<1<133ww<1aj3:j"233333333:333:333j333333"2333"233333333331::::aj3:j"233333333:333:333j333333"2333"23333333333rcftg}g}||k(}|stjd|fd||fdtjvstj
|rtj|ndtj|dz}dd|iz}ttj|dx}}y)z.An empty list of pages produces an empty list.r%rlrrmrnr)N)	rr7r=r5r6r8r9r:r;)rrFrArorCs     rtest_batch_emptyrdsdBG7b=7b77br)zhttps://example.com/pagez	text/html)*__doc__builtinsr5_pytest.assertion.rewrite	assertionrewriter7pytestbs4rcore.kb.contractsrrcore.kb.extractorrrr	r
rrstrrrrJrRrUrYr[r]rjrprsr|r~rrrrrrrrrrrrs
;&@#.	S	s	 	3>	''
'")(	(	,3,E
("
%'(@4=+40r