
    *j                        d Z ddlmZ ddlZddlmc mZ ddl	Z	ddl
mZ ddlZddlmZmZmZmZ  ee          j        j        dz  dz  ZdZd	 Zd
 Zd Zd Zd Zd Zd Zd ZdS )z'TDD tests for compiler/stages/split.py.    )annotationsN)Path)HOLDOUT_RATE_parse_paragraphsassert_holdout_disjointsplit_corpusfixtureshttpa  ---
source_id: test-src
title: Test
rights: public-domain
fetch_type: gutenberg
---

# Test Title

This is the first paragraph of sufficient length to be included in the corpus.

This is the second paragraph and it too has enough characters to meet the minimum.

Third paragraph here with enough words to pass the minimum character threshold check.

Fourth paragraph here with enough words to pass the minimum character threshold check.

Fifth paragraph here with enough words to pass the minimum character threshold check.

Sixth paragraph here with enough words to pass the minimum character threshold check.

Seventh paragraph here with enough words to pass the minimum character threshold check.

Eighth paragraph here with enough words to pass the minimum character threshold check.

Ninth paragraph here with enough words to pass the minimum character threshold check.

Tenth paragraph here with enough words to pass the minimum character threshold check.
c                    t          t                    } | D ]I}d}||v}|st          j        d|fd||f          t          j        |          dt          j                    v st          j        |          rt          j        |          nddz  }dd|iz  }t          t          j	        |                    d x}}d}||v}|st          j        d|fd||f          t          j        |          dt          j                    v st          j        |          rt          j        |          nddz  }dd|iz  }t          t          j	        |                    d x}}Kd S )	N	source_id)not in)z%(py1)s not in %(py3)sp)py1py3zassert %(py5)spy5z---)
r   	SAMPLE_MD
@pytest_ar_call_reprcompare	_saferepr@py_builtinslocals_should_repr_global_nameAssertionError_format_explanation)parasr   @py_assert0@py_assert2@py_format4@py_format6s         B/Users/jameslopez/projects/MentorCore/tests/compiler/test_split.py(test_parse_paragraphs_strips_frontmatterr!   5   s   i((E  #{!##########{!#####{###########!#####!###########################uA~uAuAA     c                    d} d|  }t          |          }d |D             }t          |          }| }|sddt          j                    v st	          j        t                    rt	          j        t                    ndt	          j        |          t	          j        |          dz  }t          t	          j        |                    d x}x}}d |D             }t          |          }|sddt          j                    v st	          j        t                    rt	          j        t                    ndt	          j        |          t	          j        |          dz  }t          t	          j        |                    d x}}d S )	NzqThis is a long enough paragraph to pass the minimum character threshold. More words added here to make it longer.z#---
foo: bar
---

# Title

Short.

c              3  "   K   | ]
}|d k    V  dS )zShort.N .0r   s     r    	<genexpr>z6test_parse_paragraphs_filters_short.<locals>.<genexpr>C   s&      00Q1=000000r"   z0assert not %(py4)s
{%(py4)s = %(py0)s(%(py2)s)
}anypy0py2py4c              3     K   | ]}d |v V  	dS )zlong enoughNr%   r&   s     r    r(   z6test_parse_paragraphs_filters_short.<locals>.<genexpr>D   s'      11a}!111111r"   z,assert %(py4)s
{%(py4)s = %(py0)s(%(py2)s)
})	r   r)   r   r   r   r   r   r   r   )	long_paramdr   @py_assert1@py_assert3@py_assert5r   @py_format5s           r    #test_parse_paragraphs_filters_shortr5   <   s   	4  
Cy	B	BBb!!E00%0000s0000000000000000000s00000s0000000000000000000000000000000001151111311111111111111113111113111111111111111111111111111111111r"   c                    t           dz                                  } ddlm}  ||           }d| }t	          |          }t          |          }d}||k    }|st          j        d|fd||f          dt          j	                    v st          j
        t
                    rt          j        t
                    nddt          j	                    v st          j
        |          rt          j        |          ndt          j        |          t          j        |          d	z  }d
d|iz  }	t          t          j        |	                    d x}x}}d S )Nzgutenberg_2680.txtr   )_strip_gutenbergz---
source_id: x
---

# Title

>z/%(py3)s
{%(py3)s = %(py0)s(%(py1)s)
} > %(py6)slenr   r+   r   r   py6assert %(py8)spy8)FIXTURES_HTTP	read_textcompiler.stages.normalizer7   r   r;   r   r   r   r   r   r   r   r   )
rawr7   body	pseudo_mdr   r   r3   @py_assert4@py_format7@py_format9s
             r    7test_parse_paragraphs_returns_nonempty_for_real_fixturerI   G   su   //
:
:
<
<C::::::C  D>>>Ii((Eu:::>:33uu:r"   c                ~   | dz  dz  }|j                             d           |                    t                     | dz  }t	          d|i|           |dz  }|j        } |            }|sdd	t          j                    v st          j	        |          rt          j
        |          nd	t          j
        |          t          j
        |          d
z  }t          t          j        |                    d x}}d |                                                                D             }t          |          }d}	||	k    }
|
st          j        d|
fd||	f          dt          j                    v st          j	        t                    rt          j
        t                    nddt          j                    v st          j	        |          rt          j
        |          ndt          j
        |          t          j
        |	          dz  }dd|iz  }t          t          j        |                    d x}x}
}	d S )Ncorpustest-src.mdTparentsstyletest-srcholdout.jsonlzAassert %(py4)s
{%(py4)s = %(py2)s
{%(py2)s = %(py0)s.exists
}()
}holdoutr*   c                ^    g | ]*}|                                 t          j        |          +S r%   stripjsonloadsr'   lns     r    
<listcomp>z;test_split_corpus_produces_holdout_file.<locals>.<listcomp>^   -    WWW"BHHJJWtz"~~WWWr"   r   r8   r:   r;   recordsr<   r>   r?   )parentmkdir
write_textr   r   existsr   r   r   r   r   r   r   rA   
splitlinesr;   r   )tmp_pathcorpus_path	style_dirrR   r1   r2   r4   r\   r   r3   rF   rG   rH   s                r    'test_split_corpus_produces_holdout_filere   T   s\   X%5KT***9%%%7"I*k*I666/)G>>>77>WW(9(9(;(;(F(F(H(HWWWGw<<!<!<!33ww<!r"   c                   | dz  dz  }|j                             d           |                    t                     | dz  }t	          d|i|          }|dz  }d |                                                                D             }t          |d                   t          |          z   }|rt          |          |z  nd	}d
}t          |z  }	d}
|	|
z   }||k    }|st          j
        d|fd||f          dt          j                    v st          j        |          rt          j        |          nddt          j                    v st          j        t                    rt          j        t                    ndt          j        |          t          j        |
          dz  }dd|iz  }t          t          j        |                    d x}x}x}	x}
}d S )NrK   rL   TrM   rO   rP   rQ   c                ^    g | ]*}|                                 t          j        |          +S r%   rT   rX   s     r    rZ   zHtest_split_corpus_holdout_rate_approximately_correct.<locals>.<listcomp>k   s-    YYYrbhhjjYDJrNNYYYr"   r      g?)<=)z*%(py0)s <= ((%(py2)s * %(py4)s) + %(py7)s)actual_rater   )r+   r,   r-   py7zassert %(py10)spy10)r]   r^   r_   r   r   rA   ra   r;   r   r   r   r   r   r   r   r   r   )rb   rc   rd   	indexableholdout_pathheldtotalrj   r2   r3   @py_assert6@py_assert8r1   rH   @py_format11s                  r    4test_split_corpus_holdout_rate_approximately_correctrt   b   s   X%5KT***9%%%7"Ij+6	BBI.LYY\%;%;%=%=%H%H%J%JYYYD	*%&&T2E',3#d))e##!K)*1,*1T1*T11;11111111111;1111111111111;11111;1111111111111,11111,1111111111T111111111111111111111111111111111r"   c                    | dz  dz  }|j                             d           |                    t                     | dz  }t	          d|i|          }t          ||           d S )NrK   rL   TrM   rO   rP   )r]   r^   r_   r   r   r   )rb   rc   rd   rm   s       r    test_holdout_disjoint_passesrv   r   ss    X%5KT***9%%%7"Ij+6	BBIIy11111r"   c                0   | dz  }|                                  d}|dz                      t          j        d|d          dz              d|dgi}t	          j        t          d	          5  t          ||           d d d            d S # 1 swxY w Y   d S )
NrO   z.This paragraph is in both indexed and holdout.rQ   s)r   text
z)Another paragraph that is not in holdout.rR   )match)r^   r_   rV   dumpspytestraisesr   r   )rb   rd   leaking_pararm   s       r    &test_holdout_disjoint_fails_on_overlapr   |   s    7"IOOCL ,,
l;;<<tC   |%PQRI	~Y	7	7	7 6 6	95556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s   -BBBc                   | dz                                    | dz  }t          d          D ]#}d| dz                      t                     $fdt          d          D             }t	          ||          }t          ||           |dz  }d |                                                                D             }d	 |D             }t          |          }d}	||	k    }
|
st          j
        d
|
fd||	f          dt          j                    v st          j        t                    rt          j        t                    nddt          j                    v st          j        |          rt          j        |          ndt          j        |          t          j        |	          dz  }dd|iz  }t          t          j        |                    d x}x}
}	d S )NrK   rO      src-.mdc                *    i | ]}d | d | dz  S )r   r   r%   )r'   i
corpus_dirs     r    
<dictcomp>z6test_split_corpus_multiple_sources.<locals>.<dictcomp>   s.    MMMqJ1JJ
]A]]] :MMMr"   rQ   c                ^    g | ]*}|                                 t          j        |          +S r%   rT   rX   s     r    rZ   z6test_split_corpus_multiple_sources.<locals>.<listcomp>   r[   r"   c                    h | ]
}|d          S )r   r%   )r'   rs     r    	<setcomp>z5test_split_corpus_multiple_sources.<locals>.<setcomp>   s    222Q!K.222r"   )==)z0%(py3)s
{%(py3)s = %(py0)s(%(py1)s)
} == %(py6)sr;   
source_idsr<   r>   r?   )r^   ranger_   r   r   r   rA   ra   r;   r   r   r   r   r   r   r   r   )rb   rd   r   corpus_pathsrm   rR   r\   r   r   r3   rF   rG   rH   r   s                @r    "test_split_corpus_multiple_sourcesr      s	   H$J7"I1XX ; ;	mQmmm	#//	::::MMMME!HHMMML\955IIy111/)GWW(9(9(;(;(F(F(H(HWWWG22'222Jz??a?a?a33zz?ar"   )__doc__
__future__r   builtinsr   _pytest.assertion.rewrite	assertionrewriter   rV   pathlibr   r}   compiler.stages.splitr   r   r   r   __file__r]   r@   r   r!   r5   rI   re   rt   rv   r   r   r%   r"   r    <module>r      s@   - - " " " " " "                                 X%,z9FB	F  2 2 2    2 2 2 2 2 26 6 6         r"   