# Obeliks tokenisation rules - Part 1: pre-processing.
#
# One rule per line: <search regex> == > <replacement>.
# Lines starting with '#' are comments; a separator of '-->' (instead of '==>')
# enables case-insensitive matching (see the E-mail rule below).

# XML/HTML entities
\< ==> &lt;
\> ==> &gt;

# Punctuation, paragraphs, spaces
([[\u0000-\u001F\u0021-\u002F\u003A-\u0040\u005B-\u0060\u007B-\u009F\u00A1-\u00A9\u00AB-\u00B1\u00B4\u00B6-\u00B8\u00BB\u00BF\u00D7\u00F7\u0250-\u1DFF\u1E9C-\u1E9F\u1EFA-\uFFFF]&&[^\s]&&[^\p{L}\p{M}]]) ==> <c>\1</c>
<c>&</c>((amp)|(lt)|(gt))<c>;</c> ==> <c>&\1;</c>
^ ==> <p>
$ ==> </p>
\s ==> <S/>

# Words
<S/> ==> </w><S/><w>
<c> ==> </w><c>
</c> ==> </c><w>
<p> ==> <p><w>
</p> ==> </w></p>

# Cleanup
</p>(\n*)</w><S/><w> ==> </p>\1
<w></w> ==>

# E-mail (case-insensitive regex matching is denoted with '-->')
# (TB.)*TB@(TB.)*TB.TLD
# TB: ((<w>[\p{L}0-9]+</w>)|(<c>[_-]</c>))+
# TLD: <w>[\p{L}]{2,6}</w>
(((<w>[A-Za-z0-9]+</w>)|(<c>[_-]</c>))+<c>\.</c>)*((<w>[A-Za-z0-9]+</w>)|(<c>[_-]</c>))+<c>@</c>(((<w>[A-Za-z0-9]+</w>)|(<c>[_-]</c>))+<c>\.</c>)*((<w>[A-Za-z0-9]+</w>)|(<c>[_-]</c>))+<c>\.</c><w>[A-Za-z]{2,6}</w> --> <w>$txt</w>

# URL
#(<w>((ftp)|(https?))</w><c>:</c><c>/</c><c>/</c>)?(((<w>[0-9\p{L}]+</w>)|(<c>[~_!*'-]</c>))+<c>[\.-]</c>)*<w>([0-9\p{L}][0-9\p{L}-]{0,61})?[0-9\p{L}]</w><c>\.</c><w>[\p{L}]{2,6}</w>(<c>/</c>((<w>[0-9\p{L}]+</w>)|(<c>[~_!*'().;?:@=+$,%#-]</c>)|(<c>&</c>))*((<w>[0-9\p{L}]+</w>)|(<c>[~_*'@=+$%#-]</c>)|(<c>&</c>)))*(<c>/</c>)? --> <w>$txt</w>
(<w>((ftp)|(https?))</w><c>:</c><c>/</c><c>/</c>)?(((<w>[0-9A-Za-z]+</w>)|(<c>[~_!*'-]</c>))+<c>[\.-]</c>)*<w>([0-9A-Za-z][0-9A-Za-z-]{0,61})?[0-9A-Za-z]</w><c>\.</c><w>[A-Za-z]{2,6}</w>(<c>/</c>((<w>[0-9A-Za-z]+</w>)|(<c>[~_!*'().;?:@=+$,%#-]</c>)|(<c>&</c>))*((<w>[0-9A-Za-z]+</w>)|(<c>[~_*'@=+$%#-]</c>)|(<c>&</c>)))*(<c>/</c>)? --> <w>$txt</w>

# Brioni rule #1.2
<w>([^<.]+)\.(\p{Lu}) ==> <w>\1</w><c>.</c></s><s><w>\2

# Suffixes (incl. Brioni rule #2.6)
<w>([^<]+)</w><c>([-–—])</c><w>((a)|(e)|(em)|(ema)|(ev)|(eva)|(evca)|(evce)|(evcem)|(evcema)|(evcev)|(evci)|(evcih)|(evcimi)|(evcu)|(eve)|(evec)|(evega)|(evem)|(evemu)|(evi)|(evih)|(evim)|(evima)|(evimi)|(evk)|(evka)|(evkah)|(evkam)|(evkama)|(evkami)|(evke)|(evki)|(evko)|(evo)|(evska)|(evske)|(evskega)|(evskem)|(evskemu)|(evski)|(evskih)|(evskim)|(evskima)|(evskimi)|(evsko)|(i)|(ih)|(ja)|(je)|(jem)|(jema)|(jev)|(jeva)|(jevca)|(jevce)|(jevcem)|(jevcema)|(jevcev)|(jevci)|(jevcih)|(jevcimi)|(jevcu)|(jeve)|(jevec)|(jevega)|(jevem)|(jevemu)|(jevi)|(jevih)|(jevim)|(jevima)|(jevimi)|(jevk)|(jevka)|(jevkah)|(jevkam)|(jevkama)|(jevkami)|(jevke)|(jevki)|(jevko)|(jevo)|(jevska)|(jevske)|(jevskega)|(jevskem)|(jevskemu)|(jevski)|(jevskih)|(jevskim)|(jevskima)|(jevskimi)|(jevsko)|(ji)|(jih)|(jin)|(jina)|(jine)|(jinega)|(jinem)|(jinemu)|(jini)|(jinih)|(jinim)|(jinima)|(jinimi)|(jino)|(ju)|(om)|(oma)|(ov)|(ova)|(ovca)|(ovce)|(ovcem)|(ovcema)|(ovcev)|(ovci)|(ovcih)|(ovcimi)|(ovcu)|(ove)|(ovec)|(ovega)|(ovem)|(ovemu)|(ovi)|(ovih)|(ovim)|(ovima)|(ovimi)|(ovk)|(ovka)|(ovkah)|(ovkam)|(ovkama)|(ovkami)|(ovke)|(ovki)|(ovko)|(ovo)|(ovska)|(ovske)|(ovskega)|(ovskem)|(ovskemu)|(ovski)|(ovskih)|(ovskim)|(ovskima)|(ovskimi)|(ovsko)|(u))</w> ==> <w>\1\2\3</w>

<w>(\d+)</w><c>([-–—])</c><w>((a)|(e)|(ega)|(em)|(emu)|(i)|(ih)|(im)|(ima)|(imi)|(o)|(ta)|(te)|(tega)|(tem)|(temu)|(ti)|(tih)|(tim)|(tima)|(timi)|(to))</w> ==> <w>\1\2\3</w>

</w><c>'</c><w> ==> '

(<w>\d+</w><c>[.,:]</c>)+<w>\d+\p{L}*</w> ==> <w>$txt</w>

# Throw out control characters
<c>[\u0000-\u001F\u007F\u0080-\u009F]</c> ==>

#
# End of Part 1
#
