219 lines
7.4 KiB
Python
219 lines
7.4 KiB
Python
|
""" Reads the Morphalou dataset, in its TSV form """
|
||
|
|
|
||
|
|
import typing as t
|
||
|
|
from lxml import etree
|
||
|
|
from pathlib import Path
|
||
|
|
import itertools
|
||
|
|
|
||
|
|
from .word_db import Adjectif, Adverbe, Genre, Nom, Nombre, Temps, Verbe, WordDb
|
||
|
|
|
||
|
|
TSV_NS = {
|
||
|
|
"tsv": "http://www.tei-c.org/ns/1.0",
|
||
|
|
"xml": "http://www.w3.org/XML/1998/namespace",
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
class MorphalouSet:
|
||
|
|
MORPHALOU_DIR_PATH = (
|
||
|
|
Path(__file__).parent.parent
|
||
|
|
/ "data/raw/morphalou/morphalou/5/Morphalou3.1_formatTEI"
|
||
|
|
)
|
||
|
|
MORPHALOU_FILENAME_TEMPLATE = "{cat_name}_Morphalou3.1_TEI.xml"
|
||
|
|
|
||
|
|
CAT_MAPPING: dict[t.Type[t.NamedTuple], str] = {
|
||
|
|
Nom: "commonNoun",
|
||
|
|
Adjectif: "adjective",
|
||
|
|
Verbe: "verb",
|
||
|
|
Adverbe: "adverb",
|
||
|
|
}
|
||
|
|
|
||
|
|
word_db: WordDb
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
self.word_db = WordDb()
|
||
|
|
|
||
|
|
def parse(self):
|
||
|
|
"""Parses the dataset"""
|
||
|
|
for cat, cat_file in self.__class__.CAT_MAPPING.items():
|
||
|
|
word_db_elt = WordDb.CATEGORY_TO_ATTR[cat]
|
||
|
|
setattr(
|
||
|
|
self.word_db,
|
||
|
|
word_db_elt,
|
||
|
|
getattr(self, f"_parse_{word_db_elt}")(
|
||
|
|
self.__class__.MORPHALOU_DIR_PATH
|
||
|
|
/ self.__class__.MORPHALOU_FILENAME_TEMPLATE.format(
|
||
|
|
cat_name=cat_file
|
||
|
|
)
|
||
|
|
),
|
||
|
|
)
|
||
|
|
|
||
|
|
def _tsv_elems(self, tsv_path: Path):
|
||
|
|
"""Opens a TSV file, and returns the <body> node, direct parent of all the
|
||
|
|
relevant nodes"""
|
||
|
|
with tsv_path.open("r") as h:
|
||
|
|
tree = etree.parse(h)
|
||
|
|
root = tree.getroot()
|
||
|
|
body = root.find("./tsv:text/tsv:body", TSV_NS)
|
||
|
|
return body
|
||
|
|
|
||
|
|
def _parse_noms(self, tsv_path: Path) -> list[Nom]:
|
||
|
|
"""Parse the nouns"""
|
||
|
|
root = self._tsv_elems(tsv_path)
|
||
|
|
out: list[Nom] = []
|
||
|
|
|
||
|
|
for entry in root.iterfind("./tsv:entry", TSV_NS):
|
||
|
|
try:
|
||
|
|
genre = self._genre(
|
||
|
|
entry.find(
|
||
|
|
"./tsv:form[@type='lemma']/tsv:gramGrp/tsv:gen", TSV_NS
|
||
|
|
).text
|
||
|
|
)
|
||
|
|
except AttributeError:
|
||
|
|
continue # some nouns don't have a gender defined, somehow -- ignore
|
||
|
|
|
||
|
|
forms = {}
|
||
|
|
for inflected in entry.iterfind("./tsv:form[@type='inflected']", TSV_NS):
|
||
|
|
orth = inflected.find("./tsv:orth", TSV_NS).text
|
||
|
|
nombres = self._nombre_set(
|
||
|
|
inflected.find("./tsv:gramGrp/tsv:number", TSV_NS).text
|
||
|
|
)
|
||
|
|
for form in nombres:
|
||
|
|
forms[form] = orth
|
||
|
|
try:
|
||
|
|
out.append(
|
||
|
|
Nom(
|
||
|
|
genre=genre,
|
||
|
|
sing=forms[Nombre.SING],
|
||
|
|
plur=forms[Nombre.PLUR],
|
||
|
|
)
|
||
|
|
)
|
||
|
|
except KeyError:
|
||
|
|
continue # cannot be inflected to all required forms: skip
|
||
|
|
|
||
|
|
return out
|
||
|
|
|
||
|
|
def _parse_adjectifs(self, tsv_path: Path) -> list[Adjectif]:
|
||
|
|
"""Parse the adjectives"""
|
||
|
|
root = self._tsv_elems(tsv_path)
|
||
|
|
out: list[Adjectif] = []
|
||
|
|
|
||
|
|
for entry in root.iterfind("./tsv:entry", TSV_NS):
|
||
|
|
forms = {}
|
||
|
|
for inflected in entry.iterfind("./tsv:form[@type='inflected']", TSV_NS):
|
||
|
|
orth = inflected.find("./tsv:orth", TSV_NS).text
|
||
|
|
gram_grp = inflected.find("./tsv:gramGrp", TSV_NS)
|
||
|
|
genres = self._genre_set(gram_grp.find("./tsv:gen", TSV_NS).text)
|
||
|
|
nombres = self._nombre_set(gram_grp.find("./tsv:number", TSV_NS).text)
|
||
|
|
|
||
|
|
for form in itertools.product(genres, nombres):
|
||
|
|
forms[form] = orth
|
||
|
|
try:
|
||
|
|
out.append(
|
||
|
|
Adjectif(
|
||
|
|
masc_sing=forms[Genre.MASC, Nombre.SING],
|
||
|
|
masc_plur=forms[Genre.MASC, Nombre.PLUR],
|
||
|
|
fem_sing=forms[Genre.FEM, Nombre.SING],
|
||
|
|
fem_plur=forms[Genre.FEM, Nombre.PLUR],
|
||
|
|
)
|
||
|
|
)
|
||
|
|
except KeyError:
|
||
|
|
continue # cannot be inflected to all required forms: skip
|
||
|
|
|
||
|
|
return out
|
||
|
|
|
||
|
|
def _parse_verbes(self, tsv_path: Path) -> list[Verbe]:
|
||
|
|
"""Parse the verbs"""
|
||
|
|
root = self._tsv_elems(tsv_path)
|
||
|
|
out: list[Verbe] = []
|
||
|
|
|
||
|
|
for entry in root.iterfind("./tsv:entry", TSV_NS):
|
||
|
|
forms = {}
|
||
|
|
for inflected in entry.iterfind("./tsv:form[@type='inflected']", TSV_NS):
|
||
|
|
gram_grp = inflected.find("./tsv:gramGrp", TSV_NS)
|
||
|
|
|
||
|
|
# Order of tests is important! If mood == 'participle', there is no
|
||
|
|
# 'person' defined.
|
||
|
|
if (
|
||
|
|
gram_grp.find("./tsv:mood", TSV_NS).text != "indicative"
|
||
|
|
or gram_grp.find("./tsv:per", TSV_NS).text != "thirdPerson"
|
||
|
|
):
|
||
|
|
continue # irrelevant for us
|
||
|
|
|
||
|
|
temps = self._tense(gram_grp.find("./tsv:tns", TSV_NS).text)
|
||
|
|
if temps is None:
|
||
|
|
continue # irrelevant for us
|
||
|
|
|
||
|
|
nombres = self._nombre_set(gram_grp.find("./tsv:number", TSV_NS).text)
|
||
|
|
|
||
|
|
orth = inflected.find("./tsv:orth", TSV_NS).text
|
||
|
|
for nombre in nombres:
|
||
|
|
forms[(temps, nombre)] = orth
|
||
|
|
try:
|
||
|
|
out.append(
|
||
|
|
Verbe(
|
||
|
|
present_sing=forms[Temps.PRESENT, Nombre.SING],
|
||
|
|
present_plur=forms[Temps.PRESENT, Nombre.PLUR],
|
||
|
|
futur_sing=forms[Temps.FUTUR, Nombre.SING],
|
||
|
|
futur_plur=forms[Temps.FUTUR, Nombre.PLUR],
|
||
|
|
imparfait_sing=forms[Temps.IMPARFAIT, Nombre.SING],
|
||
|
|
imparfait_plur=forms[Temps.IMPARFAIT, Nombre.PLUR],
|
||
|
|
)
|
||
|
|
)
|
||
|
|
except KeyError:
|
||
|
|
continue # cannot be inflected to all required forms: skip
|
||
|
|
|
||
|
|
return out
|
||
|
|
|
||
|
|
def _parse_adverbes(self, tsv_path: Path) -> list[Adverbe]:
|
||
|
|
"""Parse the adverbs"""
|
||
|
|
root = self._tsv_elems(tsv_path)
|
||
|
|
out: list[Adverbe] = []
|
||
|
|
|
||
|
|
for entry in root.iterfind("./tsv:entry", TSV_NS):
|
||
|
|
# We're only interested in the lemma form
|
||
|
|
orth = entry.find("./tsv:form[@type='lemma']/tsv:orth", TSV_NS)
|
||
|
|
assert orth is not None
|
||
|
|
adv = orth.text
|
||
|
|
out.append(Adverbe(adv=adv))
|
||
|
|
|
||
|
|
return out
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _genre_set(genre: str) -> list[Genre]:
|
||
|
|
return {
|
||
|
|
"masculine": [Genre.MASC],
|
||
|
|
"feminine": [Genre.FEM],
|
||
|
|
"invariable": [Genre.MASC, Genre.FEM],
|
||
|
|
}[genre]
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _genre(genre: str) -> Genre:
|
||
|
|
return {
|
||
|
|
"masculine": Genre.MASC,
|
||
|
|
"feminine": Genre.FEM,
|
||
|
|
"invariable": Genre.INV,
|
||
|
|
}[genre]
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _nombre(nombre: str) -> Nombre:
|
||
|
|
return {
|
||
|
|
"singular": Nombre.SING,
|
||
|
|
"plural": Nombre.PLUR,
|
||
|
|
}[nombre]
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _nombre_set(nombre: str) -> list[Nombre]:
|
||
|
|
return {
|
||
|
|
"singular": [Nombre.SING],
|
||
|
|
"plural": [Nombre.PLUR],
|
||
|
|
"invariable": [Nombre.SING, Nombre.PLUR],
|
||
|
|
}[nombre]
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _tense(tense: str) -> t.Optional[Temps]:
|
||
|
|
return {
|
||
|
|
"present": Temps.PRESENT,
|
||
|
|
"imperfect": Temps.IMPARFAIT,
|
||
|
|
"future": Temps.FUTUR,
|
||
|
|
}.get(tense, None)
|