Compare commits
2 commits
f3df51ae26
...
a9c3c90405
Author | SHA1 | Date | |
---|---|---|---|
a9c3c90405 | |||
b086b9a08d |
2 changed files with 261 additions and 2 deletions
218
pwgen_fr/morphalou.py
Normal file
218
pwgen_fr/morphalou.py
Normal file
|
@ -0,0 +1,218 @@
|
||||||
|
""" Reads the Morphalou dataset, in its TSV form """
|
||||||
|
|
||||||
|
import typing as t
|
||||||
|
from lxml import etree
|
||||||
|
from pathlib import Path
|
||||||
|
import itertools
|
||||||
|
|
||||||
|
from .word_db import Adjectif, Adverbe, Genre, Nom, Nombre, Temps, Verbe, WordDb
|
||||||
|
|
||||||
|
TSV_NS = {
|
||||||
|
"tsv": "http://www.tei-c.org/ns/1.0",
|
||||||
|
"xml": "http://www.w3.org/XML/1998/namespace",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class MorphalouSet:
|
||||||
|
MORPHALOU_DIR_PATH = (
|
||||||
|
Path(__file__).parent.parent
|
||||||
|
/ "data/raw/morphalou/morphalou/5/Morphalou3.1_formatTEI"
|
||||||
|
)
|
||||||
|
MORPHALOU_FILENAME_TEMPLATE = "{cat_name}_Morphalou3.1_TEI.xml"
|
||||||
|
|
||||||
|
CAT_MAPPING: dict[t.Type[t.NamedTuple], str] = {
|
||||||
|
Nom: "commonNoun",
|
||||||
|
Adjectif: "adjective",
|
||||||
|
Verbe: "verb",
|
||||||
|
Adverbe: "adverb",
|
||||||
|
}
|
||||||
|
|
||||||
|
word_db: WordDb
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.word_db = WordDb()
|
||||||
|
|
||||||
|
def parse(self):
|
||||||
|
"""Parses the dataset"""
|
||||||
|
for cat, cat_file in self.__class__.CAT_MAPPING.items():
|
||||||
|
word_db_elt = WordDb.CATEGORY_TO_ATTR[cat]
|
||||||
|
setattr(
|
||||||
|
self.word_db,
|
||||||
|
word_db_elt,
|
||||||
|
getattr(self, f"_parse_{word_db_elt}")(
|
||||||
|
self.__class__.MORPHALOU_DIR_PATH
|
||||||
|
/ self.__class__.MORPHALOU_FILENAME_TEMPLATE.format(
|
||||||
|
cat_name=cat_file
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _tsv_elems(self, tsv_path: Path):
|
||||||
|
"""Opens a TSV file, and returns the <body> node, direct parent of all the
|
||||||
|
relevant nodes"""
|
||||||
|
with tsv_path.open("r") as h:
|
||||||
|
tree = etree.parse(h)
|
||||||
|
root = tree.getroot()
|
||||||
|
body = root.find("./tsv:text/tsv:body", TSV_NS)
|
||||||
|
return body
|
||||||
|
|
||||||
|
def _parse_noms(self, tsv_path: Path) -> list[Nom]:
|
||||||
|
"""Parse the nouns"""
|
||||||
|
root = self._tsv_elems(tsv_path)
|
||||||
|
out: list[Nom] = []
|
||||||
|
|
||||||
|
for entry in root.iterfind("./tsv:entry", TSV_NS):
|
||||||
|
try:
|
||||||
|
genre = self._genre(
|
||||||
|
entry.find(
|
||||||
|
"./tsv:form[@type='lemma']/tsv:gramGrp/tsv:gen", TSV_NS
|
||||||
|
).text
|
||||||
|
)
|
||||||
|
except AttributeError:
|
||||||
|
continue # some nouns don't have a gender defined, somehow -- ignore
|
||||||
|
|
||||||
|
forms = {}
|
||||||
|
for inflected in entry.iterfind("./tsv:form[@type='inflected']", TSV_NS):
|
||||||
|
orth = inflected.find("./tsv:orth", TSV_NS).text
|
||||||
|
nombres = self._nombre_set(
|
||||||
|
inflected.find("./tsv:gramGrp/tsv:number", TSV_NS).text
|
||||||
|
)
|
||||||
|
for form in nombres:
|
||||||
|
forms[form] = orth
|
||||||
|
try:
|
||||||
|
out.append(
|
||||||
|
Nom(
|
||||||
|
genre=genre,
|
||||||
|
sing=forms[Nombre.SING],
|
||||||
|
plur=forms[Nombre.PLUR],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except KeyError:
|
||||||
|
continue # cannot be inflected to all required forms: skip
|
||||||
|
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_adjectifs(self, tsv_path: Path) -> list[Adjectif]:
|
||||||
|
"""Parse the adjectives"""
|
||||||
|
root = self._tsv_elems(tsv_path)
|
||||||
|
out: list[Adjectif] = []
|
||||||
|
|
||||||
|
for entry in root.iterfind("./tsv:entry", TSV_NS):
|
||||||
|
forms = {}
|
||||||
|
for inflected in entry.iterfind("./tsv:form[@type='inflected']", TSV_NS):
|
||||||
|
orth = inflected.find("./tsv:orth", TSV_NS).text
|
||||||
|
gram_grp = inflected.find("./tsv:gramGrp", TSV_NS)
|
||||||
|
genres = self._genre_set(gram_grp.find("./tsv:gen", TSV_NS).text)
|
||||||
|
nombres = self._nombre_set(gram_grp.find("./tsv:number", TSV_NS).text)
|
||||||
|
|
||||||
|
for form in itertools.product(genres, nombres):
|
||||||
|
forms[form] = orth
|
||||||
|
try:
|
||||||
|
out.append(
|
||||||
|
Adjectif(
|
||||||
|
masc_sing=forms[Genre.MASC, Nombre.SING],
|
||||||
|
masc_plur=forms[Genre.MASC, Nombre.PLUR],
|
||||||
|
fem_sing=forms[Genre.FEM, Nombre.SING],
|
||||||
|
fem_plur=forms[Genre.FEM, Nombre.PLUR],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except KeyError:
|
||||||
|
continue # cannot be inflected to all required forms: skip
|
||||||
|
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_verbes(self, tsv_path: Path) -> list[Verbe]:
|
||||||
|
"""Parse the verbs"""
|
||||||
|
root = self._tsv_elems(tsv_path)
|
||||||
|
out: list[Verbe] = []
|
||||||
|
|
||||||
|
for entry in root.iterfind("./tsv:entry", TSV_NS):
|
||||||
|
forms = {}
|
||||||
|
for inflected in entry.iterfind("./tsv:form[@type='inflected']", TSV_NS):
|
||||||
|
gram_grp = inflected.find("./tsv:gramGrp", TSV_NS)
|
||||||
|
|
||||||
|
# Order of tests is important! If mood == 'participle', there is no
|
||||||
|
# 'person' defined.
|
||||||
|
if (
|
||||||
|
gram_grp.find("./tsv:mood", TSV_NS).text != "indicative"
|
||||||
|
or gram_grp.find("./tsv:per", TSV_NS).text != "thirdPerson"
|
||||||
|
):
|
||||||
|
continue # irrelevant for us
|
||||||
|
|
||||||
|
temps = self._tense(gram_grp.find("./tsv:tns", TSV_NS).text)
|
||||||
|
if temps is None:
|
||||||
|
continue # irrelevant for us
|
||||||
|
|
||||||
|
nombres = self._nombre_set(gram_grp.find("./tsv:number", TSV_NS).text)
|
||||||
|
|
||||||
|
orth = inflected.find("./tsv:orth", TSV_NS).text
|
||||||
|
for nombre in nombres:
|
||||||
|
forms[(temps, nombre)] = orth
|
||||||
|
try:
|
||||||
|
out.append(
|
||||||
|
Verbe(
|
||||||
|
present_sing=forms[Temps.PRESENT, Nombre.SING],
|
||||||
|
present_plur=forms[Temps.PRESENT, Nombre.PLUR],
|
||||||
|
futur_sing=forms[Temps.FUTUR, Nombre.SING],
|
||||||
|
futur_plur=forms[Temps.FUTUR, Nombre.PLUR],
|
||||||
|
imparfait_sing=forms[Temps.IMPARFAIT, Nombre.SING],
|
||||||
|
imparfait_plur=forms[Temps.IMPARFAIT, Nombre.PLUR],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except KeyError:
|
||||||
|
continue # cannot be inflected to all required forms: skip
|
||||||
|
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_adverbes(self, tsv_path: Path) -> list[Adverbe]:
|
||||||
|
"""Parse the adverbs"""
|
||||||
|
root = self._tsv_elems(tsv_path)
|
||||||
|
out: list[Adverbe] = []
|
||||||
|
|
||||||
|
for entry in root.iterfind("./tsv:entry", TSV_NS):
|
||||||
|
# We're only interested in the lemma form
|
||||||
|
orth = entry.find("./tsv:form[@type='lemma']/tsv:orth", TSV_NS)
|
||||||
|
assert orth is not None
|
||||||
|
adv = orth.text
|
||||||
|
out.append(Adverbe(adv=adv))
|
||||||
|
|
||||||
|
return out
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _genre_set(genre: str) -> list[Genre]:
|
||||||
|
return {
|
||||||
|
"masculine": [Genre.MASC],
|
||||||
|
"feminine": [Genre.FEM],
|
||||||
|
"invariable": [Genre.MASC, Genre.FEM],
|
||||||
|
}[genre]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _genre(genre: str) -> Genre:
|
||||||
|
return {
|
||||||
|
"masculine": Genre.MASC,
|
||||||
|
"feminine": Genre.FEM,
|
||||||
|
"invariable": Genre.INV,
|
||||||
|
}[genre]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _nombre(nombre: str) -> Nombre:
|
||||||
|
return {
|
||||||
|
"singular": Nombre.SING,
|
||||||
|
"plural": Nombre.PLUR,
|
||||||
|
}[nombre]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _nombre_set(nombre: str) -> list[Nombre]:
|
||||||
|
return {
|
||||||
|
"singular": [Nombre.SING],
|
||||||
|
"plural": [Nombre.PLUR],
|
||||||
|
"invariable": [Nombre.SING, Nombre.PLUR],
|
||||||
|
}[nombre]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _tense(tense: str) -> t.Optional[Temps]:
|
||||||
|
return {
|
||||||
|
"present": Temps.PRESENT,
|
||||||
|
"imperfect": Temps.IMPARFAIT,
|
||||||
|
"future": Temps.FUTUR,
|
||||||
|
}.get(tense, None)
|
|
@ -8,6 +8,7 @@ import json
|
||||||
class Genre(Enum):
|
class Genre(Enum):
|
||||||
MASC = "masculin"
|
MASC = "masculin"
|
||||||
FEM = "féminin"
|
FEM = "féminin"
|
||||||
|
INV = "invariable" # pour les noms uniquement
|
||||||
|
|
||||||
|
|
||||||
class Nombre(Enum):
|
class Nombre(Enum):
|
||||||
|
@ -35,6 +36,15 @@ class Nom(t.NamedTuple):
|
||||||
"""Accorde en nombre"""
|
"""Accorde en nombre"""
|
||||||
return getattr(self, nombre.name.lower())
|
return getattr(self, nombre.name.lower())
|
||||||
|
|
||||||
|
@property
|
||||||
|
def serialized(self):
|
||||||
|
return {"genre": self.genre.name, "sing": self.sing, "plur": self.plur}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def unserialized(cls, **kwargs):
|
||||||
|
genre = Genre(kwargs.pop("genre"))
|
||||||
|
return cls(**kwargs, genre=genre)
|
||||||
|
|
||||||
|
|
||||||
class Adjectif(t.NamedTuple):
|
class Adjectif(t.NamedTuple):
|
||||||
masc_sing: str
|
masc_sing: str
|
||||||
|
@ -49,6 +59,14 @@ class Adjectif(t.NamedTuple):
|
||||||
"""Accorde en genre et en nombre"""
|
"""Accorde en genre et en nombre"""
|
||||||
return getattr(self, f"{genre.name.lower()}_{nombre.name.lower()}")
|
return getattr(self, f"{genre.name.lower()}_{nombre.name.lower()}")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def serialized(self):
|
||||||
|
return self._asdict()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def unserialized(cls, **kwargs):
|
||||||
|
return cls(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
class Verbe(t.NamedTuple):
|
class Verbe(t.NamedTuple):
|
||||||
present_sing: str
|
present_sing: str
|
||||||
|
@ -65,6 +83,14 @@ class Verbe(t.NamedTuple):
|
||||||
"""Accorde en temps et en nombre (seule la 3è pers. est utilisée)"""
|
"""Accorde en temps et en nombre (seule la 3è pers. est utilisée)"""
|
||||||
return getattr(self, f"{temps.name.lower()}_{nombre.name.lower()}")
|
return getattr(self, f"{temps.name.lower()}_{nombre.name.lower()}")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def serialized(self):
|
||||||
|
return self._asdict()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def unserialized(cls, **kwargs):
|
||||||
|
return cls(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
class Adverbe(t.NamedTuple):
|
class Adverbe(t.NamedTuple):
|
||||||
"""Packed as named tuple for consistence"""
|
"""Packed as named tuple for consistence"""
|
||||||
|
@ -78,6 +104,14 @@ class Adverbe(t.NamedTuple):
|
||||||
"""for consistence"""
|
"""for consistence"""
|
||||||
return self.adv
|
return self.adv
|
||||||
|
|
||||||
|
@property
|
||||||
|
def serialized(self):
|
||||||
|
return self._asdict()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def unserialized(cls, **kwargs):
|
||||||
|
return cls(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
class WordDb:
|
class WordDb:
|
||||||
"""Base de donnée de mots, sérialisable"""
|
"""Base de donnée de mots, sérialisable"""
|
||||||
|
@ -89,6 +123,13 @@ class WordDb:
|
||||||
"adverbes": Adverbe,
|
"adverbes": Adverbe,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
CATEGORY_TO_ATTR: dict = {
|
||||||
|
Nom: "noms",
|
||||||
|
Adjectif: "adjectifs",
|
||||||
|
Verbe: "verbes",
|
||||||
|
Adverbe: "adverbes",
|
||||||
|
}
|
||||||
|
|
||||||
noms: list[Nom]
|
noms: list[Nom]
|
||||||
adjectifs: list[Adjectif]
|
adjectifs: list[Adjectif]
|
||||||
verbes: list[Verbe]
|
verbes: list[Verbe]
|
||||||
|
@ -109,7 +150,7 @@ class WordDb:
|
||||||
def serialize(self) -> dict:
|
def serialize(self) -> dict:
|
||||||
"""Serialize to plain dictionary (no classes)"""
|
"""Serialize to plain dictionary (no classes)"""
|
||||||
return {
|
return {
|
||||||
attr: [x._asdict() for x in getattr(self, attr)]
|
attr: [x.serialized for x in getattr(self, attr)]
|
||||||
for attr in self.__class__._serialize_data
|
for attr in self.__class__._serialize_data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -123,7 +164,7 @@ class WordDb:
|
||||||
"""Reverses :serialize:"""
|
"""Reverses :serialize:"""
|
||||||
parsed = {}
|
parsed = {}
|
||||||
for attr, attr_cls in cls._serialize_data.items():
|
for attr, attr_cls in cls._serialize_data.items():
|
||||||
parsed[attr] = list(map(attr_cls, data[attr]))
|
parsed[attr] = list(map(attr_cls.unserialized, data[attr]))
|
||||||
return cls(**parsed)
|
return cls(**parsed)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|
Loading…
Reference in a new issue