Skip to content

Commit

Permalink
Black reformat
Browse files Browse the repository at this point in the history
  • Loading branch information
fbanados committed Nov 18, 2024
1 parent 682e08b commit e40820d
Show file tree
Hide file tree
Showing 14 changed files with 122 additions and 46 deletions.
4 changes: 3 additions & 1 deletion src/morphodict/frontend/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@
from morphodict.search import presentation, search_with_affixes
from morphodict.frontend.forms import WordSearchForm
from morphodict.paradigm.views import paradigm_context_for_lemma
from morphodict.phrase_translate.fst import fst_analyses as phrase_translate_fst_analyses
from morphodict.phrase_translate.fst import (
fst_analyses as phrase_translate_fst_analyses,
)
from crkeng.app.preferences import DisplayMode, AnimateEmoji, ShowEmoji
from morphodict.lexicon.models import Wordform

Expand Down
5 changes: 4 additions & 1 deletion src/morphodict/lexicon/management/commands/importjsondict.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@
from tqdm import tqdm

from morphodict.paradigm.generation import default_paradigm_manager
from morphodict.phrase_translate.to_source import TranslationStats, translate_single_definition
from morphodict.phrase_translate.to_source import (
TranslationStats,
translate_single_definition,
)
from morphodict.utils.english_keyword_extraction import stem_keywords
from morphodict.analysis import RichAnalysis, strict_generator
from morphodict.lexicon import DEFAULT_IMPORTJSON_FILE
Expand Down
1 change: 1 addition & 0 deletions src/morphodict/paradigm/recordings.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import requests
import urllib


def get_recordings_from_url(search_terms, url, speech_db_eq):
matched_recordings = {}
query_params = [("q", term) for term in search_terms]
Expand Down
29 changes: 15 additions & 14 deletions src/morphodict/phrase_translate/fst.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,30 +57,31 @@ def foma_lookup(fst, thing_to_lookup):
raise FomaLookupMultipleFoundException(thing_to_lookup, l)
return l[0].decode("UTF-8")


def inflect_target_noun_phrase(tagged_phrase):
return foma_lookup(eng_noun_entry_to_inflected_phrase_fst(), tagged_phrase)


def inflect_target_verb_phrase(tagged_phrase):
return foma_lookup(eng_verb_entry_to_inflected_phrase_fst(), tagged_phrase)


def source_phrase_analyses(query):
return [
r.decode("UTF-8") for r in eng_phrase_to_crk_features_fst()[query]
]
return [r.decode("UTF-8") for r in eng_phrase_to_crk_features_fst()[query]]


def fst_analyses(text):
def decode_foma_results(fst, query):
return [r.decode("UTF-8") for r in fst[query]]

return {
"eng_noun_entry2inflected-phrase": decode_foma_results(
eng_noun_entry_to_inflected_phrase_fst(), text
),
"eng_verb_entry2inflected-phrase": decode_foma_results(
eng_verb_entry_to_inflected_phrase_fst(), text
),
"eng_phrase_to_crk_features": decode_foma_results(
eng_phrase_to_crk_features_fst(), text
),
}
"eng_noun_entry2inflected-phrase": decode_foma_results(
eng_noun_entry_to_inflected_phrase_fst(), text
),
"eng_verb_entry2inflected-phrase": decode_foma_results(
eng_verb_entry_to_inflected_phrase_fst(), text
),
"eng_phrase_to_crk_features": decode_foma_results(
eng_phrase_to_crk_features_fst(), text
),
}
1 change: 1 addition & 0 deletions src/morphodict/phrase_translate/source_tag_map.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from morphodict.analysis.tag_map import TagMap

# TODO Migrate to json files

COPY_TAG_NAME = TagMap.COPY_TAG_NAME
Expand Down
14 changes: 10 additions & 4 deletions src/morphodict/phrase_translate/tag_maps.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ def read_phrase_translate_json(filename, default=[]):
# Note that, in general, this might interact with the JSON file contents, but we are not using # for
# anything in our json files yet, so we can use them.
try:
with open(settings.BASE_DIR / "resources" / "phrase_translate" / filename, "r") as f:
with open(
settings.BASE_DIR / "resources" / "phrase_translate" / filename, "r"
) as f:
contents = f.read()
ans = json.loads(re.sub("(^|\s)+#.*\n", "", contents))
except FileNotFoundError:
Expand All @@ -31,9 +33,9 @@ def read_phrase_translate_json(filename, default=[]):
# +V+TA and before +1Sg, so it needs precedence values for all of those tags.
#
# So we list everything.
noun_passthrough_tags = read_phrase_translate_json("noun_passthrough_tags.json",{})
noun_passthrough_tags = read_phrase_translate_json("noun_passthrough_tags.json", {})

verb_passthrough_tags = read_phrase_translate_json("verb_passthrough_tags.json",{})
verb_passthrough_tags = read_phrase_translate_json("verb_passthrough_tags.json", {})


def passthrough_tags_to_tuples(passthrough_tags):
Expand All @@ -43,18 +45,22 @@ def passthrough_tags_to_tuples(passthrough_tags):
for tag in tag_list
)


def lists_to_tuple(l):
def element_process(x):
if isinstance(x,list):
if isinstance(x, list):
return tuple(x)
if x == "TagMap.DEFAULT":
return TagMap.DEFAULT
return x

return tuple(element_process(x) for x in l)


def tagmap_json_to_tuples(json):
return [lists_to_tuple(t) for t in json]


verb_tag_map = TagMap(
*tagmap_json_to_tuples(read_phrase_translate_json("verb_tag_map.json")),
*passthrough_tags_to_tuples(verb_passthrough_tags)
Expand Down
4 changes: 3 additions & 1 deletion src/morphodict/phrase_translate/test_definition_cleanup.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import pytest

from morphodict.phrase_translate.definition_cleanup import cleanup_target_definition_for_translation
from morphodict.phrase_translate.definition_cleanup import (
cleanup_target_definition_for_translation,
)

SHOULD_NOT_CHANGE = object()

Expand Down
72 changes: 60 additions & 12 deletions src/morphodict/phrase_translate/test_inflect_target_phrase.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,29 @@
("analysis", "definition", "english_phrase"),
(
(
((), "target_lemma_for_book_(irrelevant_for_test)", ("+N", "+Dim", "+Px1Sg", "+Loc")),
(
(),
"target_lemma_for_book_(irrelevant_for_test)",
("+N", "+Dim", "+Px1Sg", "+Loc"),
),
"book",
"in my little book",
),
(
((), "target_lemma_for_book_(irrelevant_for_test)", ("+N", "+I", "+Px12Pl", "+Pl")),
(
(),
"target_lemma_for_book_(irrelevant_for_test)",
("+N", "+I", "+Px12Pl", "+Pl"),
),
"book",
"your and our books",
),
(
((), "target_lemma_for_star_(irrelevant_for_test)", ("+N", "+A", "+Der/Dim", "+N", "+A", "+Obv")),
(
(),
"target_lemma_for_star_(irrelevant_for_test)",
("+N", "+A", "+Der/Dim", "+N", "+A", "+Obv"),
),
"star",
"an/other little star(s)",
),
Expand All @@ -33,47 +45,83 @@
"in my little Cree book",
),
(
(("PV/e+",), "target_lemma_for_long_definition_(irrelevant_for_test)", ("+V", "+TA", "+Cnj", "+1Sg", "+2SgO")),
(
("PV/e+",),
"target_lemma_for_long_definition_(irrelevant_for_test)",
("+V", "+TA", "+Cnj", "+1Sg", "+2SgO"),
),
"s/he sits with s.o., s/he stays with s.o., s/he is present with s.o.",
"I sit with you, I stay with you, I am present with you",
),
(
((), "target_arbitrary_lemma_(irrelevant_for_test)", ("+V", "+TA", "+Fut", "+Cond", "+2Sg", "+3SgO")),
(
(),
"target_arbitrary_lemma_(irrelevant_for_test)",
("+V", "+TA", "+Fut", "+Cond", "+2Sg", "+3SgO"),
),
ARBITRARY_DEFINITION,
"when you see him/her",
),
(
((), "target_arbitrary_lemma_(irrelevant_for_test)", ("+V", "+TA", "+Imp", "+Imm", "+12Pl", "+3SgO")),
(
(),
"target_arbitrary_lemma_(irrelevant_for_test)",
("+V", "+TA", "+Imp", "+Imm", "+12Pl", "+3SgO"),
),
ARBITRARY_DEFINITION,
"let you and us see him/her now",
),
(
((), "target_arbitrary_lemma_(irrelevant_for_test)", ("+V", "+TA", "+Imp", "+Del", "+2Sg", "+3PlO")),
(
(),
"target_arbitrary_lemma_(irrelevant_for_test)",
("+V", "+TA", "+Imp", "+Del", "+2Sg", "+3PlO"),
),
ARBITRARY_DEFINITION,
"(you) see them later",
),
(
(("PV/ki+",), "target_arbitrary_lemma_(irrelevant_for_test)", ("+V", "+TA", "+Ind", "+2Pl", "+1PlO")),
(
("PV/ki+",),
"target_arbitrary_lemma_(irrelevant_for_test)",
("+V", "+TA", "+Ind", "+2Pl", "+1PlO"),
),
ARBITRARY_DEFINITION,
"you all saw us",
),
(
(("PV/wi+",), "target_arbitrary_lemma_(irrelevant_for_test)", ("+V", "+TA", "+Ind", "+4Sg/Pl", "+3PlO")),
(
("PV/wi+",),
"target_arbitrary_lemma_(irrelevant_for_test)",
("+V", "+TA", "+Ind", "+4Sg/Pl", "+3PlO"),
),
ARBITRARY_DEFINITION,
"another/others are going to see them",
),
(
(("PV/e+", "PV/wi+"), "target_arbitrary_lemma_(irrelevant_for_test)", ("+V", "+TA", "+Cnj", "+1Sg", "+3PlO")),
(
("PV/e+", "PV/wi+"),
"target_arbitrary_lemma_(irrelevant_for_test)",
("+V", "+TA", "+Cnj", "+1Sg", "+3PlO"),
),
ARBITRARY_DEFINITION,
"I am going to see them",
),
(
(("PV/ka+",), "target_arbitrary_lemma_(irrelevant_for_test)", ("+V", "+TA", "+Ind", "+4Sg/Pl", "+3SgO")),
(
("PV/ka+",),
"target_arbitrary_lemma_(irrelevant_for_test)",
("+V", "+TA", "+Ind", "+4Sg/Pl", "+3SgO"),
),
ARBITRARY_DEFINITION,
"another/others will see him/her",
),
(
(("PV/ka+",), "target_arbitrary_lemma_(irrelevant_for_test)", ("+V", "+TA", "+Cnj", "+1Sg", "+2PlO")),
(
("PV/ka+",),
"target_arbitrary_lemma_(irrelevant_for_test)",
("+V", "+TA", "+Cnj", "+1Sg", "+2PlO"),
),
ARBITRARY_DEFINITION,
"for me to see you all",
),
Expand Down
2 changes: 1 addition & 1 deletion src/morphodict/phrase_translate/to_source/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
from .translate import translate_single_definition, TranslationStats
from .translate import translate_single_definition, TranslationStats
25 changes: 19 additions & 6 deletions src/morphodict/phrase_translate/to_source/translate.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,14 @@

import django

from morphodict.phrase_translate.definition_cleanup import cleanup_target_definition_for_translation
from morphodict.phrase_translate.definition_cleanup import (
cleanup_target_definition_for_translation,
)
from morphodict.phrase_translate.to_target import inflect_target_language_phrase
from morphodict.phrase_translate.fst import FomaLookupNotFoundException, FomaLookupMultipleFoundException
from morphodict.phrase_translate.fst import (
FomaLookupNotFoundException,
FomaLookupMultipleFoundException,
)
from morphodict.analysis.tag_map import UnknownTagError

if typing.TYPE_CHECKING:
Expand Down Expand Up @@ -55,7 +60,9 @@ def translate_and_print_wordforms(wordforms: Iterable[Wordform]):
continue

print(f" definition: {d} →")
phrase = inflect_target_language_phrase(wordform.analysis, d.core_definition)
phrase = inflect_target_language_phrase(
wordform.analysis, d.core_definition
)
if phrase is None:
phrase = "(not supported)"
print(f" {phrase}")
Expand Down Expand Up @@ -96,7 +103,9 @@ def __str__(self):
return "\n".join(ret)


def translate_single_definition(inflected_wordform, lemma_definition, stats: TranslationStats):
def translate_single_definition(
inflected_wordform, lemma_definition, stats: TranslationStats
):
stats.wordforms_examined += 1

assert inflected_wordform.analysis
Expand Down Expand Up @@ -124,7 +133,9 @@ def translate_single_definition(inflected_wordform, lemma_definition, stats: Tra

phrase = inflect_target_language_phrase(inflected_wordform.analysis, input_text)
except UnknownTagError as e:
logger.debug(f"Unknown tag in {inflected_wordform.text} {inflected_wordform.analysis}: {e}")
logger.debug(
f"Unknown tag in {inflected_wordform.text} {inflected_wordform.analysis}: {e}"
)
stats.unknown_tags_during_auto_translation[e.args[0]] += 1
return None
except FomaLookupNotFoundException as e:
Expand All @@ -137,7 +148,9 @@ def translate_single_definition(inflected_wordform, lemma_definition, stats: Tra
return None

if not phrase:
logger.debug(f"no translation for {inflected_wordform.text} {inflected_wordform.analysis}")
logger.debug(
f"no translation for {inflected_wordform.text} {inflected_wordform.analysis}"
)
stats.no_translation_count += 1
return None

Expand Down
4 changes: 2 additions & 2 deletions src/morphodict/phrase_translate/to_target/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
)
from morphodict.phrase_translate.fst import (
inflect_target_noun_phrase,
inflect_target_verb_phrase
inflect_target_verb_phrase,
)


Expand All @@ -33,4 +33,4 @@ def inflect_target_language_phrase(analysis, lemma_definition):
logger.debug("tagged_phrase = %s\n", tagged_phrase)
phrase = inflect_target_verb_phrase(tagged_phrase)
logger.debug("phrase = %s\n", phrase)
return phrase.strip()
return phrase.strip()
1 change: 1 addition & 0 deletions src/morphodict/phrase_translate/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
re.VERBOSE,
)


class PhraseAnalyzedQuery:
"""A structured object holding pieces of, and info about, a phrase query.
Expand Down
3 changes: 0 additions & 3 deletions src/morphodict/search/espt.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,3 @@ def _generate_inflected_results(self) -> list[_EipResult]:
)
)
return results



3 changes: 2 additions & 1 deletion src/morphodict/utils/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from django.urls import reverse


def url_for_query(user_query: str) -> str:
"""
Produces a relative URL to search for the given user query.
Expand All @@ -14,4 +15,4 @@ def url_for_query(user_query: str) -> str:
query=urlencode((("q", user_query),)),
fragment="",
)
return urlunparse(parts)
return urlunparse(parts)

0 comments on commit e40820d

Please sign in to comment.