Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 31 additions & 3 deletions pipeline/src/additional_methods/by_name.py.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
match: str = "equals",
all: bool = False,
case_sensitive: bool = True,
ignore_accents: bool = False,
):
"""
Search for instances in the openMINDS instance library based on their name.
Expand All @@ -24,6 +25,10 @@
(the given string contains the name-like property).
all (bool, optional): Whether to return all objects that match the name, or only the first. Defaults to False.
case_sensitive (bool, optional): Whether the search should be case-sensitive. Defaults to True.
ignore_accents (bool, optional): Whether to ignore accents (acute, grave, circumflex) and
other diacritical marks (cedilla, tilde, ring, etc.) when matching. Also treat
special letters (ß, œ, æ, ø, ł, etc.) as their closest plain-letter equivalents
(e.g. "ß" as "ss"). Defaults to False.
"""
namelike_properties = ("name", "lookup_label", "family_name", "full_name", "short_name", "abbreviation")
if cls._instance_lookup is None:
Expand All @@ -43,16 +48,39 @@
else:
cls._instance_lookup[key] = [instance]

def remove_accents(s):
import unicodedata

special = str.maketrans({
"Ł": "L", "ł": "l",
"Ø": "O", "ø": "o",
"Đ": "D", "đ": "d",
"Ð": "D", "ð": "d",
"Þ": "Th", "þ": "th",
"Æ": "AE", "æ": "ae",
"Œ": "OE", "œ": "oe",
"ß": "ss", "ẞ": "SS",
"Ə": "E", "ə": "e",
"ı": "i",
})
nfd_form = unicodedata.normalize("NFD", s)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This doesn't handle Ł ø đ æ ß ə ı

For example:

>>> import unicodedata
>>> s = "Łódź"
>>> unicodedata.normalize("NFD", s)
'Łódź'

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right, I missed it.

  • "NFKD" (instead of "NFD") doesn't help either
  • unidecode handles those letters
from unidecode import unidecode

for s in ["Ł", "ø", "đ", "æ", "ß", "ə", "ı", "µA", "m³", "Ελλάδα"]:
    print(f"{s} -> {unidecode(s)}")

### Output
Ł -> L
ø -> o
đ -> d
æ -> ae
ß -> ss
ə -> @
ı -> i
µA -> uA
m³ -> m3
Ελλάδα -> Ellada

but it adds a runtime dependency and transliterates too aggressively (e.g. it maps ə to @, which is wrong for eg Azerbaijan's synonym "Azərbaycan").

  • My preferred option: keep NFD for accents and add a small explicit map (no dependency, full control):
_SPECIAL = str.maketrans({
    "Ł": "L", "ł": "l",
    "Ø": "O", "ø": "o",
    "Đ": "D", "đ": "d",
    "Æ": "AE", "æ": "ae",
    "ß": "ss",
    "Ə": "E", "ə": "e",
    "ı": "i",
})

def remove_accents(s):
    s = s.translate(_SPECIAL)                 
    nfd_form = unicodedata.normalize("NFD", s)
    return "".join(c for c in nfd_fo

Happy to implement it this way if you agree

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, I'm happy with that approach

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK. It's now implemented in 21e0397 with an updated test.

stripped = "".join(c for c in nfd_form if not unicodedata.combining(c))
return stripped.translate(special)

def normalize(s):
return s if case_sensitive else s.casefold()
if not case_sensitive:
s = s.casefold()
if ignore_accents:
s = remove_accents(s)
return s

if match == "equals":
if case_sensitive:
if case_sensitive and not ignore_accents:
matches = cls._instance_lookup.get(name, [])
else:
matches = []
for key, instances in cls._instance_lookup.items():
if key.casefold() == name.casefold():
if normalize(key) == normalize(name):
matches.extend(instances)
elif match == "contains":
matches = []
Expand Down
35 changes: 35 additions & 0 deletions pipeline/tests/test_regressions.py
Original file line number Diff line number Diff line change
Expand Up @@ -728,3 +728,38 @@ def test_pr0100_by_name_match_within(om):
# but none of those full names is itself a substring of "Macaca".
assert Species.by_name("Macaca", match="contains", all=True) is not None
assert Species.by_name("Macaca", match="within", all=True) is None


@pytest.mark.parametrize("om", [openminds.latest])
def test_pr0103_by_name_ignore_accents(om):
# https://github.com/openMetadataInitiative/openMINDS_Python/pull/103
# by_name(..., ignore_accents=True) strips accents/diacritics (Unicode NFD) before matching
SovereignState = om.controlled_terms.SovereignState

# (query, case_sensitive, ignore_accents, should match France)
cases = [
("République française", True, False, True), # exact
("Republique francaise", True, True, True), # accents differ
("république française", False, False, True), # case differs
("republique francaise", False, True, True), # case and accents differ
("republique francaise", True, False, False), # defaults: neither absorbed
("Republique francaise", True, False, False), # accents still matter
("république française", True, True, False), # case still matters
]
for query, case_sensitive, ignore_accents, should_match in cases:
match = SovereignState.by_name(query, case_sensitive=case_sensitive, ignore_accents=ignore_accents)
assert (match is not None and match.name == "France") == should_match

# ignore_accents also has to map special letters
special_letter_cases = [
# (query, ignore_accents, expected_country_or_None)
("Azərbaycan Respublikası", False, "Azerbaijan"), # exact
("Azerbaycan Respublikasi", True, "Azerbaijan"),
("Azerbaycan Respublikasi", False, None),
("Wááshindoon Bikéyah Ałhidadiidzooígíí", False, "United States"), # exact
("Waashindoon Bikeyah Alhidadiidzooigii", True, "United States"),
("Waashindoon Bikeyah Alhidadiidzooigii", False, None),
]
for query, ignore_accents, expected_name in special_letter_cases:
match = SovereignState.by_name(query, ignore_accents=ignore_accents)
assert (match.name if match else None) == expected_name
Loading