import marisa_trie
import json
import xml.etree.ElementTree as ET
import os
from pathlib import Path
from collections import defaultdict, Counter
from language_data.names import normalize_name
from language_data.util import data_filename
from language_data.registry_parser import parse_registry
# Naming things is hard, especially languages
# ===========================================
#
# CLDR is supposed to avoid ambiguous language names, particularly among its
# core languages. But it seems that languages are incompletely disambiguated.
#
# It's convenient to be able to get a language by its name, without having to
# also refer to the language that the name is in. In most cases, we can do this
# unambiguously. With the disambiguations and overrides here, this will work
# in a lot of cases. However, some names such as 'Dongo', 'Fala', 'Malayo', and
# 'Tonga' are ambiguous in ways that can only be disambiguated by specifying
# the language the name is in.
#
# Ambiguous names can arise from:
#
# - Ambiguities in the scope of a name. These tend to span languages, and the
# data files mask the fact that these names are generally ambiguous *within*
# a language. This is why we have codes.
#
# - Names that just happen to be ambiguous between different things with
# different etymologies.
#
# Most doubly-claimed language names have standard ways to disambiguate
# them in CLDR, but names such as 'Tonga' and 'Fala' have complex
# inter-language ambiguities.
#
# Our approach is:
#
# - Fix conflicts that seem to arise simply from errors in the data, by
# overriding the data.
#
# - Fix ambiguities in scope by preferring one scope over another. For example,
# "North America" could refer to a territory that includes Central America or
# a territory that doesn't. In any such conflict, we choose to include Central
# America.
#
# - Avoid ambiguities between different sources of data, by using an order
# of precedence. CLDR data takes priority over IANA data, which takes priority
# over Wiktionary data.
#
# - When ambiguity remains, that name is not resolvable to a language code.
# Resolving the name might require a more specific name, or specifying the
# language that the name is in.
AMBIGUOUS_PREFERENCES = {
# Prefer 'Micronesia' to refer to the Federated States of Micronesia -
# this seems to be poorly disambiguated in many languages, but we can't
# do much with a code for the general region of Micronesia
'FM': {'057'},
# Prefer the country of South Africa over the general region of southern
# Africa, in languages that don't distinguish them
'ZA': {'018'},
# Prefer territory 003 for 'North America', which includes Central America
# and the Caribbean, over territory 021, which excludes them
'003': {'021'},
# Prefer territory 005 for 'Lulli-Amerihkká' (South America), over territory
# 419, which includes Central America
'005': {'419'},
# If a name like "Amerika" is ambiguous between the Americas and the United
# States of America, choose the Americas
'019': {'US'},
# Prefer 'Swiss German' to be a specific language
'gsw': {'de-CH'},
# Of the two countries named 'Congo', prefer the one with Kinshasa
'CD': {'CG'},
# Prefer Han script to not include bopomofo
'Hani': {'Hanb'},
# Prefer the specific language Tagalog over standard Filipino, because
# the ambiguous name was probably some form of 'Tagalog'
'tl': {'fil'},
# Confusion between Ilokano and Hiligaynon
'ilo': {'hil'},
# Prefer Central Atlas Tamazight over Standard Moroccan Tamazight
'tzm': {'zgh'},
# Prefer the specific definition of Low Saxon
'nds-NL': {'nds'},
# Prefer the specific definition of Mandarin Chinese
'cmn': {'zh'},
# Prefer the territorially-specific definition of Dari
'fa-AF': {'prs', 'fa', 'gbz'},
# Ambiguity in the scope of Korean script (whether to include Han characters)
'Kore': {'Hang'},
# This ambiguity is kind of our fault, for adding an autonym for 'zsm'.
# "Bahasa Malaysia" should still resolve to the more expected 'ms'.
'ms': {'zsm'},
# I think that the CLDR data for Mazanderani confuses Latvia and Lithuania,
# and Wikipedia tells me it means Latvia. I should make this a CLDR issue
'lv': {'lt'},
'LV': {'LT'},
}
OVERRIDES = {
# When I ask Wiktionary, it tells me that "Breatnais" is Scots Gaelic for
# Welsh, not Breton, which is "Breatannais". This may be one of those
# things that's not as standardized as it sounds, but let's at least agree
# with Wiktionary and avoid a name conflict.
("gd", "br"): "Breatannais",
# 'tagaloga' should be 'tl', not 'fil'
("eu", "tl"): "Tagaloga",
("eu", "fil"): "Filipinera",
# 'Dakota' should be 'dak', not 'dar', which is "Dargwa"
("af", "dar"): "Dargwa",
("af-NA", "dar"): "Dargwa",
# 'интерлингве' should be 'ie', not 'ia', which is 'интерлингва'
("az-Cyrl", "ia"): "интерлингва",
# Don't confuse Samaritan Hebrew with Samaritan Aramaic
("en", "smp"): "Samaritan Hebrew",
# Don't confuse the Mongol language of New Guinea with Mongolian
("en", "mgt"): "Mongol (New Guinea)",
# Don't confuse Romang with Romani over the name 'Roma'
("en", "rmm"): "Romang",
# 'Tai' is a large language family, and it should not refer exclusively and
# unrelatedly to a language spoken by 900 people in New Guinea
("en", "taw"): "Kalam-Tai",
# The code for Ladin -- the language that's almost certainly being named in
# Friulian here -- is "lld". The given code of "lad" seems to be an error,
# pointing to the Judeo-Spanish language Ladino, which would be less likely
# to be what you mean when speaking Friulian.
("fur", "lad"): None,
# The Amharic data in v39 appears to have switched the words for 'Western'
# and 'Eastern'.
("am", "011"): "ምዕራባዊ አፍሪካ", # Western Africa
("am", "014"): "ምስራቃዊ አፍሪካ", # Eastern Africa
("am", "155"): "ምዕራባዊ አውሮፓ", # Western Europe
("am", "151"): "ምስራቃዊ አውሮፓ", # Eastern Europe
}
def resolve_name(key, vals, debug=False):
"""
Given a name, and a number of possible values it could resolve to,
find the single value it should resolve to, in the following way:
- Apply the priority order
- If names with the highest priority all agree, use that name
- If there is disagreement that can be resolved by AMBIGUOUS_PREFERENCES,
use that
- Otherwise, don't resolve the name (and possibly show a debugging message
when building the data)
"""
max_priority = max([val[2] for val in vals])
val_count = Counter([val[1] for val in vals if val[2] == max_priority])
if len(val_count) == 1:
unanimous = val_count.most_common(1)
return unanimous[0][0]
for pkey in val_count:
if pkey in AMBIGUOUS_PREFERENCES:
others = set(val_count)
others.remove(pkey)
if others == others & AMBIGUOUS_PREFERENCES[pkey]:
if debug:
print("Resolved: {} -> {}".format(key, pkey))
return pkey
# In debug mode, show which languages vote for which name
if debug and max_priority >= 0:
votes = defaultdict(list)
for voter, val, prio in vals:
if prio == max_priority:
votes[val].append(voter)
print("{}:".format(key))
for val, voters in sorted(votes.items()):
print("\t{}: {}".format(val, ' '.join(voters)))
# Don't use names that remain ambiguous
return None
def resolve_names(name_dict, debug=False):
resolved = {}
for key, vals in sorted(name_dict.items()):
resolved_name = resolve_name(key, vals, debug=debug)
if resolved_name is not None:
resolved[key] = resolved_name
return resolved
def read_cldr_names(language, category):
"""
Read CLDR's names for things in a particular language.
"""
filename = data_filename(
'cldr-json/cldr-json/cldr-localenames-full/main/{}/{}.json'.format(language, category)
)
fulldata = json.load(open(filename, encoding='utf-8'))
data = fulldata['main'][language]['localeDisplayNames'][category]
return data
def read_cldr_name_file(langcode, category):
data = read_cldr_names(langcode, category)
name_quads = []
for subtag, name in sorted(data.items()):
if (langcode, subtag) in OVERRIDES:
name = OVERRIDES[langcode, subtag]
if name is None:
continue
if subtag == name:
# Default entries that map a language code to itself, which
# an inattentive annotator just left there
continue
priority = 3
if subtag.endswith('-alt-menu') and name == 'mandarin':
# The -alt-menu entries are supposed to do things like alphabetize
# "Mandarin Chinese" under "Chinese, Mandarin". A few languages
# just put the string "mandarin" there, which seems wrong and
# messes up our name lookups.
continue
# CLDR assigns multiple names to one code by adding -alt-* to
# the end of the code. For example, the English name of 'az' is
# Azerbaijani, but the English name of 'az-alt-short' is Azeri.
if '-alt-' in subtag:
subtag, _ = subtag.split('-alt-', 1)
priority = 1
if normalize_name(name) == normalize_name(subtag):
# Giving the name "zh (Hans)" to "zh-Hans" is still lazy
continue
name_quads.append((langcode, subtag, name, priority))
return name_quads
def read_iana_registry_names():
language_quads = []
script_quads = []
territory_quads = []
for entry in parse_registry():
target = None
if entry['Type'] == 'language':
target = language_quads
elif entry['Type'] == 'script':
target = script_quads
elif entry['Type'] == 'region':
# IANA's terminology is 'region' where CLDR's is 'territory'
target = territory_quads
if target is not None:
subtag = entry['Subtag']
priority = 2
if 'Deprecated' in entry:
priority = 0
if ('en', subtag) in OVERRIDES:
target.append(('en', subtag, OVERRIDES['en', subtag], priority))
else:
for desc in entry['Description']:
target.append(('en', subtag, desc, priority))
return language_quads, script_quads, territory_quads
def read_iana_registry_macrolanguages():
macros = {}
for entry in parse_registry():
if entry['Type'] == 'language' and 'Macrolanguage' in entry:
macros[entry['Subtag']] = entry['Macrolanguage']
return macros
def read_iana_registry_replacements():
replacements = {}
for entry in parse_registry():
if entry['Type'] == 'language' and 'Preferred-Value' in entry:
# Replacements for language codes
replacements[entry['Subtag']] = entry['Preferred-Value']
elif 'Tag' in entry and 'Preferred-Value' in entry:
# Replacements for entire tags
replacements[entry['Tag'].lower()] = entry['Preferred-Value']
return replacements
def read_csv_names(filename):
data = open(filename, encoding='utf-8')
quads = []
for line in data:
quad = line.rstrip().split(',', 3) + [True]
quads.append(tuple(quad))
return quads
def read_wiktionary_names(filename, language):
data = open(filename, encoding='utf-8')
quads = []
for line in data:
parts = line.rstrip().split('\t')
code = parts[0]
quads.append((language, code, parts[1], -1))
names = [parts[1]]
if len(parts) > 4 and parts[4]:
names = parts[4].split(', ')
for name in names:
quads.append((language, code, name, -2))
return quads
def update_names(names_fwd, names_rev, name_quads):
for name_language, referent, name, priority in name_quads:
# Get just the language from name_language, not the territory or script.
short_language = name_language.split('-')[0]
rev_all = names_rev.setdefault('und', {})
rev_language = names_rev.setdefault(short_language, {})
for rev_dict in (rev_all, rev_language):
rev_dict.setdefault(normalize_name(name), []).append(
(name_language, referent, priority)
)
names_for_referent = names_fwd.setdefault(referent, {})
if name_language not in names_for_referent:
names_for_referent[name_language] = name
def save_trie(mapping, filename):
trie = marisa_trie.BytesTrie(
(key, value.encode('utf-8')) for (key, value) in sorted(mapping.items())
)
trie.save(filename)
def save_reverse_name_tables(category, rev_dict):
for language, lang_dict in rev_dict.items():
os.makedirs(data_filename(f"trie/{language}"), exist_ok=True)
save_trie(
resolve_names(lang_dict, debug=True),
data_filename(f"trie/{language}/name_to_{category}.marisa"),
)
def get_name_languages():
cldr_main_path = Path(data_filename("cldr-json/cldr-json/cldr-localenames-full/main"))
languages = [
subpath.name
for subpath in sorted(cldr_main_path.iterdir())
if subpath.name != 'root' and (subpath / 'languages.json').exists()
]
return [language for language in languages if 'a' <= language[-1] <= 'z']
def get_population_data():
import langcodes
filename = data_filename("supplementalData.xml")
root = ET.fromstring(open(filename).read())
territories = root.findall("./territoryInfo/territory")
language_population = defaultdict(int)
language_writing_population = defaultdict(int)
for territory in territories:
t_code = territory.attrib['type']
t_population = float(territory.attrib['population'])
t_literacy_rate = float(territory.attrib['literacyPercent']) / 100
for language in territory:
attrs = language.attrib
l_code = attrs['type'].replace('_', '-')
l_proportion = float(attrs.get('populationPercent', 0)) / 100
if 'writingPercent' in attrs:
writing_prop = float(attrs['writingPercent']) / 100
elif 'literacyPercent' in attrs:
writing_prop = float(attrs['literacyPercent']) / 100
else:
writing_prop = t_literacy_rate
l_population = t_population * l_proportion
l_writing = t_population * l_proportion * writing_prop
# Distinguish data in different territories, and also in different
# scripts when necessary, while also accumulating more general data
# We need to use maximize() on the bare language code, not just
# assume_script(), because assumed defaults like 'zh-Hans' are unwritten
# in the data. We need this if we want to count the relative use of
# Simplified vs. Traditional Chinese, for example.
written_ls = (
langcodes.get(l_code).maximize()._filter_attributes(['language', 'script'])
)
written_lst = written_ls.update_dict({'territory': t_code})
spoken_lt = written_lst._filter_attributes(['language', 'territory'])
spoken_l = written_lst._filter_attributes(['language'])
written_lt = written_lst._filter_attributes(['language', 'territory'])
written_l = written_lst._filter_attributes(['language'])
for lang in set([spoken_lt, spoken_l]):
language_population[str(lang)] += int(round(l_population))
for lang in set([written_lst, written_lt, written_ls, written_l]):
language_writing_population[str(lang)] += int(round(l_writing))
return language_population, language_writing_population
def write_python_dict(outfile, name, d):
"""
Write Python code that initializes a given dictionary, with one value on
each line.
"""
print(f"{name} = {{", file=outfile)
for key, value in sorted(d.items()):
print(f" {key!r}: {value!r},", file=outfile)
print("}", file=outfile)
def write_python_set(outfile, name, s):
print(f"{name} = {{", file=outfile)
for key in sorted(set(s)):
print(f" {key!r},", file=outfile)
print("}", file=outfile)
GENERATED_HEADER = "# This file is generated by build_data.py."
def build_data():
language_names_rev = {}
territory_names_rev = {}
script_names_rev = {}
names_fwd = {}
override_language_data = read_csv_names(data_filename('override_language_names.csv'))
update_names(names_fwd, language_names_rev, override_language_data)
for langcode in get_name_languages():
language_data = read_cldr_name_file(langcode, 'languages')
update_names(names_fwd, language_names_rev, language_data)
try:
script_data = read_cldr_name_file(langcode, 'scripts')
update_names(names_fwd, script_names_rev, script_data)
except FileNotFoundError:
pass
try:
territory_data = read_cldr_name_file(langcode, 'territories')
update_names(names_fwd, territory_names_rev, territory_data)
except FileNotFoundError:
pass
iana_languages, iana_scripts, iana_territories = read_iana_registry_names()
update_names(names_fwd, language_names_rev, iana_languages)
update_names(names_fwd, script_names_rev, iana_scripts)
update_names(names_fwd, territory_names_rev, iana_territories)
wiktionary_data = read_wiktionary_names(data_filename('wiktionary/codes-en.csv'), 'en')
update_names(names_fwd, language_names_rev, wiktionary_data)
extra_language_data = read_csv_names(data_filename('extra_language_names.csv'))
update_names(names_fwd, language_names_rev, extra_language_data)
save_reverse_name_tables('language', language_names_rev)
save_reverse_name_tables('script', script_names_rev)
save_reverse_name_tables('territory', territory_names_rev)
# Get the list of languages where we have any name data. These are base
# language codes (without scripts or territories) which contain a name for
# themselves.
name_languages = [
langcode
for langcode in get_name_languages()
if '-' not in langcode and langcode in names_fwd and langcode in names_fwd[langcode]
]
# Add the languages that have autonyms in extra_language_data, perhaps because
# we specifically put them there to get their autonyms right
name_languages += [lang1 for (lang1, lang2, _, _) in extra_language_data if lang1 == lang2]
# Write the contents of name_data.py.
with open('name_data.py', 'w', encoding='utf-8') as outfile:
print(GENERATED_HEADER, file=outfile)
write_python_set(outfile, 'LANGUAGES_WITH_NAME_DATA', name_languages)
print(file=outfile)
write_python_dict(outfile, 'CODE_TO_NAMES', names_fwd)
language_population, language_writing_population = get_population_data()
with open('population_data.py', 'w', encoding='utf-8') as outfile:
print(GENERATED_HEADER, file=outfile)
write_python_dict(outfile, 'LANGUAGE_SPEAKING_POPULATION', language_population)
write_python_dict(outfile, 'LANGUAGE_WRITING_POPULATION', language_writing_population)
if __name__ == '__main__':
build_data()