docs: Add comments on evolution of un-countries-dbHEADmain

1 files changed, 84 insertions, 64 deletions

diff --git a/get_country_info.py b/get_country_info.py
index c43a4d5..a5dda44 100644
--- a/get_country_info.py
+++ b/get_country_info.py
@@ -9,22 +9,24 @@ The United Nations Terminology Database is a multilingual database for official
 Official terminology is provided in the six official languages of the United Nations — Arabic, Chinese, English, French, Spanish and Russian.
 """
 
-GET_RECORDS_ID_URL = "https://conferences.unite.un.org/untermapi/api/term/search?query=*&page="
+GET_RECORDS_ID_URL = (
+    "https://conferences.unite.un.org/untermapi/api/term/search?query=*&page="
+)
 GET_INFO_COUNTRY_URL = "https://conferences.unite.un.org/untermapi/api/record/"
 
 OUTPUT_FILE_NAME = "un_formal_country_names.json"
 OUTPUT_FILE_NAME_SMALL = "un_formal_country_names_min.json"
 
 REQUEST_HEADERS = {
-    'content-type': 'application/json',
-    'accept': 'application/json, text/plain'
+    "content-type": "application/json",
+    "accept": "application/json, text/plain",
 }
 REQUEST_DATA = {
     "searchTerm": "*",
     "searchType": 0,
     "searchLanguages": ["ar", "es", "en", "fr", "ru", "zh"],
-    "languagesDisplay":  ["en", "fr"],
-    "datasets":  ["UNHQ"],
+    "languagesDisplay": ["en", "fr"],
+    "datasets": ["UNHQ"],
     "bodies": [],
     "subjects": ["Country names"],
     "recordTypes": [],
@@ -35,53 +37,61 @@ REQUEST_DATA = {
     "footnoteSearch": False,
     "fullTextSearch": False,
     "facetedSearch": True,
-    "buildSubjectList": False
+    "buildSubjectList": False,
 }
 
 languagesIDToLanguage = {
-    '5a504ded-6ba9-49c9-98aa-c8418cbb1ce9': 'arabic',
-    '676ccfad-a7b3-4e36-8d49-b21a9b5e0d7d': 'chinese',
-    '77ebac02-6a97-4722-859a-de5896a4d34e': 'english',
-    'ca7853cf-0f94-4aa6-be0a-ea586361c19e': 'french',
-    '496bcad8-7cd0-4f46-85c7-23b91e7a4ad2': 'russian',
-    'af320177-7bdb-40df-9521-6d2e0478f129': 'spanish',
+    "5a504ded-6ba9-49c9-98aa-c8418cbb1ce9": "arabic",
+    "676ccfad-a7b3-4e36-8d49-b21a9b5e0d7d": "chinese",
+    "77ebac02-6a97-4722-859a-de5896a4d34e": "english",
+    "ca7853cf-0f94-4aa6-be0a-ea586361c19e": "french",
+    "496bcad8-7cd0-4f46-85c7-23b91e7a4ad2": "russian",
+    "af320177-7bdb-40df-9521-6d2e0478f129": "spanish",
 }
 
 
 def main():
     countries = {}
 
+    # TODO: Put more logs
     recordsID = getRecordsID()
 
+    # NOTE: Maybe use tqdm here
     for records in recordsID:
         infoLang = getInfoCountry(records)
-        countries[infoLang['iso_a2']] = infoLang
+        countries[infoLang["iso_a2"]] = infoLang
 
     # Sort the dictionnary to get the same result every times.
     countries = {i2: countries[i2] for i2 in sorted(list(countries.keys()))}
+    # WARN: Also should sort inside countries
 
-    with open(OUTPUT_FILE_NAME, 'w') as f:
+    # TODO: Maybe add more output fileformat ?
+    with open(OUTPUT_FILE_NAME, "w") as f:
         json.dump(countries, f)
 
-    with open(OUTPUT_FILE_NAME_SMALL, 'w') as f:
+    with open(OUTPUT_FILE_NAME_SMALL, "w") as f:
         json.dump(countries, f, ensure_ascii=False)
 
+
 def getRecordsID():
     """
-        This function extracts the recordID of every results of every page so they can be queried later
+    This function extracts the recordID of every results of every page so they can be queried later
     """
+
     def getRecordsIDFromResponse(resp):
         """
         This function extracts all the recordID of a page and return them.
         """
         data = resp.json()
-        if 'results' not in data:
+        if "results" not in data:
             print("Missing results in request")
             exit(4)
 
-        return [r['recordID'] for r in data['results'] if 'recordID' in r]
+        return [r["recordID"] for r in data["results"] if "recordID" in r]
 
-    firstRequest = requests.post(GET_RECORDS_ID_URL + "0", data=json.dumps(REQUEST_DATA), headers=REQUEST_HEADERS)
+    firstRequest = requests.post(
+        GET_RECORDS_ID_URL + "0", data=json.dumps(REQUEST_DATA), headers=REQUEST_HEADERS
+    )
     if firstRequest.status_code != 200:
         print("The first request failed. Check the URL")
         print(firstRequest.status_code)
@@ -90,16 +100,21 @@ def getRecordsID():
 
     data = firstRequest.json()
 
-    if 'results' not in data:
+    if "results" not in data:
         print("Missing results")
         exit(3)
 
     recordsID = getRecordsIDFromResponse(firstRequest)
-    nbPage = int(math.ceil(data['totalHits']) / int(data['pageSize']))
+    nbPage = int(math.ceil(data["totalHits"]) / int(data["pageSize"]))
 
+    # NOTE: Maybe use tqdm here
     for page in range(1, nbPage + 1):
         print(page)
-        res = requests.post(GET_RECORDS_ID_URL + str(page), data=json.dumps(REQUEST_DATA), headers=REQUEST_HEADERS)
+        res = requests.post(
+            GET_RECORDS_ID_URL + str(page),
+            data=json.dumps(REQUEST_DATA),
+            headers=REQUEST_HEADERS,
+        )
         if res.status_code != 200:
             print("Unable to get a page that should be accessible")
             exit(5)
@@ -108,14 +123,16 @@ def getRecordsID():
 
     return recordsID
 
+
 def getInfoCountry(recordID):
     """
-        This function extract the information of a recordID.
-        Each record should match a country so this function extract the names of a country in the 6 offical UN languages.
+    This function extract the information of a recordID.
+    Each record should match a country so this function extract the names of a country in the 6 offical UN languages.
     """
+
     def removeExtraInfo(countryName):
-        if '(' in countryName:
-            return countryName[:countryName.index('(')].strip()
+        if "(" in countryName:
+            return countryName[: countryName.index("(")].strip()
         else:
             return countryName
 
@@ -127,64 +144,67 @@ def getInfoCountry(recordID):
     data = res.json()
     countryInfo = {}
 
-    for lang in sorted(data['languages']):
+    for lang in sorted(data["languages"]):
         lang = lang.lower()
-        countryInfo[lang] = {'short': '', 'full': ''}
+        countryInfo[lang] = {"short": "", "full": ""}
 
-        for i in range(len(data[lang]['terms'])):
-            countryInfo[lang][data[lang]['terms'][i]['termType']] = data[lang]['terms'][i]['term']
+        for i in range(len(data[lang]["terms"])):
+            countryInfo[lang][data[lang]["terms"][i]["termType"]] = data[lang]["terms"][
+                i
+            ]["term"]
 
         # The UK doesn't have official short name in russian. ¯\_(ツ)_/¯
-        if not countryInfo[lang]['short'] and countryInfo[lang]['full']:
-            countryInfo[lang]['short'] = countryInfo[lang]['full']
+        if not countryInfo[lang]["short"] and countryInfo[lang]["full"]:
+            countryInfo[lang]["short"] = countryInfo[lang]["full"]
 
         try:
-            countryInfo[lang]['map_usable'] = removeExtraInfo(countryInfo[lang]['short'])
+            countryInfo[lang]["map_usable"] = removeExtraInfo(
+                countryInfo[lang]["short"]
+            )
         except KeyError:
             print(f"What the fuck {lang}?!")
             print(f"Missing name for this country in {lang}", countryInfo)
 
-
-    for field in sorted(data['specialFields'], key=lambda item: item['name']):
-        match field['name']:
+    for field in sorted(data["specialFields"], key=lambda item: item["name"]):
+        match field["name"]:
             case "ISO Country alpha-2-code":
-                countryInfo['iso_a2'] = field['value']
+                countryInfo["iso_a2"] = field["value"]
 
             case "ISO Country alpha-3-code":
-                countryInfo['iso_a3'] = field['value']
+                countryInfo["iso_a3"] = field["value"]
 
-            case 'Monetary Unit':
-                if field['languageId'] in languagesIDToLanguage:
-                    lang = languagesIDToLanguage[field['languageId']]
-                    countryInfo[lang]['monetary_unit'] = field['value']
+            case "Monetary Unit":
+                if field["languageId"] in languagesIDToLanguage:
+                    lang = languagesIDToLanguage[field["languageId"]]
+                    countryInfo[lang]["monetary_unit"] = field["value"]
 
-            case 'Currency Designation':
-                if field['languageId'] in languagesIDToLanguage:
-                    lang = languagesIDToLanguage[field['languageId']]
-                    countryInfo[lang]['currency_designation'] = field['value']
+            case "Currency Designation":
+                if field["languageId"] in languagesIDToLanguage:
+                    lang = languagesIDToLanguage[field["languageId"]]
+                    countryInfo[lang]["currency_designation"] = field["value"]
 
-            case 'Currency Symbol':
-                if field['languageId'] in languagesIDToLanguage:
-                    lang = languagesIDToLanguage[field['languageId']]
-                    countryInfo[lang]['currency_symbol'] = field['value']
+            case "Currency Symbol":
+                if field["languageId"] in languagesIDToLanguage:
+                    lang = languagesIDToLanguage[field["languageId"]]
+                    countryInfo[lang]["currency_symbol"] = field["value"]
 
-            case 'Fractional Unit':
-                if field['languageId'] in languagesIDToLanguage:
-                    lang = languagesIDToLanguage[field['languageId']]
-                    countryInfo[lang]['fractional_unit'] = field['value']
+            case "Fractional Unit":
+                if field["languageId"] in languagesIDToLanguage:
+                    lang = languagesIDToLanguage[field["languageId"]]
+                    countryInfo[lang]["fractional_unit"] = field["value"]
 
-            case 'Capital City':
-                if field['languageId'] in languagesIDToLanguage:
-                    lang = languagesIDToLanguage[field['languageId']]
-                    countryInfo[lang]['capital_city'] = field['value']
+            case "Capital City":
+                if field["languageId"] in languagesIDToLanguage:
+                    lang = languagesIDToLanguage[field["languageId"]]
+                    countryInfo[lang]["capital_city"] = field["value"]
 
-            case 'Adjective':
-                if field['languageId'] in languagesIDToLanguage:
-                    lang = languagesIDToLanguage[field['languageId']]
-                    countryInfo[lang]['adjective'] = field['value']
+            case "Adjective":
+                if field["languageId"] in languagesIDToLanguage:
+                    lang = languagesIDToLanguage[field["languageId"]]
+                    countryInfo[lang]["adjective"] = field["value"]
 
-            case 'Date of Entry in UN':
-                countryInfo['un_entry_date'] = field['value']
+            case "Date of Entry in UN":
+                countryInfo["un_entry_date"] = field["value"]
 
     return countryInfo