I18N: Fix AppData international generation.

Language codes must be unique (pt is in double).
In addition, rework code to make it cleaner.

devtools/generate-metainfo.py

@@ -5,8 +5,10 @@
 
 import re
 import os
+import xml.sax.saxutils
 
-metainfo_xml_template = '''<?xml version="1.0" encoding="UTF-8"?>
+METAINFO_OUTPUT_FILE = 'dists/org.scummvm.scummvm.metainfo.xml'
+METAINFO_XML_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
 <!-- Copyright 2020-2023 The ScummVM Team -->
 <component type="desktop-application">
   <id>org.scummvm.scummvm</id>
@@ -98,163 +100,162 @@ metainfo_xml_template = '''<?xml version="1.0" encoding="UTF-8"?>
 </component>
 '''
 
-def extract_summary(file):
+SUMMARY_TAG = 'dists/org.scummvm.scummvm.metainfo.xml.cpp:32'
-    with open('../po/' + file) as f:
+SUMMARY_PAT = r'  <summary xml:lang="xy">I18N: One line summary as shown in *nix distributions</summary>'
+PAR_TAGS = [
+    'dists/org.scummvm.scummvm.metainfo.xml.cpp:37', # Paragraph 1
+    'dists/org.scummvm.scummvm.metainfo.xml.cpp:45', # Paragraph 2
+    'dists/org.scummvm.scummvm.metainfo.xml.cpp:51', # Paragraph 3
+]
+PAR_PATS = [
+    r'    <p xml:lang="xy">I18N: 1 of 3 paragraph of ScummVM description in *nix distributions</p>',
+    r'    <p xml:lang="xy">I18N: 2 of 3 paragraph of ScummVM description in *nix distributions</p>',
+    r'    <p xml:lang="xy">I18N: 3 of 3 paragraph of ScummVM description in *nix distributions</p>',
+]
+
+BASE_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
+
+def extract_po_line(file, tag):
+    with open(os.path.join(BASE_PATH, 'po', file), 'r') as f:
         content = f.read()
-    pattern = r'#: dists\/org\.scummvm\.scummvm\.metainfo\.xml\.cpp:32\nmsgid ".+"\nmsgstr "(.+)"'
+    pattern = r'#: {0}\nmsgid ".+"\nmsgstr "(.+)"\n'.format(re.escape(tag))
-    summary_match = re.search(pattern, content)
+    line_match = re.search(pattern, content)
-    if summary_match:
+    if line_match:
-        summary = summary_match.group(1)
+        return line_match.group(1)
-        return summary.strip()
     else:
         return None
 
 
-def extract_par1(file):
+def extract_po_par(file, tag):
-    with open('../po/' + file) as f:
+    with open(os.path.join(BASE_PATH, 'po', file), 'r') as f:
         content = f.read()
-    pattern = r'#: dists\/org\.scummvm\.scummvm\.metainfo\.xml\.cpp:37\nmsgid ""\n(.+\n)*msgstr ""\n((.+\n)*)'
+    pattern = r'#: {0}\nmsgid ""\n(?:".*"\n)+msgstr ""\n((?:".+"\n)+)'.format(re.escape(tag))
-    par1_match = re.search(pattern, content)
+    par_match = re.search(pattern, content)
-    if par1_match:
+    if par_match:
-        par1 = par1_match.group(2)
+        par = par_match.group(1)
-        if par1:
+        # Remove trailing \n
-            return par1.strip()
+        par = par[:-1]
-        else:
+        # Remove quotes at start and end of line
-            return None
+        return [line[1:-1] for line in par.split('\n')]
     else:
         return None
 
 
-def extract_par2(file):
+def po_to_lang(po_file_name):
-    with open('../po/' + file) as f:
+    # Remove .po extension
-        content = f.read()
+    lang = po_file_name[:-3]
-    pattern = r'#: dists\/org\.scummvm\.scummvm\.metainfo\.xml\.cpp:45\nmsgid ""\n(.+\n)*msgstr ""\n((.+\n)*)'
-    par2_match = re.search(pattern, content)
-    if par2_match:
-        par2 = par2_match.group(2)
-        if par2:
-            return par2.strip()
-        else:
-            return None
-    else:
-        return None
 
+    region_subtag = None
+    variant_subtag = None
+    # we use - for locale modifier (tarask)
+    if '-' in lang:
+        lang, variant_subtag = lang.split('-', maxsplit=1)
 
-def extract_par3(file):
+    if '_' in lang:
-    with open('../po/' + file) as f:
+        lang, region_subtag = lang.split('_', maxsplit=1)
-        content = f.read()
+
-    pattern = r'#: dists\/org\.scummvm\.scummvm\.metainfo\.xml\.cpp:51\nmsgid ""\n(.+\n)*msgstr ""\n((.+\n)*)'
+    primary_subtag = lang
-    par3_match = re.search(pattern, content)
+
-    if par3_match:
+    assert(len(primary_subtag) == 2)
-        par3 = par3_match.group(2)
+    assert(region_subtag is None or len(region_subtag) == 2)
-        if par3:
+    assert(variant_subtag is None or 6 <= len(variant_subtag) <= 8)
-            return par3.strip()
+
-        else:
+    lang = primary_subtag.lower()
-            return None
+    if region_subtag:
-    else:
+        lang += '-' + region_subtag.upper()
-        return None
+    if variant_subtag:
+        lang += '-' + variant_subtag.lower()
+
+    return primary_subtag, lang
 
 
 def get_summary_translations(po_file_names):
-    summary_translations = ""
+    summary_translations = []
 
-    # first_translation is used to determine the indentation (first translation will not require any indentation)
+    for file, lang in po_file_names.items():
-    first_translation = True
+        summary = extract_po_line(file, SUMMARY_TAG)
-
+        if summary is None:
-    for file in po_file_names:
-        summary = extract_summary(file)
-        if (summary is None):
             continue
 
-        lang = '"' + file[0] + file[1] + '"'
+        summary = xml.sax.saxutils.escape(summary)
-        summary_translations += ('' if first_translation else '  ') + '<summary xml:lang=' + \
-            lang + '>' + summary + '</summary>\n'
-        first_translation = False
 
-    summary_translations = summary_translations.rstrip('\n')
+        summary_translations.append('  <summary xml:lang="{0}">{1}</summary>'.format(
+                lang, summary))
 
-    return summary_translations
+    return '\n'.join(summary_translations)
 
 
-def substitute_summary_translations(summary_translations, xml):
+def substitute_summary_translations(po_file_names, xml):
-    pattern = r'<summary xml:lang="xy">I18N: One line summary as shown in \*nix distributions<\/summary>'
+    summary_translations = get_summary_translations(po_file_names)
-
+    return xml.replace(SUMMARY_PAT, summary_translations)
-    metainfo_xml = re.sub(pattern, summary_translations, xml)
-
-    return metainfo_xml
 
 
-def get_parx_translations(x, po_file_names):
+def get_parx_translations(po_file_names, tag):
-    parx_translations = ""
+    parx_translations = []
-    first_translation = True
 
-    for file in po_file_names:
+    for file, lang in po_file_names.items():
-        parx = ""
+        parx = extract_po_par(file, tag)
-        if (x == 1):
-            parx = extract_par1(file)
-        elif (x == 2):
-            parx = extract_par2(file)
-        else:
-            parx = extract_par3(file)
 
-        if (parx is None):
+        if parx is None:
             continue
 
-        # parx also contains " (quotes) around the text; so we need to replace them with empty character
+        # In XML a newline will be replace by a space character
-        # otherwise " (quotes) will appear in scummvm.metainfo.xml generated file
+        # Join everything making it pretty and remove trailing spaces
-        parx = parx.replace('"', '')
+        parx = '\n'.join(line.rstrip(' ') for line in parx)
+        parx = xml.sax.saxutils.escape(parx)
 
-        lang = '"' + file[0] + file[1] + '"'
+        parx_translations.append('    <p xml:lang="{0}">{1}</p>'.format(lang, parx))
-        parx_translations += ('' if first_translation else '    ') + '<p xml:lang=' + \
-            lang + '>' + parx + '</p>\n'
-        first_translation = False
 
-    parx_translations = parx_translations.rstrip('\n')
+    return '\n'.join(parx_translations)
-
-    return parx_translations
 
 
-def substitute_parx_translations(x, parx_translations, xml):
+def substitute_parx_translations(po_file_names, xml):
-    pattern = r'<p xml:lang="xy">I18N: 1 of 3 paragraph of ScummVM description in \*nix distributions<\/p>'
+    for tag, pat in zip(PAR_TAGS, PAR_PATS):
-
+        parx_translations = get_parx_translations(po_file_names, tag)
-    if (x == 2):
+        xml = xml.replace(pat, parx_translations)
-        pattern = r'<p xml:lang="xy">I18N: 2 of 3 paragraph of ScummVM description in \*nix distributions<\/p>'
+    return xml
-    elif (x == 3):
-        pattern = r'<p xml:lang="xy">I18N: 3 of 3 paragraph of ScummVM description in \*nix distributions<\/p>'
-
-    metainfo_xml = re.sub(pattern, parx_translations, xml)
-    return metainfo_xml
 
 
 def get_po_files():
     po_file_names = []
-    for filename in os.listdir("../po/"):
+    for filename in os.listdir(os.path.join(BASE_PATH, 'po')):
         if filename.endswith(".po"):
             po_file_names.append(filename)
 
     po_file_names.sort()
 
-    return po_file_names
+    po_langs = {}
+    last_primary = None
+    last_file = None
+    for file in po_file_names:
+        primary_subtag, lang = po_to_lang(file)
+        if last_primary != primary_subtag:
+            # We are sorted so it's a new primary group
+            last_primary = primary_subtag
+            did_dedup = False
+            # Try with primary subtag first
+            po_langs[file] = primary_subtag
+        else:
+            if not did_dedup:
+                # Fix last file name because we have duplicate
+                po_langs[last_file] = po_to_lang(last_file)[1]
+                did_dedup = True
+            # We got a duplicate lang code: deduplicate
+            po_langs[file] = lang
+        last_file = file
+
+    return po_langs
 
 
 def main():
     po_file_names = get_po_files()
 
-    summary_translations = get_summary_translations(po_file_names)
     xml = substitute_summary_translations(
-        summary_translations, metainfo_xml_template)
+        po_file_names, METAINFO_XML_TEMPLATE)
 
-    par1_translations = get_parx_translations(1, po_file_names)
+    xml = substitute_parx_translations(po_file_names, xml)
-    xml = substitute_parx_translations(1, par1_translations, xml)
 
-    par2_translations = get_parx_translations(2, po_file_names)
+    with open(os.path.join(BASE_PATH, METAINFO_OUTPUT_FILE), 'w') as f:
-    xml = substitute_parx_translations(2, par2_translations, xml)
-
-    par3_translations = get_parx_translations(3, po_file_names)
-    xml = substitute_parx_translations(3, par3_translations, xml)
-
-    # write to org.scummvm.scummvm.metainfo.xml file
-    with open("../dists/org.scummvm.scummvm.metainfo.xml", "w") as f:
         f.write(xml)
 
 

dists/org.scummvm.scummvm.metainfo.xml

@@ -7,7 +7,7 @@
   <name>ScummVM</name>
   <summary>Interpreter for numerous adventure games and role-playing games</summary>
   <summary xml:lang="ar">مترجم للعديد من ألعاب المغامرات ولعب الأدوار</summary>
-  <summary xml:lang="be">Інтэрпрэтатар многіх авантурных і ролявых гульняў</summary>
+  <summary xml:lang="be-tarask">Інтэрпрэтатар многіх авантурных і ролявых гульняў</summary>
   <summary xml:lang="ca">Intèrpret per a nombrosos jocs d'aventura i jocs de rol</summary>
   <summary xml:lang="de">Interpreter für zahlreiche Adventure-Spiele und RPGs</summary>
   <summary xml:lang="es">Intérprete de numerosas aventuras gráficas y RPG</summary>
@@ -21,8 +21,8 @@
   <summary xml:lang="nb">Programvaretolk for utallige eventyr- og rollespill</summary>
   <summary xml:lang="nl">Interpreter voor diverse adventure spellen en rollenspellen</summary>
   <summary xml:lang="pl">Interpreter wielu gier przygodowych oraz RPG</summary>
-  <summary xml:lang="pt">Interpretador para vários jogos de aventura e RPG</summary>
+  <summary xml:lang="pt-BR">Interpretador para vários jogos de aventura e RPG</summary>
-  <summary xml:lang="pt">Interpretador de vários jogos de aventura e RPGs</summary>
+  <summary xml:lang="pt-PT">Interpretador de vários jogos de aventura e RPGs</summary>
   <summary xml:lang="ru">Интерпретатор множества приключенческих и ролевых игр</summary>
   <summary xml:lang="uk">Інтерпретатор для численних пригодницьких і рольових ігор</summary>
   <developer_name>The ScummVM Team</developer_name>
@@ -43,7 +43,7 @@
 البيانات الخاصة بهم. الجزء الذكي في هذا الأمر: يستبدل ScummVM الملفات
 التنفيذية التي يتم شحنها مع اللعبة ، مما يتيح لك تشغيلها على أنظمة لم يتم
 تصميمها من أجلها مطلقًا!</p>
-    <p xml:lang="be">ScummVM — гэта праграма для запуску розных клясічных графічных point-and-
+    <p xml:lang="be-tarask">ScummVM — гэта праграма для запуску розных клясічных графічных point-and-
 click авантурных і ралявых гульняў, пры ўмове, што вы маеце зьвесткі тых
 гульняў. Карацей, ScummVM проста замяняе выканальныя файлы гульняў,
 дазваляючы гуляць на сістэмах, на якія іх ніколі не распрацоўвалі!</p>
@@ -115,7 +115,7 @@ waardoor je ze kan spelen op systemen waar ze niet voor waren gemaakt!</p>
 graficznych gier przygodowych typu „wskaż i kliknij”, oraz gier RPG - pod
 warunkiem posiadania ich danych. ScummVM podmienia oryginalny program gier,
 pozwalając na granie w nie na systemach, na które nigdy nie były napisane!</p>
-    <p xml:lang="pt">ScummVM é um programa que permite que você execute uma ampla variedade de 
+    <p xml:lang="pt-BR">ScummVM é um programa que permite que você execute uma ampla variedade de
 jogos clássicos de point-and-click (apontar e clicar) e RPG, desde que você
 já tenha seus arquivos de dados. A melhor parte: ScummVM apenas substitui os
 executáveis do jogo, permitindo que você os jogue em sistemas para os quais
@@ -139,7 +139,7 @@ eles nunca foram projetados!</p>
 المجموع. وهو يدعم العديد من الكلاسيكيات التي نشرتها الاستوديوهات الأسطورية
 مثل LucasArts و Sierra On-Line و Revolution Software و Cyan، Inc. و Westwood
 Studios.</p>
-    <p xml:lang="be">На гэты момант ScummVM падтрымлівае вялічэзную бібліятэку авантур, дзе болей 
+    <p xml:lang="be-tarask">На гэты момант ScummVM падтрымлівае вялічэзную бібліятэку авантур, дзе болей
 за 4000 гульняў. Вы зможаце дакрануцца да многіх клясічных твораў, выданых
 леґэндарнымі студыямі як LucasArts, Sierra On-Line, Revolution Software,
 Cyan, Inc. і Westwood Studios.</p>
@@ -198,7 +198,7 @@ Cyan, Inc. en Westwood Studios.</p>
 gier. Obsługuje wiele klasycznych gier wydanych przez legandarne studia
 takie jak LucasArts, Sierra On-Line, Revolution Software, Cyan, Inc. oraz
 Westwood Studios.</p>
-    <p xml:lang="pt">Atualmente o ScummVM suporta uma enorme biblioteca de aventuras com mais de 
+    <p xml:lang="pt-BR">Atualmente o ScummVM suporta uma enorme biblioteca de aventuras com mais de
 4000 jogos no total. Ele suporta muitos clássicos publicados por estúdios
 lendários como LucasArts, Sierra On-Line, Revolution Software, Cyan, Inc. e
 Westwood Studios.</p>
@@ -219,7 +219,7 @@ Revolution Software, Cyan, Inc. та Westwood Studios.</p>
     <p xml:lang="ar">بجانب العناوين الرائدة مثل سلسلة Monkey Island و Broken Sword و Myst و Blade
 Runner وعدد لا يحصى من الألعاب الأخرى ، ستجد بعض المغامرات الغامضة حقًا
 والأحجار الكريمة المخفية حقًا لاستكشافها.</p>
-    <p xml:lang="be">Разам з рэвалюцыйнымі гульнямі як Monkey Island, Broken Sword, Myst, Blade 
+    <p xml:lang="be-tarask">Разам з рэвалюцыйнымі гульнямі як Monkey Island, Broken Sword, Myst, Blade
 Runner і безьлічам іншых гульняў, вы знойдзіце насамрэч малавядомыя авантуры
 й прыхаваныя скарбы.</p>
     <p xml:lang="ca">Al costat de títols innovadors com la sèrie Monkey Island, Broken Sword,
@@ -267,7 +267,7 @@ vinden en echte verborgen pareltjes ontdekken.</p>
     <p xml:lang="pl">Poza przełomowymi tytułami, takimi jak seria Monkey Island, Broken Sword,
 Myst, Blade Runner i wieloma innymi, znajdziesz tu również wiele mało
 znanych przygód i naprawdę mało znanych perełek.</p>
-    <p xml:lang="pt">Ao lado de títulos inovadores como a série Monkey Island, Broken Sword, 
+    <p xml:lang="pt-BR">Ao lado de títulos inovadores como a série Monkey Island, Broken Sword,
 Myst, Blade Runner e inúmeros outros jogos, você encontrará algumas
 aventuras realmente obscuras e verdadeiras joias raras para explorar.</p>
     <p xml:lang="ru">Наряду с новаторскими играми, такими как серия Monkey Island, Broken Sword,