CM3D2 Converter.translations

  1import os
  2import sys
  3import csv
  4import bpy
  5import re
  6import unicodedata
  7from bl_i18n_utils import settings as bl_i18n_settings
  8from .. import compat
  9from .pgettext_functions import *
 10
 11# get_true_locale() -> Returns the locale
 12# get_locale()      -> Returns the closest locale available for translation 
 13
 14DICT = dict() # The translations dictionary
 15# {locale: {msg_key: msg_translation, ...}, ...}                                                                                                                                                      
 16# locale is either a lang iso code (fr), a lang+country code (pt_BR), a lang+variant code (sr@latin), or a full code (uz_UZ@cyrilic).                                                                 
 17# msg_key is a tupple of (context, org_message)
 18
 19# max level of verbose messages to print. -1 = Nothing.
 20verbosity = 0
 21dump_messages = True
 22
 23is_verify_contexts  = False
 24is_check_duplicates = False
 25
 26handled_locales   = set()
 27translations_folder = os.path.dirname(__file__)
 28comments_dict = dict()
 29
 30csv.register_dialect('cm3d2_converter',
 31    delimiter        = ','            ,
 32    doublequote      = False          ,
 33    escapechar       = '\\'           ,
 34    lineterminator   = '\n'           ,
 35    quotechar        = '"'            ,
 36    quoting          = csv.QUOTE_ALL  ,
 37    skipinitialspace = False          ,
 38    strict           = True         
 39)
 40
 41
 42i18n_contexts = { identifer: getattr(bpy.app.translations.contexts, identifer) for identifer in bpy.app.translations.contexts_C_to_py.values() }
 43'''
 44i18n_contexts = {
 45    default_real        = None                ,
 46    default             = '*'                 ,
 47    operator_default    = 'Operator'          ,
 48    ui_events_keymaps   = 'UI_Events_KeyMaps' ,
 49    plural              = 'Plural'            ,
 50    id_action           = 'Action'            ,
 51    id_armature         = 'Armature'          ,
 52    id_brush            = 'Brush'             ,
 53    id_camera           = 'Camera'            ,
 54    id_cachefile        = 'CacheFile'         ,
 55    id_collection       = 'Collection'        ,
 56    id_curve            = 'Curve'             ,
 57    id_fs_linestyle     = 'FreestyleLineStyle',
 58    id_gpencil          = 'GPencil'           ,
 59    id_hair             = 'Hair'              ,
 60    id_id               = 'ID'                ,
 61    id_image            = 'Image'             ,
 62    id_shapekey         = 'Key'               ,
 63    id_light            = 'Light'             ,
 64    id_library          = 'Library'           ,
 65    id_lattice          = 'Lattice'           ,
 66    id_mask             = 'Mask'              ,
 67    id_material         = 'Material'          ,
 68    id_metaball         = 'Metaball'          ,
 69    id_mesh             = 'Mesh'              ,
 70    id_movieclip        = 'MovieClip'         ,
 71    id_nodetree         = 'NodeTree'          ,
 72    id_object           = 'Object'            ,
 73    id_paintcurve       = 'PaintCurve'        ,
 74    id_palette          = 'Palette'           ,
 75    id_particlesettings = 'ParticleSettings'  ,
 76    id_pointcloud       = 'PointCloud'        ,
 77    id_lightprobe       = 'LightProbe'        ,
 78    id_scene            = 'Scene'             ,
 79    id_screen           = 'Screen'            ,
 80    id_sequence         = 'Sequence'          ,
 81    id_simulation       = 'Simulation'        ,
 82    id_speaker          = 'Speaker'           ,
 83    id_sound            = 'Sound'             ,
 84    id_texture          = 'Texture'           ,
 85    id_text             = 'Text'              ,
 86    id_vfont            = 'VFont'             ,
 87    id_volume           = 'Volume'            ,
 88    id_world            = 'World'             ,
 89    id_workspace        = 'WorkSpace'         ,
 90    id_windowmanager    = 'WindowManager'
 91}
 92'''
 93
 94def print_verbose(level:int, *args, tab:str="\t", sep:str=" ", end:str="\n", file=sys.stdout, flush:bool=False, **format_args) -> None:
 95    if level > verbosity:
 96        return
 97    message = tab * level + sep.join(str(arg) for arg in args)
 98    if format_args:
 99        message = message.format(**format_args)
100    return print(message, end=end, file=file, flush=flush)
101
102def verify_context(context: str) -> bool:
103    if not is_verify_contexts:
104        return True
105
106def check_duplicate(key: tuple, lang: str) -> bool:
107    if not is_check_duplicates:
108        return False
109    if key in DICT[lang].keys():
110        return True
111    else:
112        return False
113
114
115def get_best_locale_match(locale: str, available=handled_locales) -> str:
116    # First check for exact locale tag match
117    if locale in available:
118        return locale
119    
120    # Otherwise match based on language, country, or variant.
121    match = 'en_US' # default is English as per Blender Dev's recomendations
122    affinity = 0
123    for locale_tag in DICT.keys():
124        language, country, variant, language_country, language_variant = bpy.app.translations.locale_explode(locale_tag)
125        
126        if   affinity < 4 and language_variant in available:
127            affinity  = 4
128            match     = locale_tag
129        elif affinity < 3 and language_country in available:
130            affinity  = 3
131            match     = locale_tag
132        elif affinity < 2 and language in available:
133            affinity  = 2
134            match     = locale_tag
135        elif affinity < 1 and country in available:
136            affinity  = 1
137            match     = locale_tag
138        # Not worth checking variant alone
139        #elif affinity < 0 and variant          in available: 
140        #    match    = locale_tag
141        #    affinity = 0
142    
143    return match
144
145def generate_translations(locale: str):
146    if False:
147        # Handle any special generations here
148        # e.g. Using Spanish for Portuguese because they are similar
149        #      even though they won't be detected by get_best_locale_match()
150        pass
151    else:
152        # If not specially handled, fill it with best match
153        match = get_best_locale_match(locale)
154        DICT[locale] = DICT[match]
155
156
157def get_true_locale() -> str:
158    true_locale = ''
159    if bpy.app.translations.locale:
160        true_locale = bpy.app.translations.locale
161    else: # if built without internationalization support
162        try:
163            import locale
164            if system.language =='DEFAULT':
165                true_locale = locale.getdefaultlocale()[0]
166        except Exception as e: 
167            print("Unable to determine locale.", e)
168    return true_locale
169
170def get_locale() -> str:
171    return get_best_locale_match(get_true_locale())
172
173
174src_path_pattern = re.compile(r":?\d+$")
175py_path_pattern  = re.compile(
176    r"(?:bpy\.types)?"
177    r"(?:\.|\:|\[[\'\"])" # find a dot, colon, or bracket-reference
178    r"([^\.]+?)"          # capture identifier of module/class/attribute
179    r"(?:[\'\"]\])?"      # consume any closing brackets or quotes
180    r"(?=\.|\:|$)"        # look-ahead for a . or : or the end of the string
181)
182comment_pattern  = re.compile(r"#\s")
183class_hash_dict = dict()
184def get_message_source_file(src):
185    if not src.startswith('bpy.'):
186        file_name = os.path.basename(src)
187        file_name = src_path_pattern.sub("", file_name)
188        return file_name
189    else:
190        cls_name = py_path_pattern.match(src)[1]
191        if not class_hash_dict:
192            for cls in compat.BlRegister.classes:
193                class_hash_dict[cls.bl_rna.identifier] = cls
194        
195        cls = class_hash_dict.get(cls_name)
196        if not cls:
197
198            return cls_name
199
200        module_name, file_name = os.path.splitext(cls.__module__)
201        if not file_name:
202            return "__init__.py"
203        elif file_name == "translations":
204            return "translations/__init__.py"
205        else:
206            return file_name[1:] + ".py"
207
208def message_to_csv_line(msg, add_sources=False):
209    line = [msg.msgctxt, msg.msgid, msg.msgstr]
210    
211    sources = f'FROM <{" & ".join(get_message_source_file(src) for src in msg.sources)}>' if add_sources else ""
212    comment = comment_pattern.sub("", msg.comment_lines[-1]) if msg.is_commented else ""
213    if sources or comment:
214        if not sources or sources in comment:
215            line.append(f"# {comment}")
216        elif not comment:
217            line.append(f"# {sources}")
218        else:
219            line.append(f"# {sources} {comment}")
220
221    class filestring:
222        def __init__(self, string:str=""):
223            self.string = string
224
225        def write(self, text):
226            if type(text) == bytes:
227                text = text.decode('utf-8')
228            self.string += text
229
230        trans = str.maketrans({'\n':'', '\r':''})
231        def str(self):
232            return self.string.translate(self.trans)
233
234    string = filestring("")
235    csv_writer = csv.writer(string, dialect='cm3d2_converter')
236    csv_writer.writerow(line)
237
238    return string.str()
239
240def messages_to_csv(msgs, reports, lang=get_locale(), only_missing=True):
241    lang_dict = DICT.get(lang)
242    text_data = "Context,Original,Translation,Comments\n" \
243                "# encoding UTF-8"
244    last_src = None
245    shared = []
246
247    messages = sorted(
248        ( (key, msg, get_message_source_file(msg.sources[0])) for key, msg in msgs.items() ),
249        key = lambda x : x[2]
250    )
251    for key, msg, src in messages:
252        msg.normalize()
253        key_check = (msg.msgctxt, msg.msgid)
254        if key_check != key:
255            print(f"A message with a mismatching key was found: {key} != {key_check}")
256        if lang_dict and lang_dict.get(key):
257            if only_missing:
258                continue
259            msg.msgstr = msg.msgstr or lang_dict[key]
260            comment = comments_dict[lang].get(key)
261            if comment:
262                msg.comment_lines.append(comment)
263                msg.is_commented = True
264
265        sources = msg.sources
266        if 'KM_HIERARCHY' in sources[0]:
267            continue
268        if len(sources) > 1:
269            new_src = get_message_source_file(sources[0])
270            if any(get_message_source_file(sub_src) != new_src for sub_src in sources):
271                shared.append(msg)
272                continue
273        
274        #src = get_message_source_file(msg.sources[0])
275        if src != last_src:
276            last_src = src
277            text_data = f"{text_data}\n# {src}"
278        text_data = f"{text_data}\n{message_to_csv_line(msg)}"
279
280    if len(shared) > 0:
281        text_data = f"{text_data}\n# shared"
282        for msg in shared:
283            text_data = f"{text_data}\n{message_to_csv_line(msg, add_sources=True)}"
284
285    return text_data
286
287def reports_to_csv(reports):
288    text_data = "### Generated Reports ###"
289    for key, value in reports.items():
290        text_data = f"{text_data}\n## {key} ##"
291        if hasattr(value, '__iter__') and not isinstance(value, (str, bytes)):
292            for value_item in value:
293                text_data = f"{text_data}\n{str(value_item)}"
294        else:
295            text_data = f"{text_data}\n{str(value)}"
296    
297    return text_data
298
299# bpy.ops.cm3d2_converter.dump_py_messages(do_checks=False, only_missing=True, language='en_US')
300@compat.BlRegister()
301class CNV_OT_dump_py_messages(bpy.types.Operator):
302    bl_idname      = 'cm3d2_converter.dump_py_messages'
303    bl_label       = "Dump Py Messages"
304    bl_description = "Dump the CM3D2 Converter's messages for CSV translation"
305    bl_options     = {'REGISTER', 'UNDO'}
306
307    do_checks    = bpy.props.BoolProperty(name="Do Checks"   , default=False)
308    do_reports   = bpy.props.BoolProperty(name="Do Reports"  , default=False)
309    only_missing = bpy.props.BoolProperty(name="Only Missing", default=False)
310    only_foreign = bpy.props.BoolProperty(name="Only Foreign", default=False)
311    
312    items = { 
313        (enum_str, enum_name, "", 'NONE', enum_int) \
314            for enum_int, enum_name, enum_str in bl_i18n_settings.LANGUAGES
315    }
316    language = bpy.props.EnumProperty(items=items, name="Language", default=get_locale())
317    
318    @classmethod
319    def poll(cls, context):
320        return True
321
322    @staticmethod
323    def is_japanese(string):
324        for ch in string:
325            name = unicodedata.name(ch)
326            if 'CJK UNIFIED' in name or 'HIRAGANA' in name or 'KATAKANA' in name:
327                return True
328
329    def invoke(self, context, event):
330        return context.window_manager.invoke_props_dialog(self)
331
332    def draw(self, context):
333        self.layout.prop(self, 'do_checks')
334        self.layout.prop(self, 'do_reports')
335        self.layout.prop(self, 'only_missing')
336        self.layout.prop(self, 'language')
337        row = self.layout.row()
338        row.prop(self, 'only_foreign')
339        row.enabled = self.language in ('en_US', 'ja_JP')
340
341    def execute(self, context):
342        from . import extract_messages
343
344        msgs = dict()
345        reports = extract_messages._gen_reports(
346            extract_messages._gen_check_ctxt(bl_i18n_settings) if self.do_checks else None
347        )
348
349        extract_messages.dump_rna_messages(
350            msgs       = msgs,
351            reports    = reports,
352            settings   = bl_i18n_settings,
353            verbose    = False,
354            class_list = [ meta.bl_rna.__class__ for meta in compat.BlRegister.classes ]
355        )
356
357        extract_messages.dump_py_messages(
358            msgs        = msgs, 
359            reports     = reports,
360            addons      = [__import__("CM3D2 Converter")],
361            settings    = bl_i18n_settings,
362            addons_only = True
363        )
364
365        # Clean un-wanted messages
366        for key in tuple(msgs.keys()):
367            msg = msgs.pop(key)
368            if '_OT_' in msg.msgid:
369                continue
370            if self.only_foreign:
371                is_ja = self.is_japanese(msg.msgid)
372                if is_ja and 'ja' in self.language:
373                    continue
374                if not is_ja and 'en' in self.language:
375                    continue
376            # else put it back
377            msgs[key] = msg
378
379        txt_data = messages_to_csv(msgs, reports, lang=self.language, only_missing=self.only_missing)
380        txt_name = "cm3d2_messages_csv"
381        if txt_name in context.blend_data.texts:
382            txt = context.blend_data.texts[txt_name]
383            txt.clear()
384        else:
385            txt = context.blend_data.texts.new(txt_name)
386        txt.write(txt_data)
387
388        if self.do_reports:
389            reports_txt_data = reports_to_csv(reports)
390            reports_txt_name = "cm3d2_message_reports"
391            if reports_txt_name in context.blend_data.texts:
392                reports_txt = context.blend_data.texts[reports_txt_name]
393                reports_txt.clear()
394            else:
395                reports_txt = context.blend_data.texts.new(reports_txt_name)
396            reports_txt.write(reports_txt_data)
397
398        self.report(type={'INFO'}, message=f_tip_("Strings have been dumped to {txt_name}. See text editor.", txt_name=txt_name))
399
400        return {'FINISHED'}
401
402
403
404pre_settings = None
405
406def register(__name__=__name__):
407    global DICT
408    global comments_dict
409    global pre_settings
410    system = compat.get_system(bpy.context)
411    if hasattr(system, 'use_international_fonts'):
412        pre_settings = system.use_international_fonts
413        system.use_international_fonts = True
414
415    # Since the add-on in Japanese we want to translate even when Blender's language is set to English.
416    elif bpy.app.version >= (2, 93):
417        if system.language in {'en_US', 'DEFAULT'}         \
418            and system.use_translate_tooltips     == False \
419            and system.use_translate_interface    == False \
420            and system.use_translate_new_dataname == False:
421
422            system.use_translate_tooltips     = True
423            system.use_translate_interface    = True
424            system.use_translate_new_dataname = True
425
426    # Work around for disabled translations when language is 'en_US', fixed in 2.93LTS
427    elif bpy.app.version >= (2, 83): 
428        if system.language == 'en_US':
429            pre_settings = system.language
430            system.language = 'DEFAULT'
431            # This hack is required because when language is changed to 'DEFAULT'
432            #   all the options will be set to false next time Blender updates
433            def _set():
434                system.use_translate_tooltips     = True
435                system.use_translate_interface    = True
436                system.use_translate_new_dataname = True
437            bpy.app.timers.register(_set, first_interval=0)
438
439    # Generate locales from csv files
440    for lang in os.listdir(translations_folder):
441        lang_folder = os.path.join(translations_folder, lang)
442        if not os.path.isdir(lang_folder):
443            continue
444        if lang.startswith("_") or lang.startswith("."):
445            continue
446
447        print_verbose(0, "Loading translations for", lang)
448        lang_dict = DICT.get(lang)
449        if not lang_dict:
450            lang_dict = dict()
451            DICT[lang] = lang_dict
452            comments_dict[lang] = dict()
453        
454        orig_count = len(lang_dict)
455        for csv_file_name in os.listdir(lang_folder):
456            _, ext = os.path.splitext(csv_file_name)
457            if ext.lower() != ".csv":
458                continue
459            
460            print_verbose(1, f"Reading {csv_file_name}")
461            entry_count = 0
462            dupe_count  = 0
463            csv_file_path = os.path.join(lang_folder, csv_file_name)
464            with open(csv_file_path, 'rt', encoding='utf-8', newline='') as csv_file:
465                csv_reader = csv.reader(csv_file, dialect='cm3d2_converter')
466                try:
467                    for line, row in enumerate(csv_reader):
468                        if line == 0: # ignore header
469                            continue
470                        if len(row) < 3:
471                            continue
472                        if row[0].lstrip()[0] == "#":
473                            # ignore comments
474                            continue
475                        if row[0] not in i18n_contexts.values():
476                            print_verbose(2, f"Unknown translation context \"{row[0]}\" on line {line}")
477
478                        key = (row[0], row[1])
479                        if check_duplicate(key, lang):
480                            print_verbose(2, f"Duplicate entry on line {line}")
481                            entry_count -= 1
482                            dupe_count  += 1
483
484                        value = row[2]
485                        lang_dict[key] = value
486                        entry_count += 1
487
488                        if len(row) > 3: # entry comment
489                            comments_dict[lang][key] = row[3]
490
491                        print_verbose(3, f"{line:{4}} {key}: {value}")
492                except Error as e:
493                    print(f"Error parsing {csv_file_name} in {lang_folder}:")
494                    print(e)
495
496            print_verbose(1, f"-> Added {entry_count} translations from {csv_file_name}")
497            if is_check_duplicates:
498                print_verbose(1, f"-> Replaced {dupe_count} translations with {csv_file_name}")
499        print_verbose(0, f"-> Added {len(lang_dict) - orig_count} translations for {lang}")
500
501
502    # Any special translations that use another as a base should handle that here
503    
504    # End special translations
505
506    handled_locales = { lang for lang in DICT.keys() }
507
508
509    # Fill missing translations
510    print_verbose(0, f"Generating missing translations...")
511    gen_count = 0
512    for lang in bpy.app.translations.locales:
513        if lang not in DICT.keys():
514            print_verbose(1, f"Generating translations for '{lang}'")
515            generate_translations(lang)
516            gen_count += 1
517    # For when system.language == 'DEFAULT'
518    true_locale = get_true_locale()
519    if true_locale not in DICT.keys():
520        print_verbose(1, f"Generating translations for '{true_locale}'")
521        generate_translations(true_locale)
522        gen_count += 1
523    print_verbose(0, f"-> Generated {gen_count} missing translations")
524
525    bpy.app.translations.register(__name__, DICT)
526
527def unregister(__name__=__name__):
528    DICT = dict()
529    handled_locales = set()
530    bpy.app.translations.unregister(__name__)
531
532    global pre_settings
533    if pre_settings != None:
534        system = compat.get_system(bpy.context)
535        if hasattr(system, 'use_international_fonts'):
536            system.use_international_fonts = pre_settings
537        
538        if bpy.app.version >= (2, 83, 0) and bpy.app.version < (2, 93):
539            system.language = pre_settings
540
541    pre_settings = None
542
543
544if __name__ == "__main__":
545    register()