CM3D2 Converter.translations.extract_messages

   1# ***** BEGIN GPL LICENSE BLOCK *****
   2#
   3# This program is free software; you can redistribute it and/or
   4# modify it under the terms of the GNU General Public License
   5# as published by the Free Software Foundation; either version 2
   6# of the License, or (at your option) any later version.
   7#
   8# This program is distributed in the hope that it will be useful,
   9# but WITHOUT ANY WARRANTY; without even the implied warranty of
  10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11# GNU General Public License for more details.
  12#
  13# You should have received a copy of the GNU General Public License
  14# along with this program; if not, write to the Free Software Foundation,
  15# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16#
  17# ***** END GPL LICENSE BLOCK *****
  18
  19# <pep8 compliant>
  20
  21# Populate a template file (POT format currently) from Blender RNA/py/C data.
  22# XXX: This script is meant to be used from inside Blender!
  23#      You should not directly use this script, rather use update_msg.py!
  24
  25import collections
  26import copy
  27import datetime
  28import os
  29import re
  30import sys
  31
  32# XXX Relative import does not work here when used from Blender...
  33from bl_i18n_utils import settings as settings_i18n, utils
  34
  35import bpy
  36
  37##### Utils #####
  38
  39# check for strings like "+%f°"
  40ignore_reg = re.compile(r"^(?:[-*.()/\\+%°0-9]|%d|%f|%s|%r|\s)*$")
  41filter_message = ignore_reg.match
  42
  43
  44def init_spell_check(settings, lang="en_US"):
  45    try:
  46        from bl_i18n_utils import utils_spell_check
  47        return utils_spell_check.SpellChecker(settings, lang)
  48    except Exception as e:
  49        print("Failed to import utils_spell_check ({})".format(str(e)))
  50        return None
  51
  52
  53def _gen_check_ctxt(settings):
  54    return {
  55        "multi_rnatip": set(),
  56        "multi_lines": set(),
  57        "py_in_rna": set(),
  58        "not_capitalized": set(),
  59        "end_point": set(),
  60        "undoc_ops": set(),
  61        "spell_checker": init_spell_check(settings),
  62        "spell_errors": {},
  63    }
  64
  65
  66def _diff_check_ctxt(check_ctxt, minus_check_ctxt):
  67    """Removes minus_check_ctxt from check_ctxt"""
  68    for key in check_ctxt:
  69        if isinstance(check_ctxt[key], set):
  70            for warning in minus_check_ctxt[key]:
  71                if warning in check_ctxt[key]:
  72                    check_ctxt[key].remove(warning)
  73        elif isinstance(check_ctxt[key], dict):
  74            for warning in minus_check_ctxt[key]:
  75                if warning in check_ctxt[key]:
  76                    del check_ctxt[key][warning]
  77
  78
  79def _gen_reports(check_ctxt):
  80    return {
  81        "check_ctxt": check_ctxt,
  82        "rna_structs": [],
  83        "rna_structs_skipped": [],
  84        "rna_props": [],
  85        "rna_props_skipped": [],
  86        "py_messages": [],
  87        "py_messages_skipped": [],
  88        "src_messages": [],
  89        "src_messages_skipped": [],
  90        "messages_skipped": set(),
  91    }
  92
  93
  94def check(check_ctxt, msgs, key, msgsrc, settings):
  95    """
  96    Performs a set of checks over the given key (context, message)...
  97    """
  98    if check_ctxt is None:
  99        return
 100    multi_rnatip = check_ctxt.get("multi_rnatip")
 101    multi_lines = check_ctxt.get("multi_lines")
 102    py_in_rna = check_ctxt.get("py_in_rna")
 103    not_capitalized = check_ctxt.get("not_capitalized")
 104    end_point = check_ctxt.get("end_point")
 105    undoc_ops = check_ctxt.get("undoc_ops")
 106    spell_checker = check_ctxt.get("spell_checker")
 107    spell_errors = check_ctxt.get("spell_errors")
 108
 109    if multi_rnatip is not None:
 110        if key in msgs and key not in multi_rnatip:
 111            multi_rnatip.add(key)
 112    if multi_lines is not None:
 113        if '\n' in key[1]:
 114            multi_lines.add(key)
 115    if py_in_rna is not None:
 116        if key in py_in_rna[1]:
 117            py_in_rna[0].add(key)
 118    if not_capitalized is not None:
 119        if(key[1] not in settings.WARN_MSGID_NOT_CAPITALIZED_ALLOWED and
 120           key[1][0].isalpha() and not key[1][0].isupper()):
 121            not_capitalized.add(key)
 122    if end_point is not None:
 123        if (
 124                key[1].strip().endswith('.') and
 125                (not key[1].strip().endswith('...')) and
 126                key[1] not in settings.WARN_MSGID_END_POINT_ALLOWED
 127        ):
 128            end_point.add(key)
 129    if undoc_ops is not None:
 130        if key[1] == settings.UNDOC_OPS_STR:
 131            undoc_ops.add(key)
 132    if spell_checker is not None and spell_errors is not None:
 133        err = spell_checker.check(key[1])
 134        if err:
 135            spell_errors[key] = err
 136
 137
 138def print_info(reports, pot):
 139    def _print(*args, **kwargs):
 140        kwargs["file"] = sys.stderr
 141        print(*args, **kwargs)
 142
 143    pot.update_info()
 144
 145    _print("{} RNA structs were processed (among which {} were skipped), containing {} RNA properties "
 146           "(among which {} were skipped).".format(len(reports["rna_structs"]), len(reports["rna_structs_skipped"]),
 147                                                   len(reports["rna_props"]), len(reports["rna_props_skipped"])))
 148    _print("{} messages were extracted from Python UI code (among which {} were skipped), and {} from C source code "
 149           "(among which {} were skipped).".format(len(reports["py_messages"]), len(reports["py_messages_skipped"]),
 150                                                   len(reports["src_messages"]), len(reports["src_messages_skipped"])))
 151    _print("{} messages were rejected.".format(len(reports["messages_skipped"])))
 152    _print("\n")
 153    _print("Current POT stats:")
 154    pot.print_info(prefix="\t", output=_print)
 155    _print("\n")
 156
 157    check_ctxt = reports["check_ctxt"]
 158    if check_ctxt is None:
 159        return
 160    multi_rnatip = check_ctxt.get("multi_rnatip")
 161    multi_lines = check_ctxt.get("multi_lines")
 162    py_in_rna = check_ctxt.get("py_in_rna")
 163    not_capitalized = check_ctxt.get("not_capitalized")
 164    end_point = check_ctxt.get("end_point")
 165    undoc_ops = check_ctxt.get("undoc_ops")
 166    spell_errors = check_ctxt.get("spell_errors")
 167
 168    # XXX Temp, no multi_rnatip nor py_in_rna, see below.
 169    keys = multi_lines | not_capitalized | end_point | undoc_ops | spell_errors.keys()
 170    if keys:
 171        _print("WARNINGS:")
 172        for key in keys:
 173            if undoc_ops and key in undoc_ops:
 174                _print("\tThe following operators are undocumented!")
 175            else:
 176                _print("\t“{}”|“{}”:".format(*key))
 177                # We support multi-lines tooltips now...
 178                # ~ if multi_lines and key in multi_lines:
 179                    # ~ _print("\t\t-> newline in this message!")
 180                if not_capitalized and key in not_capitalized:
 181                    _print("\t\t-> message not capitalized!")
 182                if end_point and key in end_point:
 183                    _print("\t\t-> message with endpoint!")
 184                # XXX Hide this one for now, too much false positives.
 185                #if multi_rnatip and key in multi_rnatip:
 186                #   _print("\t\t-> tip used in several RNA items")
 187                #if py_in_rna and key in py_in_rna:
 188                #   _print("\t\t-> RNA message also used in py UI code!")
 189                if spell_errors and spell_errors.get(key):
 190                    lines = [
 191                        "\t\t-> {}: misspelled, suggestions are ({})".format(w, "'" + "', '".join(errs) + "'")
 192                        for w, errs in spell_errors[key]
 193                    ]
 194                    _print("\n".join(lines))
 195            _print("\t\t{}".format("\n\t\t".join(pot.msgs[key].sources)))
 196
 197
 198def process_msg(msgs, msgctxt, msgid, msgsrc, reports, check_ctxt, settings):
 199    if filter_message(msgid):
 200        reports["messages_skipped"].add((msgid, msgsrc))
 201        return
 202    if not msgctxt:
 203        # We do *not* want any "" context!
 204        msgctxt = settings.DEFAULT_CONTEXT
 205    # Always unescape keys!
 206    msgctxt = utils.I18nMessage.do_unescape(msgctxt)
 207    msgid = utils.I18nMessage.do_unescape(msgid)
 208    key = (msgctxt, msgid)
 209    check(check_ctxt, msgs, key, msgsrc, settings)
 210    msgsrc = settings.PO_COMMENT_PREFIX_SOURCE_CUSTOM + msgsrc
 211    if key not in msgs:
 212        msgs[key] = utils.I18nMessage([msgctxt], [msgid], [], [msgsrc], settings=settings)
 213    else:
 214        msgs[key].comment_lines.append(msgsrc)
 215
 216
 217##### RNA #####   # CM3D2 Converter: added class_list param
 218def dump_rna_messages(msgs, reports, settings, verbose=False, class_list=bpy.types.ID.__base__.__subclasses__()): 
 219    """
 220    Dump into messages dict all RNA-defined UI messages (labels en tooltips).
 221    """
 222    def class_blacklist():
 223        blacklist_rna_class = {getattr(bpy.types, cls_id) for cls_id in (
 224            # core classes
 225            "Context", "Event", "Function", "UILayout", "UnknownType", "Property", "Struct",
 226            # registerable classes
 227            "Panel", "Menu", "Header", "RenderEngine", "Operator", "OperatorMacro", "Macro", "KeyingSetInfo",
 228            # window classes
 229            "Window",
 230        )
 231        }
 232
 233        # More builtin classes we don't need to parse.
 234        blacklist_rna_class |= {cls for cls in bpy.types.Property.__subclasses__()}
 235
 236        return blacklist_rna_class
 237
 238    check_ctxt_rna = check_ctxt_rna_tip = None
 239    check_ctxt = reports["check_ctxt"]
 240    if check_ctxt:
 241        check_ctxt_rna = {
 242            "multi_lines": check_ctxt.get("multi_lines"),
 243            "not_capitalized": check_ctxt.get("not_capitalized"),
 244            "end_point": check_ctxt.get("end_point"),
 245            "undoc_ops": check_ctxt.get("undoc_ops"),
 246            "spell_checker": check_ctxt.get("spell_checker"),
 247            "spell_errors": check_ctxt.get("spell_errors"),
 248        }
 249        check_ctxt_rna_tip = check_ctxt_rna
 250        check_ctxt_rna_tip["multi_rnatip"] = check_ctxt.get("multi_rnatip")
 251
 252    default_context = settings.DEFAULT_CONTEXT
 253
 254    # Function definitions
 255    def walk_properties(cls):
 256        bl_rna = cls.bl_rna
 257        # Get our parents' properties, to not export them multiple times.
 258        bl_rna_base = bl_rna.base
 259        if bl_rna_base:
 260            bl_rna_base_props = set(bl_rna_base.properties.values())
 261        else:
 262            bl_rna_base_props = set()
 263
 264        props = sorted(bl_rna.properties, key=lambda p: p.identifier)
 265        for prop in props:
 266            # Only write this property if our parent hasn't got it.
 267            if prop in bl_rna_base_props:
 268                continue
 269            if prop.identifier == "rna_type":
 270                continue
 271            reports["rna_props"].append((cls, prop))
 272
 273            msgsrc = "bpy.types.{}.{}".format(bl_rna.identifier, prop.identifier)
 274            msgctxt = prop.translation_context or default_context
 275
 276            if prop.name and (prop.name != prop.identifier or msgctxt != default_context):
 277                process_msg(msgs, msgctxt, prop.name, msgsrc, reports, check_ctxt_rna, settings)
 278            if prop.description:
 279                process_msg(msgs, default_context, prop.description, msgsrc, reports, check_ctxt_rna_tip, settings)
 280
 281            if isinstance(prop, bpy.types.EnumProperty):
 282                done_items = set()
 283                for item in prop.enum_items:
 284                    msgsrc = "bpy.types.{}.{}:'{}'".format(bl_rna.identifier, prop.identifier, item.identifier)
 285                    done_items.add(item.identifier)
 286                    if item.name and item.name != item.identifier:
 287                        process_msg(msgs, msgctxt, item.name, msgsrc, reports, check_ctxt_rna, settings)
 288                    if item.description:
 289                        process_msg(msgs, default_context, item.description, msgsrc, reports, check_ctxt_rna_tip,
 290                                    settings)
 291                for item in prop.enum_items_static:
 292                    if item.identifier in done_items:
 293                        continue
 294                    msgsrc = "bpy.types.{}.{}:'{}'".format(bl_rna.identifier, prop.identifier, item.identifier)
 295                    done_items.add(item.identifier)
 296                    if item.name and item.name != item.identifier:
 297                        process_msg(msgs, msgctxt, item.name, msgsrc, reports, check_ctxt_rna, settings)
 298                    if item.description:
 299                        process_msg(msgs, default_context, item.description, msgsrc, reports, check_ctxt_rna_tip,
 300                                    settings)
 301
 302    def walk_tools_definitions(cls):
 303        from bl_ui.space_toolsystem_common import ToolDef
 304
 305        bl_rna = cls.bl_rna
 306        op_default_context = bpy.app.translations.contexts.operator_default
 307
 308        def process_tooldef(tool_context, tool):
 309            if not isinstance(tool, ToolDef):
 310                if callable(tool):
 311                    for t in tool(None):
 312                        process_tooldef(tool_context, t)
 313                return
 314            msgsrc = "bpy.types.{} Tools: '{}', '{}'".format(bl_rna.identifier, tool_context, tool.idname)
 315            if tool.label:
 316                process_msg(msgs, op_default_context, tool.label, msgsrc, reports, check_ctxt_rna, settings)
 317            # Callable (function) descriptions must handle their translations themselves.
 318            if tool.description and not callable(tool.description):
 319                process_msg(msgs, default_context, tool.description, msgsrc, reports, check_ctxt_rna_tip, settings)
 320
 321        for tool_context, tools_defs in cls.tools_all():
 322            for tools_group in tools_defs:
 323                if tools_group is None:
 324                    continue
 325                elif isinstance(tools_group, tuple) and not isinstance(tools_group, ToolDef):
 326                    for tool in tools_group:
 327                        process_tooldef(tool_context, tool)
 328                else:
 329                    process_tooldef(tool_context, tools_group)
 330
 331    blacklist_rna_class = class_blacklist()
 332
 333    def walk_class(cls):
 334        if not hasattr(cls, 'bl_rna'):
 335            reports["rna_structs_skipped"].append(cls)
 336            return
 337        bl_rna = cls.bl_rna
 338        msgsrc = "bpy.types." + bl_rna.identifier # CM3D2 Converter: this does crazy things like change the name of our operators!
 339        #msgsrc = "bpy.types." + cls.__name__
 340        # Catch operators and redirect through bpy.ops
 341        if '_OT_' in bl_rna.identifier:
 342            module, operator = cls.bl_idname.split('.')
 343            instance = getattr(getattr(bpy.ops, module), operator)
 344            cls = instance.get_rna_type().__class__
 345            bl_rna = cls.bl_rna
 346        #print(cls, msgsrc)
 347        msgctxt = bl_rna.translation_context or default_context
 348
 349        if bl_rna.name and (bl_rna.name != bl_rna.identifier or msgctxt != default_context):
 350            process_msg(msgs, msgctxt, bl_rna.name, msgsrc, reports, check_ctxt_rna, settings)
 351        
 352        if hasattr(bl_rna, 'bl_description'):
 353            process_msg(msgs, default_context, bl_rna.bl_description, msgsrc, reports, check_ctxt_rna_tip, settings)
 354        if bl_rna.description:
 355            process_msg(msgs, default_context, bl_rna.description, msgsrc, reports, check_ctxt_rna_tip, settings)
 356        elif cls.__doc__:  # XXX Some classes (like KeyingSetInfo subclasses) have void description... :(
 357            process_msg(msgs, default_context, cls.__doc__, msgsrc, reports, check_ctxt_rna_tip, settings)
 358
 359        # Panels' "tabs" system.
 360        if hasattr(bl_rna, 'bl_category') and bl_rna.bl_category:
 361            process_msg(msgs, default_context, bl_rna.bl_category, msgsrc, reports, check_ctxt_rna, settings)
 362
 363        if hasattr(bl_rna, 'bl_label') and bl_rna.bl_label:
 364            process_msg(msgs, msgctxt, bl_rna.bl_label, msgsrc, reports, check_ctxt_rna, settings)
 365
 366        # Tools Panels definitions.
 367        if hasattr(bl_rna, 'tools_all') and bl_rna.tools_all:
 368            walk_tools_definitions(cls)
 369
 370        walk_properties(cls)
 371
 372    def walk_keymap_hierarchy(hier, msgsrc_prev):
 373        km_i18n_context = bpy.app.translations.contexts.id_windowmanager
 374        for lvl in hier:
 375            msgsrc = msgsrc_prev + "." + lvl[1]
 376            if isinstance(lvl[0], str):  # Can be a function too, now, with tool system...
 377                process_msg(msgs, km_i18n_context, lvl[0], msgsrc, reports, None, settings)
 378            if lvl[3]:
 379                walk_keymap_hierarchy(lvl[3], msgsrc)
 380
 381    # Dump Messages
 382    operator_categories = {}
 383
 384    def process_cls_list(cls_list):
 385        if not cls_list:
 386            return
 387
 388        def full_class_id(cls):
 389            """Gives us 'ID.Light.AreaLight' which is best for sorting."""
 390            # Always the same issue, some classes listed in blacklist should actually no more exist (they have been
 391            # unregistered), but are still listed by __subclasses__() calls... :/
 392            if cls in blacklist_rna_class:
 393                return cls.__name__
 394            cls_id = ""
 395            bl_rna = None
 396            if hasattr(cls, 'bl_rna'):
 397                bl_rna = cls.bl_rna
 398            while bl_rna:
 399                cls_id = bl_rna.identifier + "." + cls_id
 400                bl_rna = bl_rna.base
 401            return cls_id
 402
 403        def operator_category(cls):
 404            """Extract operators' categories, as displayed in 'search' space menu."""
 405            # NOTE: keep in sync with C code in ui_searchbox_region_draw_cb__operator().
 406            if issubclass(cls, bpy.types.OperatorProperties) and "_OT_" in cls.__name__:
 407                cat_id = cls.__name__.split("_OT_")[0]
 408                if cat_id not in operator_categories:
 409                    cat_str = cat_id.capitalize() + ":"
 410                    operator_categories[cat_id] = cat_str
 411
 412        if verbose:
 413            print(cls_list)
 414        cls_list.sort(key=full_class_id)
 415        for cls in cls_list:
 416            if verbose:
 417                print(cls)
 418            reports["rna_structs"].append(cls)
 419            # Ignore those Operator sub-classes (anyway, will get the same from OperatorProperties sub-classes!)...
 420            if (cls in blacklist_rna_class): #or issubclass(cls, bpy.types.Operator):
 421                reports["rna_structs_skipped"].append(cls)
 422            else:
 423                operator_category(cls)
 424                walk_class(cls)
 425            # Recursively process subclasses.
 426            process_cls_list(cls.__subclasses__())
 427
 428    # Parse everything (recursively parsing from bpy_struct "class"...).
 429    process_cls_list(class_list)
 430
 431    # Finalize generated 'operator categories' messages.
 432    for cat_str in operator_categories.values():
 433        process_msg(msgs, bpy.app.translations.contexts.operator_default, cat_str, "Generated operator category",
 434                    reports, check_ctxt_rna, settings)
 435
 436    # And parse keymaps!
 437    from bl_keymap_utils import keymap_hierarchy
 438    walk_keymap_hierarchy(keymap_hierarchy.generate(), "KM_HIERARCHY")
 439
 440
 441##### Python source code #####
 442def dump_py_messages_from_files(msgs, reports, files, settings):
 443    """
 444    Dump text inlined in the python files given, e.g. 'My Name' in:
 445        layout.prop("someprop", text="My Name")
 446    """
 447    import ast
 448
 449    bpy_struct = bpy.types.ID.__base__
 450    i18n_contexts = bpy.app.translations.contexts
 451
 452    root_paths = tuple(bpy.utils.resource_path(t) for t in ('USER', 'LOCAL', 'SYSTEM'))
 453
 454    def make_rel(path):
 455        for rp in root_paths:
 456            if path.startswith(rp):
 457                try:  # can't always find the relative path (between drive letters on windows)
 458                    return os.path.relpath(path, rp)
 459                except ValueError:
 460                    return path
 461        # Use binary's dir as fallback...
 462        try:  # can't always find the relative path (between drive letters on windows)
 463            return os.path.relpath(path, os.path.dirname(bpy.app.binary_path))
 464        except ValueError:
 465            return path
 466
 467    # Helper function
 468    def extract_strings_ex(node, is_split=False):
 469        """
 470        Recursively get strings, needed in case we have "Blah" + "Blah", passed as an argument in that case it won't
 471        evaluate to a string. However, break on some kind of stopper nodes, like e.g. Subscript.
 472        """
 473        if type(node) == ast.Str:
 474            eval_str = ast.literal_eval(node)
 475            if eval_str:
 476                yield (is_split, eval_str, (node,))
 477        else:
 478            is_split = (type(node) in separate_nodes)
 479            for nd in ast.iter_child_nodes(node):
 480                if type(nd) not in stopper_nodes:
 481                    yield from extract_strings_ex(nd, is_split=is_split)
 482
 483    def _extract_string_merge(estr_ls, nds_ls):
 484        return "".join(s for s in estr_ls if s is not None), tuple(n for n in nds_ls if n is not None)
 485
 486    def extract_strings(node):
 487        estr_ls = []
 488        nds_ls = []
 489        for is_split, estr, nds in extract_strings_ex(node):
 490            estr_ls.append(estr)
 491            nds_ls.extend(nds)
 492        ret = _extract_string_merge(estr_ls, nds_ls)
 493        return ret
 494
 495    def extract_strings_split(node):
 496        """
 497        Returns a list args as returned by 'extract_strings()', but split into groups based on separate_nodes, this way
 498        expressions like ("A" if test else "B") won't be merged but "A" + "B" will.
 499        """
 500        estr_ls = []
 501        nds_ls = []
 502        bag = []
 503        for is_split, estr, nds in extract_strings_ex(node):
 504            if is_split:
 505                bag.append((estr_ls, nds_ls))
 506                estr_ls = []
 507                nds_ls = []
 508
 509            estr_ls.append(estr)
 510            nds_ls.extend(nds)
 511
 512        bag.append((estr_ls, nds_ls))
 513
 514        return [_extract_string_merge(estr_ls, nds_ls) for estr_ls, nds_ls in bag]
 515
 516    i18n_ctxt_ids = {v for v in bpy.app.translations.contexts_C_to_py.values()}
 517
 518    def _ctxt_to_ctxt(node):
 519        # We must try, to some extend, to get contexts from vars instead of only literal strings...
 520        ctxt = extract_strings(node)[0]
 521        if ctxt:
 522            return ctxt
 523        # Basically, we search for attributes matching py context names, for now.
 524        # So non-literal contexts should be used that way:
 525        #     i18n_ctxt = bpy.app.translations.contexts
 526        #     foobar(text="Foo", text_ctxt=i18n_ctxt.id_object)
 527        if type(node) == ast.Attribute:
 528            if node.attr in i18n_ctxt_ids:
 529                #print(node, node.attr, getattr(i18n_contexts, node.attr))
 530                return getattr(i18n_contexts, node.attr)
 531        return i18n_contexts.default
 532
 533    def _op_to_ctxt(node):
 534        # Some smart coders like things like:
 535        #    >>> row.operator("preferences.addon_disable" if is_enabled else "preferences.addon_enable", ...)
 536        # We only take first arg into account here!
 537        bag = extract_strings_split(node)
 538        opname, _ = bag[0]
 539        if not opname:
 540            return i18n_contexts.default
 541        op = bpy.ops
 542        for n in opname.split('.'):
 543            op = getattr(op, n)
 544        try:
 545            return op.get_rna_type().translation_context
 546        except Exception as e:
 547            default_op_context = i18n_contexts.operator_default
 548            print("ERROR: ", str(e))
 549            print("       Assuming default operator context '{}'".format(default_op_context))
 550            return default_op_context
 551
 552    # Gather function names.
 553    # In addition of UI func, also parse pgettext ones...
 554    # Tuples of (module name, (short names, ...)).
 555    pgettext_variants = (
 556        ("pgettext",       ("_"     , "f_"      , )),
 557        ("pgettext_iface", ("iface_", "f_iface_", )),
 558        ("pgettext_tip",   ("tip_"  , "f_tip_"  , )),
 559        ("pgettext_data",  ("data_" , "f_data_" , )),
 560    )
 561    pgettext_variants_args = {"msgid": (0, {"msgctxt": 1})}
 562
 563    # key: msgid keywords.
 564    # val: tuples of ((keywords,), context_getter_func) to get a context for that msgid.
 565    #      Note: order is important, first one wins!
 566    translate_kw = {
 567        "text": (
 568            (("text_ctxt",), _ctxt_to_ctxt),
 569            (("operator",), _op_to_ctxt),
 570        ),
 571        "msgid": (
 572            (("msgctxt",), _ctxt_to_ctxt),
 573        ),
 574        "message": (),
 575        "report_message": (), # CM3D2 Converter : for report_cancel()
 576    }
 577
 578    context_kw_set = {}
 579    for k, ctxts in translate_kw.items():
 580        s = set()
 581        for c, _ in ctxts:
 582            s |= set(c)
 583        context_kw_set[k] = s
 584
 585    # {func_id: {msgid: (arg_pos,
 586    #                    {msgctxt: arg_pos,
 587    #                     ...
 588    #                    }
 589    #                   ),
 590    #            ...
 591    #           },
 592    #  ...
 593    # }
 594    func_translate_args = {}
 595
 596    # First, functions from UILayout
 597    # First loop is for msgid args, second one is for msgctxt args.
 598    for func_id, func in bpy.types.UILayout.bl_rna.functions.items():
 599        # check it has one or more arguments as defined in translate_kw
 600        for arg_pos, (arg_kw, arg) in enumerate(func.parameters.items()):
 601            if ((arg_kw in translate_kw) and (not arg.is_output) and (arg.type == 'STRING')):
 602                func_translate_args.setdefault(func_id, {})[arg_kw] = (arg_pos, {})
 603    for func_id, func in bpy.types.UILayout.bl_rna.functions.items():
 604        if func_id not in func_translate_args:
 605            continue
 606        for arg_pos, (arg_kw, arg) in enumerate(func.parameters.items()):
 607            if (not arg.is_output) and (arg.type == 'STRING'):
 608                for msgid, msgctxts in context_kw_set.items():
 609                    if arg_kw in msgctxts:
 610                        func_translate_args[func_id][msgid][1][arg_kw] = arg_pos
 611    # The report() func of operators.
 612    for func_id, func in bpy.types.Operator.bl_rna.functions.items():
 613        # check it has one or more arguments as defined in translate_kw
 614        for arg_pos, (arg_kw, arg) in enumerate(func.parameters.items()):
 615            if ((arg_kw in translate_kw) and (not arg.is_output) and (arg.type == 'STRING')):
 616                func_translate_args.setdefault(func_id, {})[arg_kw] = (arg_pos, {})
 617    # We manually add funcs from bpy.app.translations
 618    for func_id, func_ids in pgettext_variants:
 619        func_translate_args[func_id] = pgettext_variants_args
 620        for func_id in func_ids:
 621            func_translate_args[func_id] = pgettext_variants_args
 622    
 623    # CM3D2 Converter : manually add report_cancel()
 624    # XXX Consider removing the use of report_cancel() entierly in model/anim import/export
 625    func_translate_args.setdefault('report_cancel', {})['report_message'] = (1, {})
 626
 627    # Break recursive nodes look up on some kind of nodes.
 628    # E.g. we don't want to get strings inside subscripts (blah["foo"])!
 629    #      we don't want to get strings from comparisons (foo.type == 'BAR').
 630    stopper_nodes = {ast.Subscript, ast.Compare}
 631    # Consider strings separate: ("a" if test else "b")
 632    separate_nodes = {ast.IfExp}
 633    
 634    check_ctxt_py = None
 635    if reports["check_ctxt"]:
 636        check_ctxt = reports["check_ctxt"]
 637        check_ctxt_py = {
 638            "py_in_rna": (check_ctxt.get("py_in_rna"), set(msgs.keys())),
 639            "multi_lines": check_ctxt.get("multi_lines"),
 640            "not_capitalized": check_ctxt.get("not_capitalized"),
 641            "end_point": check_ctxt.get("end_point"),
 642            "spell_checker": check_ctxt.get("spell_checker"),
 643            "spell_errors": check_ctxt.get("spell_errors"),
 644        }
 645
 646    for fp in files:
 647        with open(fp, 'r', encoding="utf8") as filedata:
 648            root_node = ast.parse(filedata.read(), fp, 'exec')
 649
 650        fp_rel = make_rel(fp)
 651
 652        for node in ast.walk(root_node):
 653            if type(node) == ast.Call:
 654                # print("found function at")
 655                # print("%s:%d" % (fp, node.lineno))
 656
 657                # We can't skip such situations! from blah import foo\nfoo("bar") would also be an ast.Name func!
 658                if type(node.func) == ast.Name:
 659                    func_id = node.func.id
 660                elif hasattr(node.func, "attr"):
 661                    func_id = node.func.attr
 662                # Ugly things like getattr(self, con.type)(context, box, con)
 663                else:
 664                    continue
 665
 666                func_args = func_translate_args.get(func_id, {})
 667
 668                # First try to get i18n contexts, for every possible msgid id.
 669                msgctxts = dict.fromkeys(func_args.keys(), "")
 670                for msgid, (_, context_args) in func_args.items():
 671                    context_elements = {}
 672                    for arg_kw, arg_pos in context_args.items():
 673                        if arg_pos < len(node.args):
 674                            context_elements[arg_kw] = node.args[arg_pos]
 675                        else:
 676                            for kw in node.keywords:
 677                                if kw.arg == arg_kw:
 678                                    context_elements[arg_kw] = kw.value
 679                                    break
 680                    # print(context_elements)
 681                    for kws, proc in translate_kw[msgid]:
 682                        if set(kws) <= context_elements.keys():
 683                            args = tuple(context_elements[k] for k in kws)
 684                            #print("running ", proc, " with ", args)
 685                            ctxt = proc(*args)
 686                            if ctxt:
 687                                msgctxts[msgid] = ctxt
 688                                break
 689
 690                # print(translate_args)
 691                # do nothing if not found
 692                for arg_kw, (arg_pos, _) in func_args.items():
 693                    msgctxt = msgctxts[arg_kw]
 694                    estr_lst = [(None, ())]
 695                    if arg_pos < len(node.args):
 696                        estr_lst = extract_strings_split(node.args[arg_pos])
 697                        #print(estr, nds)
 698                    else:
 699                        for kw in node.keywords:
 700                            if kw.arg == arg_kw:
 701                                estr_lst = extract_strings_split(kw.value)
 702                                break
 703                        #print(estr, nds)
 704                    for estr, nds in estr_lst:
 705                        if estr:
 706                            if nds:
 707                                msgsrc = "{}:{}".format(fp_rel, sorted({nd.lineno for nd in nds})[0])
 708                            else:
 709                                msgsrc = "{}:???".format(fp_rel)
 710                            process_msg(msgs, msgctxt, estr, msgsrc, reports, check_ctxt_py, settings)
 711                            reports["py_messages"].append((msgctxt, estr, msgsrc))
 712
 713
 714def dump_py_messages(msgs, reports, addons, settings, addons_only=False):
 715    def _get_files(path):
 716        if not os.path.exists(path):
 717            return []
 718        if os.path.isdir(path):
 719            return [os.path.join(dpath, fn) for dpath, _, fnames in os.walk(path) for fn in fnames
 720                    if not fn.startswith("_") and fn.endswith(".py")]
 721        return [path]
 722
 723    files = []
 724    if not addons_only:
 725        for path in settings.CUSTOM_PY_UI_FILES:
 726            for root in (bpy.utils.resource_path(t) for t in ('USER', 'LOCAL', 'SYSTEM')):
 727                files += _get_files(os.path.join(root, path))
 728
 729    # Add all given addons.
 730    for mod in addons:
 731        fn = mod.__file__
 732        if os.path.basename(fn) == "__init__.py":
 733            files += _get_files(os.path.dirname(fn))
 734        else:
 735            files.append(fn)
 736
 737    dump_py_messages_from_files(msgs, reports, sorted(files), settings)
 738
 739
 740##### C source code #####
 741def dump_src_messages(msgs, reports, settings):
 742    def get_contexts():
 743        """Return a mapping {C_CTXT_NAME: ctxt_value}."""
 744        return {k: getattr(bpy.app.translations.contexts, n) for k, n in bpy.app.translations.contexts_C_to_py.items()}
 745
 746    contexts = get_contexts()
 747
 748    # Build regexes to extract messages (with optional contexts) from C source.
 749    pygettexts = tuple(re.compile(r).search for r in settings.PYGETTEXT_KEYWORDS)
 750
 751    _clean_str = re.compile(settings.str_clean_re).finditer
 752
 753    def clean_str(s):
 754        return "".join(m.group("clean") for m in _clean_str(s))
 755
 756    def dump_src_file(path, rel_path, msgs, reports, settings):
 757        def process_entry(_msgctxt, _msgid):
 758            # Context.
 759            msgctxt = settings.DEFAULT_CONTEXT
 760            if _msgctxt:
 761                if _msgctxt in contexts:
 762                    msgctxt = contexts[_msgctxt]
 763                elif '"' in _msgctxt or "'" in _msgctxt:
 764                    msgctxt = clean_str(_msgctxt)
 765                else:
 766                    print("WARNING: raw context “{}” couldn’t be resolved!".format(_msgctxt))
 767            # Message.
 768            msgid = ""
 769            if _msgid:
 770                if '"' in _msgid or "'" in _msgid:
 771                    msgid = clean_str(_msgid)
 772                else:
 773                    print("WARNING: raw message “{}” couldn’t be resolved!".format(_msgid))
 774            return msgctxt, msgid
 775
 776        check_ctxt_src = None
 777        if reports["check_ctxt"]:
 778            check_ctxt = reports["check_ctxt"]
 779            check_ctxt_src = {
 780                "multi_lines": check_ctxt.get("multi_lines"),
 781                "not_capitalized": check_ctxt.get("not_capitalized"),
 782                "end_point": check_ctxt.get("end_point"),
 783                "spell_checker": check_ctxt.get("spell_checker"),
 784                "spell_errors": check_ctxt.get("spell_errors"),
 785            }
 786
 787        data = ""
 788        with open(path) as f:
 789            data = f.read()
 790        for srch in pygettexts:
 791            m = srch(data)
 792            line = pos = 0
 793            while m:
 794                d = m.groupdict()
 795                # Line.
 796                line += data[pos:m.start()].count('\n')
 797                msgsrc = rel_path + ":" + str(line)
 798                _msgid = d.get("msg_raw")
 799                # First, try the "multi-contexts" stuff!
 800                _msgctxts = tuple(d.get("ctxt_raw{}".format(i)) for i in range(settings.PYGETTEXT_MAX_MULTI_CTXT))
 801                if _msgctxts[0]:
 802                    for _msgctxt in _msgctxts:
 803                        if not _msgctxt:
 804                            break
 805                        msgctxt, msgid = process_entry(_msgctxt, _msgid)
 806                        process_msg(msgs, msgctxt, msgid, msgsrc, reports, check_ctxt_src, settings)
 807                        reports["src_messages"].append((msgctxt, msgid, msgsrc))
 808                else:
 809                    _msgctxt = d.get("ctxt_raw")
 810                    msgctxt, msgid = process_entry(_msgctxt, _msgid)
 811                    process_msg(msgs, msgctxt, msgid, msgsrc, reports, check_ctxt_src, settings)
 812                    reports["src_messages"].append((msgctxt, msgid, msgsrc))
 813
 814                pos = m.end()
 815                line += data[m.start():pos].count('\n')
 816                m = srch(data, pos)
 817
 818    forbidden = set()
 819    forced = set()
 820    if os.path.isfile(settings.SRC_POTFILES):
 821        with open(settings.SRC_POTFILES) as src:
 822            for l in src:
 823                if l[0] == '-':
 824                    forbidden.add(l[1:].rstrip('\n'))
 825                elif l[0] != '#':
 826                    forced.add(l.rstrip('\n'))
 827    for root, dirs, files in os.walk(settings.POTFILES_SOURCE_DIR):
 828        if "/.svn" in root:
 829            continue
 830        for fname in files:
 831            if os.path.splitext(fname)[1] not in settings.PYGETTEXT_ALLOWED_EXTS:
 832                continue
 833            path = os.path.join(root, fname)
 834            try:  # can't always find the relative path (between drive letters on windows)
 835                rel_path = os.path.relpath(path, settings.SOURCE_DIR)
 836            except ValueError:
 837                rel_path = path
 838            if rel_path in forbidden:
 839                continue
 840            elif rel_path not in forced:
 841                forced.add(rel_path)
 842    for rel_path in sorted(forced):
 843        path = os.path.join(settings.SOURCE_DIR, rel_path)
 844        if os.path.exists(path):
 845            dump_src_file(path, rel_path, msgs, reports, settings)
 846
 847
 848##### Main functions! #####
 849def dump_messages(do_messages, do_checks, settings):
 850    bl_ver = "Blender " + bpy.app.version_string
 851    bl_hash = bpy.app.build_hash
 852    bl_date = datetime.datetime.strptime(bpy.app.build_date.decode() + "T" + bpy.app.build_time.decode(),
 853                                         "%Y-%m-%dT%H:%M:%S")
 854    pot = utils.I18nMessages.gen_empty_messages(settings.PARSER_TEMPLATE_ID, bl_ver, bl_hash, bl_date, bl_date.year,
 855                                                settings=settings)
 856    msgs = pot.msgs
 857
 858    # Enable all wanted addons.
 859    # For now, enable all official addons, before extracting msgids.
 860    addons = utils.enable_addons(support={"OFFICIAL"})
 861    # Note this is not needed if we have been started with factory settings, but just in case...
 862    # XXX This is not working well, spent a whole day trying to understand *why* we still have references of
 863    #     those removed calsses in things like `bpy.types.OperatorProperties.__subclasses__()`
 864    #     (could not even reproduce it from regular py console in Blender with UI...).
 865    #     For some reasons, cleanup does not happen properly, *and* we have no way to tell which class is valid
 866    #     and which has been unregistered. So for now, just go for the dirty, easy way: do not disable add-ons. :(
 867    # ~ utils.enable_addons(support={"COMMUNITY", "TESTING"}, disable=True)
 868
 869    reports = _gen_reports(_gen_check_ctxt(settings) if do_checks else None)
 870
 871    # Get strings from RNA.
 872    dump_rna_messages(msgs, reports, settings)
 873
 874    # Get strings from UI layout definitions text="..." args.
 875    dump_py_messages(msgs, reports, addons, settings)
 876
 877    # Get strings from C source code.
 878    dump_src_messages(msgs, reports, settings)
 879
 880    # Get strings from addons' categories.
 881    for uid, label, tip in bpy.types.WindowManager.addon_filter[1]['items'](bpy.context.window_manager, bpy.context):
 882        process_msg(msgs, settings.DEFAULT_CONTEXT, label, "Add-ons' categories", reports, None, settings)
 883        if tip:
 884            process_msg(msgs, settings.DEFAULT_CONTEXT, tip, "Add-ons' categories", reports, None, settings)
 885
 886    # Get strings specific to translations' menu.
 887    for lng in settings.LANGUAGES:
 888        process_msg(msgs, settings.DEFAULT_CONTEXT, lng[1], "Languages’ labels from bl_i18n_utils/settings.py",
 889                    reports, None, settings)
 890    for cat in settings.LANGUAGES_CATEGORIES:
 891        process_msg(msgs, settings.DEFAULT_CONTEXT, cat[1],
 892                    "Language categories’ labels from bl_i18n_utils/settings.py", reports, None, settings)
 893
 894    # pot.check()
 895    pot.unescape()  # Strings gathered in py/C source code may contain escaped chars...
 896    print_info(reports, pot)
 897    # pot.check()
 898
 899    if do_messages:
 900        print("Writing messages…")
 901        pot.write('PO', settings.FILE_NAME_POT)
 902
 903    print("Finished extracting UI messages!")
 904
 905    return pot  # Not used currently, but may be useful later (and to be consistent with dump_addon_messages!).
 906
 907
 908def dump_addon_messages(module_name, do_checks, settings):
 909    import addon_utils
 910
 911    # Get current addon state (loaded or not):
 912    was_loaded = addon_utils.check(module_name)[1]
 913
 914    # Enable our addon.
 915    addon = utils.enable_addons(addons={module_name})[0]
 916
 917    addon_info = addon_utils.module_bl_info(addon)
 918    ver = addon_info["name"] + " " + ".".join(str(v) for v in addon_info["version"])
 919    rev = 0
 920    date = datetime.datetime.now()
 921    pot = utils.I18nMessages.gen_empty_messages(settings.PARSER_TEMPLATE_ID, ver, rev, date, date.year,
 922                                                settings=settings)
 923    msgs = pot.msgs
 924
 925    minus_pot = utils.I18nMessages.gen_empty_messages(settings.PARSER_TEMPLATE_ID, ver, rev, date, date.year,
 926                                                      settings=settings)
 927    minus_msgs = minus_pot.msgs
 928
 929    check_ctxt = _gen_check_ctxt(settings) if do_checks else None
 930    minus_check_ctxt = _gen_check_ctxt(settings) if do_checks else None
 931
 932    # Get strings from RNA, our addon being enabled.
 933    print("A")
 934    reports = _gen_reports(check_ctxt)
 935    print("B")
 936    dump_rna_messages(msgs, reports, settings)
 937    print("C")
 938
 939    # Now disable our addon, and rescan RNA.
 940    utils.enable_addons(addons={module_name}, disable=True)
 941    print("D")
 942    reports["check_ctxt"] = minus_check_ctxt
 943    print("E")
 944    dump_rna_messages(minus_msgs, reports, settings)
 945    print("F")
 946
 947    # Restore previous state if needed!
 948    if was_loaded:
 949        utils.enable_addons(addons={module_name})
 950
 951    # and make the diff!
 952    for key in minus_msgs:
 953        if key != settings.PO_HEADER_KEY:
 954            if key in msgs.keys():
 955                del msgs[key]
 956
 957    if check_ctxt:
 958        _diff_check_ctxt(check_ctxt, minus_check_ctxt)
 959
 960    # and we are done with those!
 961    del minus_pot
 962    del minus_msgs
 963    del minus_check_ctxt
 964
 965    # get strings from UI layout definitions text="..." args
 966    reports["check_ctxt"] = check_ctxt
 967    dump_py_messages(msgs, reports, {addon}, settings, addons_only=True)
 968
 969    pot.unescape()  # Strings gathered in py/C source code may contain escaped chars...
 970    print_info(reports, pot)
 971
 972    print("Finished extracting UI messages!")
 973
 974    return pot
 975
 976
 977def main():
 978    try:
 979        import bpy
 980    except ImportError:
 981        print("This script must run from inside blender")
 982        return
 983
 984    import sys
 985    import argparse
 986
 987    # Get rid of Blender args!
 988    argv = sys.argv[sys.argv.index("--") + 1:] if "--" in sys.argv else []
 989
 990    parser = argparse.ArgumentParser(description="Process UI messages from inside Blender.")
 991    parser.add_argument('-c', '--no_checks', default=True, action="store_false", help="No checks over UI messages.")
 992    parser.add_argument('-m', '--no_messages', default=True, action="store_false", help="No export of UI messages.")
 993    parser.add_argument('-o', '--output', default=None, help="Output POT file path.")
 994    parser.add_argument('-s', '--settings', default=None,
 995                        help="Override (some) default settings. Either a JSon file name, or a JSon string.")
 996    args = parser.parse_args(argv)
 997
 998    settings = settings_i18n.I18nSettings()
 999    settings.load(args.settings)
1000
1001    if args.output:
1002        settings.FILE_NAME_POT = args.output
1003
1004    dump_messages(do_messages=args.no_messages, do_checks=args.no_checks, settings=settings)
1005
1006
1007if __name__ == "__main__":
1008    print("\n\n *** Running {} *** \n".format(__file__))
1009    main()