Merge pull request #128 from Hoikas/python-pak

Python Packing

installer/Installer.nsi

@@ -211,6 +211,14 @@ SectionGroup /e "Korman"
     SectionEnd
 SectionGroupEnd
 
+Section "Python 2.2"
+    SectionIn 1 2
+
+    SetOutPath    "$TEMP\Korman"
+    File "Files\x86\Python-2.2.3.exe"
+    ExecWait "$TEMP\Korman\Python-2.2.3.exe /S"
+SectionEnd
+
 Section #TheRemover
     WriteRegStr HKLM "Software\Korman" "" $INSTDIR
     WriteUninstaller "$INSTDIR\korman_uninstall.exe"

korman/__init__.py

@@ -14,6 +14,7 @@
 #    along with Korman.  If not, see <http://www.gnu.org/licenses/>.
 
 import bpy
+from . import addon_prefs
 from . import exporter, render
 from . import properties, ui
 from . import nodes
@@ -26,8 +27,7 @@ bl_info = {
     "location":    "File > Import-Export",
     "description": "Exporter for Cyan Worlds' Plasma Engine",
     "warning":     "beta",
-    "category":    "System",  # Eventually, we will hide some of the default
+    "category":    "System",
-                              # Blender panels (think materials)
 }
 
 

korman/addon_prefs.py

@@ -0,0 +1,148 @@
+#    This file is part of Korman.
+#
+#    Korman is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    Korman is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with Korman.  If not, see <http://www.gnu.org/licenses/>.
+
+import bpy
+from bpy.props import *
+from . import korlib
+
+game_versions = [("pvPrime", "Ages Beyond Myst (63.11)", "Targets the original Uru (Live) game"),
+                 ("pvPots", "Path of the Shell (63.12)", "Targets the most recent offline expansion pack"),
+                 ("pvMoul", "Myst Online: Uru Live (70)", "Targets the most recent online game")]
+
+class PlasmaGame(bpy.types.PropertyGroup):
+    name = StringProperty(name="Name",
+                          description="Name of the Plasma Game",
+                          options=set())
+    path = StringProperty(name="Path",
+                          description="Path to this Plasma Game",
+                          options=set())
+    version = EnumProperty(name="Version",
+                           description="Plasma version of this game",
+                           items=game_versions,
+                           options=set())
+
+
+class KormanAddonPreferences(bpy.types.AddonPreferences):
+    bl_idname = __package__
+
+    games = CollectionProperty(type=PlasmaGame)
+    active_game_index = IntProperty(options={"SKIP_SAVE"})
+
+    def _check_py22_exe(self, context):
+        if self._ensure_abspath((2, 2)):
+            self._check_python((2, 2))
+    def _check_py23_exe(self, context):
+        if self._ensure_abspath((2, 3)):
+            self._check_python((2, 3))
+    def _check_py27_exe(self, context):
+        if self._ensure_abspath((2, 7)):
+            self._check_python((2, 7))
+
+    python22_executable = StringProperty(name="Python 2.2",
+                                         description="Path to the Python 2.2 executable",
+                                         options=set(),
+                                         subtype="FILE_PATH",
+                                         update=_check_py22_exe)
+    python23_executable = StringProperty(name="Python 2.3",
+                                         description="Path to the Python 2.3 executable",
+                                         options=set(),
+                                         subtype="FILE_PATH",
+                                         update=_check_py23_exe)
+    python27_executable = StringProperty(name="Python 2.7",
+                                         description="Path to the Python 2.7 executable",
+                                         options=set(),
+                                         subtype="FILE_PATH",
+                                         update=_check_py27_exe)
+
+    def _validate_py_exes(self):
+        if not self.is_property_set("python22_valid"):
+            self._check_python((2, 2))
+        if not self.is_property_set("python23_valid"):
+            self._check_python((2, 3))
+        if not self.is_property_set("python27_valid"):
+            self._check_python((2, 7))
+        return True
+
+    # Internal error states
+    python22_valid = BoolProperty(options={"HIDDEN", "SKIP_SAVE"})
+    python23_valid = BoolProperty(options={"HIDDEN", "SKIP_SAVE"})
+    python27_valid = BoolProperty(options={"HIDDEN", "SKIP_SAVE"})
+    python_validated = BoolProperty(get=_validate_py_exes, options={"HIDDEN", "SKIP_SAVE"})
+
+    def _check_python(self, py_version):
+        py_exe = getattr(self, "python{}{}_executable".format(*py_version))
+        if py_exe:
+            valid = korlib.verify_python(py_version, py_exe)
+        else:
+            valid = True
+        setattr(self, "python{}{}_valid".format(*py_version), valid)
+
+    def _ensure_abspath(self, py_version):
+        attr = "python{}{}_executable".format(*py_version)
+        path = getattr(self, attr)
+        if path.startswith("//"):
+            setattr(self, attr, bpy.path.abspath(path))
+            return False
+        return True
+
+    def draw(self, context):
+        layout = self.layout
+        split = layout.split()
+        main_col = split.column()
+
+        main_col.label("Plasma Games:")
+        row = main_col.row()
+        row.template_list("PlasmaGameListRW", "games", self, "games", self,
+                          "active_game_index", rows=3)
+        col = row.column(align=True)
+        col.operator("world.plasma_game_add", icon="ZOOMIN", text="")
+        col.operator("world.plasma_game_remove", icon="ZOOMOUT", text="")
+        col.operator("world.plasma_game_convert", icon="IMPORT", text="")
+
+        # Game Properties
+        active_game_index = self.active_game_index
+        if bool(self.games) and active_game_index < len(self.games):
+            active_game = self.games[active_game_index]
+
+            col = split.column()
+            col.label("Game Configuration:")
+            box = col.box().column()
+
+            box.prop(active_game, "path", emboss=False)
+            box.prop(active_game, "version")
+            box.separator()
+
+            row = box.row(align=True)
+            op = row.operator("world.plasma_game_add", icon="FILE_FOLDER", text="Change Path")
+            op.filepath = active_game.path
+            op.game_index = active_game_index
+
+        # Python Installs
+        assert self.python_validated
+        col = layout.column()
+        col.label("Python Executables:")
+        col.alert = not self.python22_valid
+        col.prop(self, "python22_executable")
+        col.alert = not self.python23_valid
+        col.prop(self, "python23_executable")
+        col.alert = not self.python27_valid
+        col.prop(self, "python27_executable")
+
+    @classmethod
+    def register(cls):
+        # Register the old-timey per-world Plasma Games for use in the conversion
+        # operator. What fun. I guess....
+        from .properties.prop_world import PlasmaGames
+        PlasmaGames.games = CollectionProperty(type=PlasmaGame)

korman/exporter/__init__.py

@@ -18,4 +18,5 @@ from PyHSPlasma import *
 
 from .convert import *
 from .explosions import *
+from .python import *
 from . import utils

korman/exporter/convert.py

@@ -27,9 +27,9 @@ from . import image
 from . import logger
 from . import manager
 from . import mesh
+from . import outfile
 from . import physics
 from . import rtlight
-from . import sumfile
 from . import utils
 
 class Exporter:
@@ -40,10 +40,6 @@ class Exporter:
         self.node_trees_exported = set()
         self.want_node_trees = {}
 
-    @property
-    def age_name(self):
-        return Path(self._op.filepath).stem
-
     def run(self):
         log = logger.ExportVerboseLogger if self._op.verbose else logger.ExportProgressLogger
         with ConsoleToggler(self._op.show_console), log(self._op.filepath) as self.report:
@@ -53,7 +49,7 @@ class Exporter:
             self.physics = physics.PhysicsConverter(self)
             self.light = rtlight.LightConverter(self)
             self.animation = animation.AnimationConverter(self)
-            self.sumfile = sumfile.SumFile()
+            self.output = outfile.OutputFiles(self, self._op.filepath)
             self.camera = camera.CameraConverter(self)
             self.image = image.ImageCache(self)
 
@@ -66,6 +62,7 @@ class Exporter:
             self.report.progress_add_step("Exporting Scene Objects")
             self.report.progress_add_step("Exporting Logic Nodes")
             self.report.progress_add_step("Finalizing Plasma Logic")
+            self.report.progress_add_step("Handling Snakes")
             self.report.progress_add_step("Exporting Textures")
             self.report.progress_add_step("Composing Geometry")
             self.report.progress_add_step("Saving Age Files")
@@ -98,6 +95,9 @@ class Exporter:
                 #          processing that needs to inspect those objects
                 self._post_process_scene_objects()
 
+                # Step 3.3: Ensure any helper Python files are packed
+                self._pack_ancillary_python()
+
                 # Step 4: Finalize...
                 self.mesh.material.finalize()
                 self.mesh.finalize()
@@ -111,6 +111,11 @@ class Exporter:
                 self.report.progress_end()
                 self.report.save()
 
+                # Step 5.2: If any nonfatal errors were encountered during the export, we will
+                #           raise them here, now that everything is finished, to draw attention
+                #           to whatever the problem might be.
+                self.report.raise_errors()
+
     def _bake_static_lighting(self):
         lighting_method = self._op.lighting_method
         if lighting_method != "skip":
@@ -345,15 +350,45 @@ class Exporter:
                     proc(self, bl_obj, sceneobject)
             inc_progress()
 
+    def _pack_ancillary_python(self):
+        texts = bpy.data.texts
+        self.report.progress_advance()
+        self.report.progress_range = len(texts)
+        inc_progress = self.report.progress_increment
+
+        for i in texts:
+            if i.name.endswith(".py") and self.output.want_py_text(i):
+                self.output.add_python_code(i.name, text_id=i)
+            inc_progress()
+
     def _save_age(self):
         self.report.progress_advance()
-        self.mgr.save_age(Path(self._op.filepath))
+        self.report.msg("\nWriting Age data...")
-        self.image.save()
+
+        # If something bad happens in the final flush, it would be a shame to
+        # simply toss away the potentially freshly regenerated texture cache.
+        try:
+            self.mgr.save_age()
+            self.output.save()
+        finally:
+            self.image.save()
+
+    @property
+    def age_name(self):
+        return Path(self._op.filepath).stem
+
+    @property
+    def dat_only(self):
+        return self._op.dat_only
 
     @property
     def envmap_method(self):
         return bpy.context.scene.world.plasma_age.envmap_method
 
+    @property
+    def python_method(self):
+        return bpy.context.scene.world.plasma_age.python_method
+
     @property
     def texcache_path(self):
         age = bpy.context.scene.world.plasma_age

korman/exporter/explosions.py

@@ -13,6 +13,15 @@
 #    You should have received a copy of the GNU General Public License
 #    along with Korman.  If not, see <http://www.gnu.org/licenses/>.
 
+class NonfatalExportError(Exception):
+    def __init__(self, *args, **kwargs):
+        assert args
+        if len(args) > 1:
+            super(Exception, self).__init__(args[0].format(*args[1:], **kwargs))
+        else:
+            super(Exception, self).__init__(args[0])
+
+
 class ExportError(Exception):
     def __init__(self, value="Undefined Export Error"):
         super(Exception, self).__init__(value)

korman/exporter/logger.py

@@ -14,6 +14,7 @@
 #    along with Korman.  If not, see <http://www.gnu.org/licenses/>.
 
 from ..korlib import ConsoleToggler
+from .explosions import NonfatalExportError
 from pathlib import Path
 import threading
 import time
@@ -24,6 +25,7 @@ _MAX_TIME_UNTIL_ELIPSES = 2.0
 
 class _ExportLogger:
     def __init__(self, print_logs, age_path=None):
+        self._errors = []
         self._porting = []
         self._warnings = []
         self._age_path = Path(age_path) if age_path is not None else None
@@ -42,10 +44,23 @@ class _ExportLogger:
 
     def __exit__(self, type, value, traceback):
         if value is not None:
-            ConsoleToggler().keep_console = True
+            ConsoleToggler().keep_console = not isinstance(value, NonfatalExportError)
         self._file.close()
         return False
 
+    def error(self, *args, **kwargs):
+        assert args
+        indent = kwargs.get("indent", 0)
+        msg = "{}ERROR: {}".format("    " * indent, args[0])
+        if len(args) > 1:
+            msg = msg.format(*args[1:], **kwargs)
+        if self._file is not None:
+            self._file.writelines((msg, "\n"))
+        if self._print_logs:
+            print(msg)
+        cache = args[0] if len(args) == 1 else args[0].format(*args[1:])
+        self._errors.append(cache)
+
     def msg(self, *args, **kwargs):
         assert args
         indent = kwargs.get("indent", 0)
@@ -67,7 +82,8 @@ class _ExportLogger:
             self._file.writelines((msg, "\n"))
         if self._print_logs:
             print(msg)
-        self._porting.append(args[0])
+        cache = args[0] if len(args) == 1 else args[0].format(*args[1:])
+        self._porting.append(cache)
 
 
     def progress_add_step(self, name):
@@ -92,6 +108,14 @@ class _ExportLogger:
             self.msg("Exporting '{}'", self._age_path.name)
         self._time_start_overall = time.perf_counter()
 
+    def raise_errors(self):
+        num_errors = len(self._errors)
+        if num_errors == 1:
+            raise NonfatalExportError(self._errors[0])
+        elif num_errors:
+            raise NonfatalExportError("""{} errors were encountered during export. Check the export log for more details:
+                                         {}""", num_errors, self._file.name)
+
     def save(self):
         # TODO
         pass
@@ -106,7 +130,8 @@ class _ExportLogger:
             self._file.writelines((msg, "\n"))
         if self._print_logs:
             print(msg)
-        self._warnings.append(args[0])
+        cache = args[0] if len(args) == 1 else args[0].format(*args[1:])
+        self._warnings.append(cache)
 
 
 class ExportProgressLogger(_ExportLogger):

korman/exporter/manager.py

@@ -19,6 +19,8 @@ from PyHSPlasma import *
 import weakref
 
 from . import explosions
+from .. import korlib
+from ..plasma_magic import *
 
 # These objects have to be in the plSceneNode pool in order to be loaded...
 # NOTE: We are using Factory indices because I doubt all of these classes are implemented.
@@ -136,15 +138,19 @@ class ExportManager:
     def create_builtins(self, age, textures):
         # BuiltIn.prp
         if bpy.context.scene.world.plasma_age.age_sdl:
-            builtin = self.create_page(age, "BuiltIn", -2, True)
+            self._create_builtin_pages(age)
-            sdl = self.add_object(plSceneObject, name="AgeSDLHook", loc=builtin)
+            self._pack_agesdl_hook(age)
-            pfm = self.add_object(plPythonFileMod, name="VeryVerySpecialPythonFileMod", so=sdl)
-            pfm.filename = age
 
         # Textures.prp
         if textures:
             self.create_page(age, "Textures", -1, True)
 
+    def _create_builtin_pages(self, age):
+        builtin = self.create_page(age, "BuiltIn", -2, True)
+        sdl = self.add_object(plSceneObject, name="AgeSDLHook", loc=builtin)
+        pfm = self.add_object(plPythonFileMod, name="VeryVerySpecialPythonFileMod", so=sdl)
+        pfm.filename = age
+
     def create_page(self, age, name, id, builtin=False):
         location = plLocation(self.mgr.getVer())
         location.prefix = bpy.context.scene.world.plasma_age.seq_prefix
@@ -233,27 +239,46 @@ class ExportManager:
         else:
             return key.location
 
-    def save_age(self, path):
+    def _pack_agesdl_hook(self, age):
-        ageName = path.stem
+        get_text = bpy.data.texts.get
-        sumfile = self._exporter().sumfile
+        output = self._exporter().output
 
-        sumfile.append(path)
+        # AgeSDL Hook Python
-        self.mgr.WriteAge(str(path), self._age_info)
+        fixed_agename = korlib.replace_python2_identifier(age)
-        self._write_fni(path)
+        py_filename = "{}.py".format(fixed_agename)
-        self._write_pages(path)
+        age_py = get_text(py_filename, None)
+        if output.want_py_text(age_py):
+            py_code = age_py.as_string()
+        else:
+            py_code = very_very_special_python.format(age_name=fixed_agename).lstrip()
+        output.add_python_mod(py_filename, text_id=age_py, str_data=py_code)
+
+        # AgeSDL
+        sdl_filename = "{}.sdl".format(fixed_agename)
+        age_sdl = get_text(sdl_filename)
+        if age_sdl is not None:
+            sdl_code = None
+        else:
+            sdl_code = very_very_special_sdl.format(age_name=fixed_agename).lstrip()
+        output.add_sdl(sdl_filename, text_id=age_sdl, str_data=sdl_code)
 
-        if self.getVer() != pvMoul:
+    def save_age(self):
-            sumpath = path.with_suffix(".sum")
+        self._write_age()
-            sumfile.write(sumpath, self.getVer())
+        self._write_fni()
+        self._write_pages()
 
-    def _write_fni(self, path):
+    def _write_age(self):
-        if self.mgr.getVer() <= pvMoul:
+        f = "{}.age".format(self._age_info.name)
-            enc = plEncryptedStream.kEncXtea
+        output = self._exporter().output
-        else:
-            enc = plEncryptedStream.kEncAES
-        fname = path.with_suffix(".fni")
 
-        with plEncryptedStream(self.mgr.getVer()).open(str(fname), fmWrite, enc) as stream:
+        with output.generate_dat_file(f, enc=plEncryptedStream.kEncAuto) as stream:
+            self._age_info.writeToStream(stream)
+
+    def _write_fni(self):
+        f = "{}.fni".format(self._age_info.name)
+        output = self._exporter().output
+
+        with output.generate_dat_file(f, enc=plEncryptedStream.kEncAuto) as stream:
             fni = bpy.context.scene.world.plasma_fni
             stream.writeLine("Graphics.Renderer.SetClearColor {} {} {}".format(*fni.clear_color))
             if fni.fog_method != "none":
@@ -263,17 +288,14 @@ class ExportManager:
             elif fni.fog_method == "exp2":
                 stream.writeLine("Graphics.Renderer.Fog.SetDefExp2 {} {}".format(fni.fog_end, fni.fog_density))
             stream.writeLine("Graphics.Renderer.SetYon {}".format(fni.yon))
-        self._exporter().sumfile.append(fname)
 
-    def _write_pages(self, path):
+    def _write_pages(self):
+        age_name = self._age_info.name
+        output = self._exporter().output
         for loc in self._pages.values():
             page = self.mgr.FindPage(loc) # not cached because it's C++ owned
-            # I know that plAgeInfo has its own way of doing this, but we'd have
+            chapter = "_District_" if self.mgr.getVer() <= pvMoul else "_"
-            # to do some looping and stuff. This is easier.
+            f = "{}{}{}.prp".format(age_name, chapter, page.page)
-            if self.mgr.getVer() <= pvMoul:
+
-                chapter = "_District_"
+            with output.generate_dat_file(f) as stream:
-            else:
+                self.mgr.WritePage(stream, page)
-                chapter = "_"
-            f = path.with_name("{}{}{}".format(path.stem, chapter, page.page)).with_suffix(".prp")
-            self.mgr.WritePage(str(f), page)
-            self._exporter().sumfile.append(f)

korman/exporter/outfile.py

@@ -0,0 +1,429 @@
+#    This file is part of Korman.
+#
+#    Korman is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    Korman is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with Korman.  If not, see <http://www.gnu.org/licenses/>.
+
+from contextlib import contextmanager
+import enum
+from hashlib import md5
+from .. import korlib
+import locale
+import os
+from pathlib import Path
+from ..plasma_magic import plasma_python_glue
+from PyHSPlasma import *
+import shutil
+import time
+import weakref
+import zipfile
+
+_CHUNK_SIZE = 0xA00000
+_encoding = locale.getpreferredencoding(False)
+
+def _hashfile(filename, hasher, block=0xFFFF):
+    with open(str(filename), "rb") as handle:
+        h = hasher()
+        data = handle.read(block)
+        while data:
+            h.update(data)
+            data = handle.read(block)
+        return h.digest()
+
+@enum.unique
+class _FileType(enum.Enum):
+    generated_dat = 0
+    sfx = 1
+    sdl = 2
+    python_code = 3
+    generated_ancillary = 4
+
+
+class _OutputFile:
+    def __init__(self, **kwargs):
+        self.file_type = kwargs.get("file_type")
+        self.dirname = kwargs.get("dirname")
+        self.filename = kwargs.get("filename")
+        self.skip_hash = kwargs.get("skip_hash", False)
+        self.internal = kwargs.get("internal", False)
+        self.file_path = None
+        self.mod_time = None
+
+        if self.file_type in (_FileType.generated_dat, _FileType.generated_ancillary):
+            self.file_data = kwargs.get("file_data", None)
+            self.file_path = kwargs.get("file_path", None)
+            self.mod_time = Path(self.file_path).stat().st_mtime if self.file_path else None
+
+            # need either a data buffer OR a file path
+            assert bool(self.file_data) ^ bool(self.file_path)
+
+        if self.file_type == _FileType.sfx:
+            self.id_data = kwargs.get("id_data")
+            path = Path(self.id_data.filepath)
+            try:
+                if path.exists():
+                    self.file_path = str(path.resolve())
+                    self.mod_time = path.stat().st_mtime
+            except OSError:
+                pass
+
+            if self.id_data.packed_file is not None:
+                self.file_data = self.id_data.packed_file.data
+            else:
+                self.file_data = None
+
+        if self.file_type in (_FileType.sdl, _FileType.python_code):
+            self.id_data = kwargs.get("id_data")
+            self.file_data = kwargs.get("file_data")
+            self.needs_glue = kwargs.get("needs_glue", True)
+            assert bool(self.id_data) or bool(self.file_data)
+
+            self.mod_time = None
+            self.file_path = None
+            if self.id_data is not None:
+                path = Path(self.id_data.filepath)
+                try:
+                    if path.exists():
+                        self.file_path = str(path.resolve())
+                        self.mod_time = path.stat().st_mtime
+                except OSError:
+                    pass
+
+            if self.file_data is None:
+                self.file_data = self.id_data.as_string()
+
+        # Last chance for encryption...
+        enc = kwargs.get("enc", None)
+        if enc is not None:
+            self._encrypt(enc)
+
+    def _encrypt(self, enc):
+        backing_stream = hsRAMStream()
+        with plEncryptedStream().open(backing_stream, fmCreate, enc) as enc_stream:
+            if self.file_path:
+                if plEncryptedStream.IsFileEncrypted(self.file_path):
+                    with plEncryptedStream().open(self.file_path, fmRead, plEncryptedStream.kEncAuto) as dec_stream:
+                        self._enc_spin_wash(enc_stream, dec_stream)
+                else:
+                    with hsFileStream().open(self.file_path, fmRead) as dec_stream:
+                        self._enc_spin_wash(enc_stream, dec_stream)
+            elif self.file_data:
+                if isinstance(self.file_data, str):
+                    enc_stream.write(self.file_data.encode(_encoding))
+                else:
+                    enc_stream.write(self.file_data)
+            else:
+                raise RuntimeError()
+
+        self.file_data = backing_stream.buffer
+        # do NOT copy over an unencrypted file...
+        self.file_path = None
+
+    def _enc_spin_wash(self, enc_stream, dec_stream):
+        while True:
+            size_rem = dec_stream.size - dec_stream.pos
+            readsz = min(size_rem, _CHUNK_SIZE)
+            if readsz == 0:
+                break
+            data = dec_stream.read(readsz)
+            enc_stream.write(data)
+
+    def __eq__(self, rhs):
+        return str(self) == str(rhs)
+
+    def __hash__(self):
+        return hash(str(self))
+
+    def hash_md5(self):
+        if self.file_path:
+            with open(self.file_path, "rb") as handle:
+                h = md5()
+                data = handle.read(_CHUNK_SIZE)
+                while data:
+                    h.update(data)
+                    data = handle.read(_CHUNK_SIZE)
+                return h.digest()
+        elif self.file_data is not None:
+            if isinstance(self.file_data, str):
+                return md5(self.file_data.encode(_encoding)).digest()
+            else:
+                return md5(self.file_data).digest()
+        else:
+            raise RuntimeError()
+
+    def __str__(self):
+        return "{}/{}".format(self.dirname, self.filename)
+
+
+class OutputFiles:
+    def __init__(self, exporter, path):
+        self._exporter = weakref.ref(exporter)
+        self._export_file = Path(path).resolve()
+        if exporter.dat_only:
+            self._export_path = self._export_file.parent
+        else:
+            self._export_path = self._export_file.parent.parent
+        self._files = set()
+        self._is_zip = self._export_file.suffix.lower() == ".zip"
+        self._py_files = set()
+        self._time = time.time()
+
+    def add_python_code(self, filename, text_id=None, str_data=None):
+        assert filename not in self._py_files
+        of = _OutputFile(file_type=_FileType.python_code,
+                         dirname="Python", filename=filename,
+                         id_data=text_id, file_data=str_data,
+                         skip_hash=True,
+                         internal=(self._version != pvMoul),
+                         needs_glue=False)
+        self._files.add(of)
+        self._py_files.add(filename)
+
+    def add_python_mod(self, filename, text_id=None, str_data=None):
+        assert filename not in self._py_files
+        of = _OutputFile(file_type=_FileType.python_code,
+                         dirname="Python", filename=filename,
+                         id_data=text_id, file_data=str_data,
+                         skip_hash=True,
+                         internal=(self._version != pvMoul),
+                         needs_glue=True)
+        self._files.add(of)
+        self._py_files.add(filename)
+
+    def add_sdl(self, filename, text_id=None, str_data=None):
+        of = _OutputFile(file_type=_FileType.sdl,
+                         dirname="SDL", filename=filename,
+                         id_data=text_id, file_data=str_data,
+                         enc=self.super_secure_encryption)
+        self._files.add(of)
+
+
+    def add_sfx(self, sound_id):
+        of = _OutputFile(file_type=_FileType.sfx,
+                         dirname="sfx", filename=sound_id.name,
+                         id_data=sound_id)
+        self._files.add(of)
+
+    @contextmanager
+    def generate_dat_file(self, filename, **kwargs):
+        dat_only = self._exporter().dat_only
+        dirname = kwargs.get("dirname", "dat")
+        bogus = dat_only and dirname != "dat"
+
+        if self._is_zip or bogus:
+            stream = hsRAMStream(self._version)
+        else:
+            if dat_only:
+                file_path = str(self._export_file.parent / filename)
+            else:
+                file_path = str(self._export_path / dirname / filename)
+            stream = hsFileStream(self._version)
+            stream.open(file_path, fmCreate)
+        backing_stream = stream
+
+        # No sense in wasting time encrypting data that isn't going to be used in the export
+        if not bogus:
+            enc = kwargs.get("enc", None)
+            if enc is not None:
+                stream = plEncryptedStream(self._version)
+                stream.open(backing_stream, fmCreate, enc)
+
+        # The actual export code is run at the "yield" statement. If an error occurs, we
+        # do not want to track this file. Note that the except block is required for the
+        # else block to be legal. ^_^
+        try:
+            yield stream
+        except:
+            raise
+        else:
+            # Must call the EncryptedStream close to actually encrypt the data
+            stream.close()
+            if not stream is backing_stream:
+                backing_stream.close()
+
+            # Not passing enc as a keyword argument to the output file definition. It makes more
+            # sense to yield an encrypted stream from this context manager and encrypt as we go
+            # instead of doing lots of buffer copying to encrypt as a post step.
+            if not bogus:
+                kwargs = {
+                    "file_type": _FileType.generated_dat if dirname == "dat" else
+                                 _FileType.generated_ancillary,
+                    "dirname": dirname,
+                    "filename": filename,
+                    "skip_hash": kwargs.get("skip_hash", False),
+                    "internal": kwargs.get("internal", False),
+                }
+                if isinstance(backing_stream, hsRAMStream):
+                    kwargs["file_data"] = backing_stream.buffer
+                else:
+                    kwargs["file_path"] = file_path
+                self._files.add(_OutputFile(**kwargs))
+
+    def _generate_files(self, func=None):
+        dat_only = self._exporter().dat_only
+        for i in self._files:
+            if dat_only and i.dirname != "dat":
+                continue
+            if func is not None:
+                if func(i):
+                    yield i
+            else:
+                yield i
+
+    def _package_compyled_python(self):
+        func = lambda x: x.file_type == _FileType.python_code
+        report = self._exporter().report
+        version = self._version
+
+        # There can be some debate about what the correct Python version for pvMoul is.
+        # I, quite frankly, don't give a rat's ass at the moment because CWE will only
+        # load Python.pak and no ancillary packages. Maybe someone should fix that, mm?
+        if version <= pvPots:
+            py_version = (2, 2)
+        else:
+            py_version = (2, 3)
+
+        try:
+            pyc_objects = []
+            for i in self._generate_files(func):
+                if i.needs_glue:
+                    py_code = "{}\n\n{}\n".format(i.file_data, plasma_python_glue)
+                else:
+                    py_code = i.file_data
+                result, pyc = korlib.compyle(i.filename, py_code, py_version, report, indent=1)
+                if result:
+                    pyc_objects.append((i.filename, pyc))
+        except korlib.PythonNotAvailableError as error:
+            report.warn("Python {} is not available. Your Age scripts were not packaged.", error, indent=1)
+        else:
+            if pyc_objects:
+                with self.generate_dat_file("{}.pak".format(self._exporter().age_name),
+                                            dirname="Python", enc=self.super_secure_encryption) as stream:
+                    korlib.package_python(stream, pyc_objects)
+
+    def save(self):
+        # At this stage, all Plasma data has been generated from whatever crap is in
+        # Blender. The only remaining part is to make sure any external dependencies are
+        # copied or packed into the appropriate format OR the asset hashes are generated.
+        version = self._version
+
+        # Step 1: Handle Python
+        if self._exporter().python_method != "none" and version != pvMoul:
+            self._package_compyled_python()
+
+        # Step 2: Generate sumfile
+        if self._version != pvMoul:
+            self._write_sumfile()
+
+        # Step 3: Ensure errbody is gut
+        if self._is_zip:
+            self._write_zipfile()
+        else:
+            self._write_deps()
+
+    @property
+    def super_secure_encryption(self):
+        version = self._version
+        if version == pvEoa:
+            return plEncryptedStream.kEncAes
+        elif version == pvMoul:
+            # trollface.jpg
+            return None
+        else:
+            return plEncryptedStream.kEncXtea
+
+    def want_py_text(self, text_id):
+        if text_id is None:
+            return False
+        method = self._exporter().python_method
+        if method == "none":
+            return False
+        elif method == "all":
+            return text_id.name not in self._py_files
+        else:
+            return text_id.plasma_text.package and text_id.name not in self._py_files
+
+    def _write_deps(self):
+        times = (self._time, self._time)
+        func = lambda x: not x.internal and x.file_type not in (_FileType.generated_ancillary, _FileType.generated_dat)
+        report = self._exporter().report
+
+        for i in self._generate_files(func):
+            # Will only ever run for non-"dat" directories.
+            dst_path = str(self._export_path / i.dirname / i.filename)
+            if i.file_data:
+                mode = "w" if isinstance(i.file_data, str) else "wb"
+                with open(dst_path, mode) as handle:
+                    handle.write(i.file_data)
+                os.utime(dst_path, times)
+            elif i.file_path:
+                shutil.copy2(i.file_path, dst_path)
+            else:
+                report.warn("No data found for dependency file '{}'. It will not be copied into the export directory.",
+                            str(i.dirname / i.filename), indent=1)
+
+    def _write_sumfile(self):
+        version = self._version
+        dat_only = self._exporter().dat_only
+        enc = plEncryptedStream.kEncAes if version >= pvEoa else plEncryptedStream.kEncXtea
+        filename = "{}.sum".format(self._exporter().age_name)
+        if dat_only:
+            func = lambda x: (not x.skip_hash and not x.internal) and x.dirname == "dat"
+        else:
+            func = lambda x: not x.skip_hash and not x.internal
+
+        with self.generate_dat_file(filename, enc=enc, skip_hash=True) as stream:
+            files = list(self._generate_files(func))
+            stream.writeInt(len(files))
+            stream.writeInt(0)
+            for i in files:
+                # ABM and UU don't want the directory for PRPs... Bug?
+                extension = Path(i.filename).suffix.lower()
+                if extension == ".prp" and version < pvPots:
+                    filename = i.filename
+                else:
+                    filename = "{}\\{}".format(i.dirname, i.filename)
+                mod_time = i.mod_time if i.mod_time else self._time
+                hash_md5 = i.hash_md5()
+
+                stream.writeSafeStr(filename)
+                stream.write(hash_md5)
+                stream.writeInt(int(mod_time))
+                stream.writeInt(0)
+
+    def _write_zipfile(self):
+        dat_only = self._exporter().dat_only
+        export_time = time.localtime(self._time)[:6]
+        if dat_only:
+            func = lambda x: x.dirname == "dat" and not x.internal
+        else:
+            func = lambda x: not x.internal
+        report = self._exporter().report
+
+        with zipfile.ZipFile(str(self._export_file), 'w', zipfile.ZIP_DEFLATED) as zf:
+            for i in self._generate_files(func):
+                arcpath = i.filename if dat_only else "{}/{}".format(i.dirname, i.filename)
+                if i.file_data:
+                    if isinstance(i.file_data, str):
+                        data = i.file_data.encode(_encoding)
+                    else:
+                        data = i.file_data
+                    zi = zipfile.ZipInfo(arcpath, export_time)
+                    zf.writestr(zi, data)
+                elif i.file_path:
+                    zf.write(i.file_path, arcpath)
+                else:
+                    report.warn("No data found for dependency file '{}'. It will not be archived.", arcpath, indent=1)
+
+    @property
+    def _version(self):
+        return self._exporter().mgr.getVer()

korman/exporter/python.py

@@ -0,0 +1,177 @@
+#    This file is part of Korman.
+#
+#    Korman is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    Korman is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with Korman.  If not, see <http://www.gnu.org/licenses/>.
+
+import bpy
+from pathlib import Path
+from PyHSPlasma import *
+
+from .explosions import ExportError
+from . import logger
+from .. import korlib
+from ..plasma_magic import plasma_python_glue, very_very_special_python
+
+class PythonPackageExporter:
+    def __init__(self, filepath, version):
+        self._filepath = filepath
+        self._modules = {}
+        self._pfms = {}
+        self._version = version
+
+    def _compyle(self, report):
+        report.progress_advance()
+        report.progress_range = len(self._modules) + len(self._pfms)
+        inc_progress = report.progress_increment
+
+        age = bpy.context.scene.world.plasma_age
+        Text = bpy.types.Text
+        if self._version <= pvPots:
+            py_version = (2, 2)
+        else:
+            py_version = (2, 3)
+        py_code = []
+
+        for filename, source in self._pfms.items():
+            if isinstance(source, Text):
+                if not source.plasma_text.package and age.python_method != "all":
+                    inc_progress()
+                    continue
+                code = source.as_string()
+            else:
+                code = source
+
+            code = "{}\n\n{}\n".format(code, plasma_python_glue)
+            success, result = korlib.compyle(filename, code, py_version, report, indent=1)
+            if not success:
+                raise ExportError("Failed to compyle '{}':\n{}".format(filename, result))
+            py_code.append((filename, result))
+            inc_progress()
+
+        for filename, source in self._modules.items():
+            if isinstance(source, Text):
+                if not source.plasma_text.package and age.python_method != "all":
+                    inc_progress()
+                    continue
+                code = source.as_string()
+            else:
+                code = source
+
+            # no glue needed here, ma!
+            success, result = korlib.compyle(filename, code, py_version, report, indent=1)
+            if not success:
+                raise ExportError("Failed to compyle '{}':\n{}".format(filename, result))
+            py_code.append((filename, result))
+            inc_progress()
+
+        # man that was ugly...
+        return py_code
+
+    def _ensure_age_sdl_hook(self, report):
+        age_props = bpy.context.scene.world.plasma_age
+        if age_props.age_sdl:
+            fixed_agename = korlib.replace_python2_identifier(age_props.age_name)
+            py_filename = "{}.py".format(fixed_agename)
+            age_py = self._modules.get(py_filename)
+            if age_py is not None:
+                del self._modules[py_filename]
+                if age_py.plasma_text.package or age.python_method == "all":
+                    self._pfms[py_filename] = age_py
+                else:
+                    report.warn("AgeSDL Python Script provided, but not requested for packing... Using default Python.", indent=1)
+                    self._pfms[py_filename] = very_very_special_python.format(age_name=fixed_agename)
+            else:
+                report.msg("Packing default AgeSDL Python", indent=1)
+                very_very_special_python.format(age_name=age_props.age_name)
+                self._pfms[py_filename] = very_very_special_python.format(age_name=fixed_agename)
+
+    def _harvest_pfms(self, report):
+        objects = bpy.context.scene.objects
+        report.progress_advance()
+        report.progress_range = len(objects)
+        inc_progress = report.progress_increment
+
+        for i in objects:
+            logic = i.plasma_modifiers.advanced_logic
+            if i.plasma_object.enabled and logic.enabled:
+                for j in logic.logic_groups:
+                    tree_versions = (globals()[version] for version in j.version)
+                    if self._version in tree_versions:
+                        self._harvest_tree(j.node_tree)
+            inc_progress()
+
+    def _harvest_modules(self, report):
+        texts = bpy.data.texts
+        report.progress_advance()
+        report.progress_range = len(texts)
+        inc_progress = report.progress_increment
+
+        for i in texts:
+            if i.name.endswith(".py") and i.name not in self._pfms:
+                self._modules.setdefault(i.name, i)
+            inc_progress()
+
+    def _harvest_tree(self, tree):
+        # Search the node tree for any python file nodes. Any that we find are PFMs
+        for i in tree.nodes:
+            if i.bl_idname == "PlasmaPythonFileNode":
+                if i.filename and i.text_id:
+                    self._pfms.setdefault(i.filename, i.text_id)
+
+    def run(self):
+        """Runs a stripped-down version of the Exporter that only handles Python files"""
+        age_props = bpy.context.scene.world.plasma_age
+        log = logger.ExportVerboseLogger if age_props.verbose else logger.ExportProgressLogger
+        with korlib.ConsoleToggler(age_props.show_console), log(self._filepath) as report:
+            report.progress_add_step("Harvesting Plasma PythonFileMods")
+            report.progress_add_step("Harvesting Helper Python Modules")
+            report.progress_add_step("Compyling Python Code")
+            report.progress_add_step("Packing Compyled Code")
+            report.progress_start("PACKING PYTHON")
+
+            # Harvest the Python code
+            self._harvest_pfms(report)
+            self._harvest_modules(report)
+            self._ensure_age_sdl_hook(report)
+
+            # Compyle and package the Python
+            self._package_python(report)
+
+            # DONE
+            report.progress_end()
+            report.raise_errors()
+
+    def _package_python(self, report):
+        py_code = self._compyle(report)
+        if not py_code:
+            report.error("No Python files were packaged.")
+        self._write_python_pak(py_code, report)
+
+    def _write_python_pak(self, py_code, report):
+        report.progress_advance()
+
+        if self._version == pvEoa:
+            enc = plEncryptedStream.kEncAes
+        elif self._version == pvMoul:
+            enc = None
+        else:
+            enc = plEncryptedStream.kEncXtea
+
+        if enc is None:
+            stream = hsFileStream(self._version).open(self._filepath, fmCreate)
+        else:
+            stream = plEncryptedStream(self._version).open(self._filepath, fmCreate, enc)
+        try:
+            korlib.package_python(stream, py_code)
+        finally:
+            stream.close()

korman/exporter/sumfile.py

@@ -1,73 +0,0 @@
-#    This file is part of Korman.
-#
-#    Korman is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    Korman is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with Korman.  If not, see <http://www.gnu.org/licenses/>.
-
-import hashlib
-from pathlib import Path
-from PyHSPlasma import *
-
-def _hashfile(filename, hasher, block=0xFFFF):
-    with open(str(filename), "rb") as handle:
-        h = hasher()
-        data = handle.read(block)
-        while data:
-            h.update(data)
-            data = handle.read(block)
-        return h.digest()
-
-class SumFile:
-    def __init__(self):
-        self._files = set()
-
-    def append(self, filename):
-        self._files.add(filename)
-
-    def _collect_files(self, version):
-        files = []
-        for file in self._files:
-            filename, extension = file.name, file.suffix.lower()
-            if extension in {".age", ".csv", ".fni", ".loc", ".node", ".p2f", ".pfp", ".sub"}:
-                filename = Path("dat") / filename
-            elif extension == ".prp" and version > pvPrime:
-                # ABM and UU don't want the directory for PRPs... Bug?
-                filename = Path("dat") / filename
-            elif extension in {".pak", ".py"}:
-                filename = Path("Python") / filename
-            elif extension in {".avi", ".bik", ".oggv", ".webm"}:
-                filename = Path("avi") / filename
-            elif extension in {".ogg", ".opus", ".wav"}:
-                filename = Path("sfx") / filename
-            elif extension == ".sdl":
-                filename = Path("SDL") / filename
-            # else the filename has no directory prefix... oh well
-
-            md5 = _hashfile(file, hashlib.md5)
-            timestamp = file.stat().st_mtime
-            files.append((str(filename), md5, int(timestamp)))
-        return files
-
-
-    def write(self, sumpath, version):
-        """Writes a .sum file for Uru ABM, PotS, Myst 5, etc."""
-        files = self._collect_files(version)
-        enc = plEncryptedStream.kEncAes if version >= pvEoa else plEncryptedStream.kEncXtea
-
-        with plEncryptedStream(version).open(str(sumpath), fmWrite, enc) as stream:
-            stream.writeInt(len(files))
-            stream.writeInt(0)
-            for file in files:
-                stream.writeSafeStr(str(file[0]))
-                stream.write(file[1])
-                stream.writeInt(file[2])
-                stream.writeInt(0)

korman/korlib/__init__.py

@@ -73,8 +73,16 @@ else:
 
 finally:
     from .console import ConsoleToggler
+    from .python import *
     from .texture import TEX_DETAIL_ALPHA, TEX_DETAIL_ADD, TEX_DETAIL_MULTIPLY
 
+    _IDENTIFIER_RANGES = ((ord('0'), ord('9')), (ord('A'), ord('Z')), (ord('a'), ord('z')))
+    from keyword import kwlist as _kwlist
+    _KEYWORDS = set(_kwlist)
+    # Python 2.x keywords
+    _KEYWORDS.add("exec")
+    _KEYWORDS.add("print")
+
     def _wave_chunks(stream):
         while not stream.eof():
             chunk_name = stream.read(4)
@@ -100,3 +108,31 @@ finally:
         header.read(stream)
 
         return chunks[b"data"]["size"]
+
+    def is_legal_python2_identifier(identifier):
+        if not identifier:
+            return False
+
+        # FIXME: str.isascii in Python 3.7
+        if any(ord(i) > 0x7F for i in identifier):
+            return False
+        if is_python_keyword(identifier):
+            return False
+        return identifier.isidentifier()
+
+    is_python_keyword = _KEYWORDS.__contains__
+
+    def replace_python2_identifier(identifier):
+        """Replaces illegal characters in a Python identifier with a replacement character"""
+
+        def process(identifier):
+            # No leading digits in identifiers, so skip the first range element (0...9)
+            yield next((identifier[0] for low, high in _IDENTIFIER_RANGES[1:]
+                                      if low <= ord(identifier[0]) <= high), '_')
+            for i in identifier[1:]:
+                yield next((i for low, high in _IDENTIFIER_RANGES if low <= ord(i) <= high), '_')
+
+        if identifier:
+            return "".join(process(identifier))
+        else:
+            return ""

korman/korlib/python.py

@@ -0,0 +1,202 @@
+#    This file is part of Korman.
+#
+#    Korman is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    Korman is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with Korman.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import generators # Python 2.2
+import marshal
+import os.path
+import sys
+
+_python_executables = {}
+
+class PythonNotAvailableError(Exception):
+    pass
+
+
+def compyle(file_name, py_code, py_version, report=None, indent=0):
+    # NOTE: Should never run under Python 2.x
+    my_version = sys.version_info[:2]
+    assert my_version == (2, 7) or my_version[0] > 2
+
+    # Remember: Python 2.2 file, so no single line if statements...
+    idx = file_name.find('.')
+    if idx == -1:
+        module_name = file_name
+    else:
+        module_name = file_name[:idx] 
+
+    if report is not None:
+        report.msg("Compyling {}", file_name, indent=indent)
+
+    if my_version != py_version:
+        import subprocess
+
+        py_executable = _find_python(py_version)
+        args = (py_executable, __file__, module_name)
+        try:
+            py_code = py_code.encode("utf-8")
+        except UnicodeError:
+            if report is not None:
+                report.error("Could not encode '{}'", file_name, indent=indent+1)
+            return (False, "Could not encode file")
+        result = subprocess.run(args, input=py_code, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        if result.returncode != 0:
+            try:
+                error = result.stdout.decode("utf-8").replace('\r\n', '\n')
+            except UnicodeError:
+                error = result.stdout
+            if report is not None:
+                report.error("Compylation Error in '{}'\n{}", file_name, error, indent=indent+1)
+        return (result.returncode == 0, result.stdout)
+    else:
+        raise NotImplementedError()
+
+def _compyle(module_name, py_code):
+    # Old python versions have major issues with Windows style newlines.
+    # Also, bad things happen if there is no newline at the end.
+    py_code += '\n' # sigh, this is slow on old Python...
+    py_code = py_code.replace('\r\n', '\n')
+    py_code = py_code.replace('\r', '\n')
+    code_object = compile(py_code, module_name, "exec")
+
+    # The difference between us and the py_compile module is twofold:
+    # 1) py_compile compyles to a file. We might be exporting to memory, so that's
+    #    not what we want.
+    # 2) py_compile saves a *pyc format file containing information such as compyle
+    #    time and marshal format version. These items are not included in Cyan's
+    #    Python.pak format.
+    # Therefore, we simply return the marshalled data as a string.
+    return marshal.dumps(code_object)
+
+def _find_python(py_version):
+    def find_executable(py_version):
+        # First, try to use Blender to find the Python executable
+        try:
+            import bpy
+        except ImportError:
+            pass
+        else:
+            userprefs = bpy.context.user_preferences.addons["korman"].preferences
+            py_executable = getattr(userprefs, "python{}{}_executable".format(*py_version), None)
+            if verify_python(py_version, py_executable):
+                return py_executable
+
+        # Second, try looking Python up in the registry.
+        try:
+            import winreg
+        except ImportError:
+            pass
+        else:
+            py_executable = _find_python_reg(winreg.HKEY_CURRENT_USER, py_version)
+            if verify_python(py_version, py_executable):
+                return py_executable
+            py_executable = _find_python_reg(winreg.HKEY_LOCAL_MACHINE, py_version)
+            if verify_python(py_version, py_executable):
+                return py_executable
+
+        # I give up, you win.
+        return None
+
+    py_executable = _python_executables.setdefault(py_version, find_executable(py_version))
+    if py_executable:
+        return py_executable
+    else:
+        raise PythonNotAvailableError("{}.{}".format(*py_version))
+
+def _find_python_reg(reg_key, py_version):
+    import winreg
+    subkey_name = "Software\\Python\\PythonCore\\{}.{}\\InstallPath".format(*py_version)
+    try:
+        python_dir = winreg.QueryValue(reg_key, subkey_name)
+    except FileNotFoundError:
+        return None
+    else:
+        return os.path.join(python_dir, "python.exe")
+
+def package_python(stream, pyc_objects):
+    # Python.pak format:
+    # uint32_t numFiles
+    #     - safeStr filename
+    #     - uint32_t offset
+    # ~~~~~
+    # uint32_t filesz
+    # uint8_t data[filesz]
+    if not pyc_objects:
+        stream.writeInt(0)
+        return
+
+    # `stream` might be a plEncryptedStream, which doesn't seek very well at all.
+    # Therefore, we will go ahead and calculate the size of the index block so
+    # there is no need to seek around to write offset values
+    base_offset = 4 # uint32_t numFiles
+    data_offset = 0
+    pyc_info = [] # sad, but makes life easier...
+    for module_name, compyled_code in pyc_objects:
+        pyc_info.append((module_name, data_offset, compyled_code))
+
+        # index offset overall
+        base_offset += 2 # writeSafeStr length
+        # NOTE: This assumes that libHSPlasma's hsStream::writeSafeStr converts
+        #       the Python unicode/string object to UTF-8. Currently, this is true.
+        base_offset += len(module_name.encode("utf-8")) # writeSafeStr
+        base_offset += 4
+
+        # current file data offset
+        data_offset += 4  # uint32_t filesz
+        data_offset += len(compyled_code)
+
+    stream.writeInt(len(pyc_info))
+    for module_name, data_offset, compyled_code in pyc_info:
+        stream.writeSafeStr(module_name)
+        # offset of data == index size (base_offset) + offset to data blob (data_offset)
+        stream.writeInt(base_offset + data_offset)
+    for module_name, data_offset, compyled_code in pyc_info:
+        stream.writeInt(len(compyled_code))
+        stream.write(compyled_code)
+
+def verify_python(py_version, py_exe):
+    if not py_exe:
+        return False
+
+    import subprocess
+    try:
+        args = (py_exe, "-V")
+        result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=5)
+    except OSError:
+        return False
+    else:
+        output = result.stdout.decode()
+    try:
+        py_str, py_check = output[:6], output[7:10]
+    except IndexError:
+        return False
+    else:
+        if py_str != "Python":
+            return False
+        return "{}.{}".format(*py_version) == py_check
+
+if __name__ == "__main__":
+    # Python tries to be "helpful" on Windows by converting \n to \r\n.
+    # Therefore we must change the mode of stdout.
+    if sys.platform == "win32":
+        import os, msvcrt
+        msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+
+    try:
+        module_name = sys.argv[1]
+    except IndexError:
+        module_name = "<string>"
+    py_code_source = sys.stdin.read()
+    py_code_object = _compyle(module_name, py_code_source)
+    sys.stdout.write(py_code_object)

korman/nodes/node_python.py

@@ -15,10 +15,12 @@
 
 import bpy
 from bpy.props import *
+from contextlib import contextmanager
 from pathlib import Path
 from PyHSPlasma import *
 
 from .node_core import *
+from .node_deprecated import PlasmaVersionedNode
 from .. import idprops
 
 _single_user_attribs = {
@@ -166,30 +168,40 @@ class PlasmaAttribute(bpy.types.PropertyGroup):
     simple_value = property(_get_simple_value, _set_simple_value)
 
 
-class PlasmaPythonFileNode(PlasmaNodeBase, bpy.types.Node):
+class PlasmaPythonFileNode(PlasmaVersionedNode, bpy.types.Node):
     bl_category = "PYTHON"
     bl_idname = "PlasmaPythonFileNode"
     bl_label = "Python File"
-    bl_width_default = 210
+    bl_width_default = 290
-
-    class _NoUpdate:
-        def __init__(self, node):
-            self._node = node
-        def __enter__(self):
-            self._node.no_update = True
-        def __exit__(self, type, value, traceback):
-            self._node.no_update = False
 
     def _update_pyfile(self, context):
-        with self._NoUpdate(self) as _hack:
+        if self.no_update:
+            return
+        text_id = bpy.data.texts.get(self.filename, None)
+        if text_id:
+            self.text_id = text_id
+
+    def _update_pytext(self, context):
+        if self.no_update:
+            return
+        with self.NoUpdate():
+            self.filename = self.text_id.name
             self.attributes.clear()
             self.inputs.clear()
-        bpy.ops.node.plasma_attributes_to_node(node_path=self.node_path, python_path=self.filepath)
+        if self.text_id is not None:
+            bpy.ops.node.plasma_attributes_to_node(node_path=self.node_path, text_path=self.text_id.name)
 
-    filename = StringProperty(name="File",
+    filename = StringProperty(name="File Name",
-                              description="Python Filename")
+                              description="Python Filename",
-    filepath = StringProperty(update=_update_pyfile,
+                              update=_update_pyfile)
-                              options={"HIDDEN"})
+    filepath = StringProperty(options={"HIDDEN"})
+    text_id = PointerProperty(name="Script File",
+                              description="Script file datablock",
+                              type=bpy.types.Text,
+                              update=_update_pytext)
+
+    # This property exists for UI purposes ONLY
+    package = BoolProperty(options={"HIDDEN", "SKIP_SAVE"})
 
     attributes = CollectionProperty(type=PlasmaAttribute, options={"HIDDEN"})
     no_update = BoolProperty(default=False, options={"HIDDEN", "SKIP_SAVE"})
@@ -199,30 +211,55 @@ class PlasmaPythonFileNode(PlasmaNodeBase, bpy.types.Node):
         return { i.attribute_id: i for i in self.attributes }
 
     def draw_buttons(self, context, layout):
-        row = layout.row(align=True)
+        main_row = layout.row(align=True)
-        if self.filename:
+        row = main_row.row(align=True)
-            row.prop(self, "filename")
+        row.alert = self.text_id is None and bool(self.filename)
-            try:
+        row.prop(self, "text_id", text="Script")
-                if Path(self.filepath).exists():
+        # open operator
-                    operator = row.operator("node.plasma_attributes_to_node", icon="FILE_REFRESH", text="")
+        operator = main_row.operator("file.plasma_file_picker", icon="FILESEL", text="")
-                    operator.python_path = self.filepath
-                    operator.node_path = self.node_path
-            except OSError:
-                pass
-
-        op_text = "" if self.filename else "Select"
-        operator = row.operator("file.plasma_file_picker", icon="SCRIPT", text=op_text)
         operator.filter_glob = "*.py"
         operator.data_path = self.node_path
-        operator.filepath_property = "filepath"
         operator.filename_property = "filename"
+        # package button
+        row = main_row.row(align=True)
+        if self.text_id is not None:
+            row.enabled = True
+            icon = "PACKAGE" if self.text_id.plasma_text.package else "UGLYPACKAGE"
+            row.prop(self.text_id.plasma_text, "package", icon=icon, text="")
+        else:
+            row.enabled = False
+            row.prop(self, "package", text="", icon="UGLYPACKAGE")
+        # rescan operator
+        row = main_row.row(align=True)
+        row.enabled = self.text_id is not None
+        operator = row.operator("node.plasma_attributes_to_node", icon="FILE_REFRESH", text="")
+        if self.text_id is not None:
+            operator.text_path = self.text_id.name
+            operator.node_path = self.node_path
+
+        # This could happen on an upgrade
+        if self.text_id is None and self.filename:
+            layout.label(text="Script '{}' is not loaded in Blender".format(self.filename), icon="ERROR")
 
     def get_key(self, exporter, so):
         return exporter.mgr.find_create_key(plPythonFileMod, name=self.key_name, so=so)
 
     def export(self, exporter, bo, so):
         pfm = self.get_key(exporter, so).object
-        pfm.filename = Path(self.filename).stem
+        py_name = Path(self.filename).stem
+        pfm.filename = py_name
+
+        # Check to see if we should pack this file
+        if exporter.output.want_py_text(self.text_id):
+            exporter.report.msg("Including Python '{}' for package", self.filename, indent=3)
+            exporter.output.add_python_mod(self.filename, text_id=self.text_id)
+            # PFMs can have their own SDL...
+            sdl_text = bpy.data.texts.get("{}.sdl".format(py_name), None)
+            if sdl_text is not None:
+                exporter.report.msg("Including corresponding SDL '{}'", sdl_text.name, indent=3)
+                exporter.output.add_sdl(sdl_text.name, text_id=sdl_text)
+
+        # Handle exporting the Python Parameters
         attrib_sockets = (i for i in self.inputs if i.is_linked)
         for socket in attrib_sockets:
             attrib = socket.attribute_type
@@ -273,10 +310,18 @@ class PlasmaPythonFileNode(PlasmaNodeBase, bpy.types.Node):
         if not is_init and new_pos != old_pos:
             self.inputs.move(old_pos, new_pos)
 
+    @contextmanager
+    def NoUpdate(self):
+        self.no_update = True
+        try:
+            yield self
+        finally:
+            self.no_update = False
+
     def update(self):
         if self.no_update:
             return
-        with self._NoUpdate(self) as _no_recurse:
+        with self.NoUpdate():
             # First, we really want to make sure our junk matches up. Yes, this does dupe what
             # happens in PlasmaAttribNodeBase, but we can link much more than those node types...
             toasty_sockets = []
@@ -312,6 +357,26 @@ class PlasmaPythonFileNode(PlasmaNodeBase, bpy.types.Node):
                     while len(unconnected) > 1:
                         self.inputs.remove(unconnected.pop())
 
+    @property
+    def latest_version(self):
+        return 2
+
+    def upgrade(self):
+        # In version 1 of this node, Python scripts were referenced by their filename in the
+        # python package and by their path on the local machine. This created an undue dependency
+        # on the artist's environment. In version 2, we will use Blender's text data blocks to back
+        # Python scripts. It is still legal to export Python File nodes that are not backed by a script.
+        if self.version == 1:
+            text_id = bpy.data.texts.get(self.filename, None)
+            if text_id is None:
+                path = Path(self.filepath)
+                if path.exists():
+                    text_id = bpy.data.texts.load(self.filepath)
+            with self.NoUpdate():
+                self.text_id = text_id
+            self.property_unset("filepath")
+            self.version = 2
+
 
 class PlasmaPythonFileNodeSocket(PlasmaNodeSocketBase, bpy.types.NodeSocket):
     attribute_id = IntProperty(options={"HIDDEN"})

korman/operators/op_export.py

@@ -17,14 +17,34 @@ import bpy
 from bpy.props import *
 import cProfile
 from pathlib import Path
+from PyHSPlasma import *
 import pstats
 
+from ..addon_prefs import game_versions
 from .. import exporter
 from ..helpers import UiHelper
-from ..properties.prop_world import PlasmaAge, game_versions
+from .. import korlib
-from ..korlib import ConsoleToggler
+from ..properties.prop_world import PlasmaAge
 
-class ExportOperator(bpy.types.Operator):
+class ExportOperator:
+    def _get_default_path(self, context):
+        blend_filepath = context.blend_data.filepath
+        if not blend_filepath:
+            blend_filepath = context.scene.world.plasma_age.age_name
+        if not blend_filepath:
+            blend_filepath = "Korman"
+        return blend_filepath
+
+    @property
+    def has_reports(self):
+        return hasattr(self.report)
+
+    @classmethod
+    def poll(cls, context):
+        return context.scene.render.engine == "PLASMA_GAME"
+
+
+class PlasmaAgeExportOperator(ExportOperator, bpy.types.Operator):
     """Exports ages for Cyan Worlds' Plasma Engine"""
 
     bl_idname = "export.plasma_age"
@@ -71,6 +91,14 @@ class ExportOperator(bpy.types.Operator):
                                                    ("perengine", "Export Supported EnvMaps", "Only environment maps supported by the selected game engine are exported")],
                                          "default": "dcm2dem"}),
 
+        "python_method": (EnumProperty, {"name": "Python",
+                                         "description": "Specifies how Python should be packed",
+                                         "items": [("none", "Pack Nothing", "Don't pack any Python files."),
+                                                   ("as_requested", "Pack Requested Scripts", "Packs any script both linked as a Text file and requested for packaging."),
+                                                   ("all", "Pack All Scripts", "Packs all Python files linked as a Text file.")],
+                                         "default": "as_requested",
+                                         "options": set()}),
+
         "export_active": (BoolProperty, {"name": "INTERNAL: Export currently running",
                                          "default": False,
                                          "options": {"SKIP_SAVE"}}),
@@ -78,7 +106,7 @@ class ExportOperator(bpy.types.Operator):
 
     # This wigs out and very bad things happen if it's not directly on the operator...
     filepath = StringProperty(subtype="FILE_PATH")
-    filter_glob = StringProperty(default="*.age", options={'HIDDEN'})
+    filter_glob = StringProperty(default="*.age;*.zip", options={'HIDDEN'})
 
     version = EnumProperty(name="Version",
                            description="Plasma version to export this age for",
@@ -86,6 +114,11 @@ class ExportOperator(bpy.types.Operator):
                            default="pvPots",
                            options=set())
 
+    dat_only = BoolProperty(name="Export Only PRPs",
+                            description="Only the Age PRPs should be exported",
+                            default=True,
+                            options={"HIDDEN"})
+
     def draw(self, context):
         layout = self.layout
         age = context.scene.world.plasma_age
@@ -95,7 +128,7 @@ class ExportOperator(bpy.types.Operator):
         layout.prop(age, "texcache_method", text="")
         layout.prop(age, "lighting_method")
         row = layout.row()
-        row.enabled = ConsoleToggler.is_platform_supported()
+        row.enabled = korlib.ConsoleToggler.is_platform_supported()
         row.prop(age, "show_console")
         layout.prop(age, "verbose")
         layout.prop(age, "profile_export")
@@ -111,14 +144,6 @@ class ExportOperator(bpy.types.Operator):
         else:
             super().__setattr__(attr, value)
 
-    @property
-    def has_reports(self):
-        return hasattr(self.report)
-
-    @classmethod
-    def poll(cls, context):
-        return context.scene.render.engine == "PLASMA_GAME"
-
     def execute(self, context):
         # Before we begin, do some basic sanity checking...
         path = Path(self.filepath)
@@ -132,13 +157,18 @@ class ExportOperator(bpy.types.Operator):
                 except:
                     self.report({"ERROR"}, "Failed to create export directory")
                     return {"CANCELLED"}
+            path.touch()
 
         # We need to back out of edit mode--this ensures that all changes are committed
         if context.mode != "OBJECT":
             bpy.ops.object.mode_set(mode="OBJECT")
 
-        # Separate blender operator and actual export logic for my sanity
         ageName = path.stem
+        if korlib.is_python_keyword(ageName):
+            self.report({"ERROR"}, "The Age name conflicts with the Python keyword '{}'".format(ageName))
+            return {"CANCELLED"}
+
+        # Separate blender operator and actual export logic for my sanity
         with UiHelper(context) as _ui:
             e = exporter.Exporter(self)
             try:
@@ -151,6 +181,9 @@ class ExportOperator(bpy.types.Operator):
             except exporter.ExportError as error:
                 self.report({"ERROR"}, str(error))
                 return {"CANCELLED"}
+            except exporter.NonfatalExportError as error:
+                self.report({"ERROR"}, str(error))
+                return {"FINISHED"}
             else:
                 if self.profile_export:
                     stats_out = path.with_name("{}_profile.log".format(ageName))
@@ -166,12 +199,8 @@ class ExportOperator(bpy.types.Operator):
         # Called when a user hits "export" from the menu
         # We will prompt them for the export info, then call execute()
         if not self.filepath:
-            blend_filepath = context.blend_data.filepath
+            bfp = self._get_default_path(context)
-            if not blend_filepath:
+            self.filepath = str(Path(bfp).with_suffix(".age"))
-                blend_filepath = context.scene.world.plasma_age.age_name
-            if not blend_filepath:
-                blend_filepath = "Korman"
-            self.filepath = str(Path(blend_filepath).with_suffix(".age"))
         context.window_manager.fileselect_add(self)
         return {"RUNNING_MODAL"}
 
@@ -189,11 +218,85 @@ class ExportOperator(bpy.types.Operator):
             setattr(PlasmaAge, name, prop(**age_options))
 
 
+class PlasmaPythonExportOperator(ExportOperator, bpy.types.Operator):
+    bl_idname = "export.plasma_pak"
+    bl_label = "Package Scripts"
+    bl_description = "Package Age Python scripts"
+
+    filepath = StringProperty(subtype="FILE_PATH")
+    filter_glob = StringProperty(default="*.pak", options={'HIDDEN'})
+
+    version = EnumProperty(name="Version",
+                           description="Plasma version to export this age for",
+                           items=game_versions,
+                           default="pvPots",
+                           options=set())
+
+    def draw(self, context):
+        layout = self.layout
+        age = context.scene.world.plasma_age
+
+        # The crazy mess we're doing with props on the fly means we have to explicitly draw them :(
+        row = layout.row()
+        row.alert = age.python_method == "none"
+        row.prop(age, "python_method")
+        layout.prop(self, "version")
+        row = layout.row()
+        row.enabled = korlib.ConsoleToggler.is_platform_supported()
+        row.prop(age, "show_console")
+        layout.prop(age, "verbose")       
+
+    def execute(self, context):
+        path = Path(self.filepath)
+        if not self.filepath:
+            self.report({"ERROR"}, "No file specified")
+            return {"CANCELLED"}
+        else:
+            if not path.exists:
+                try:
+                    path.mkdir(parents=True)
+                except OSError:
+                    self.report({"ERROR"}, "Failed to create export directory")
+                    return {"CANCELLED"}
+            path.touch()
+
+        # Age names cannot be python keywords
+        age_name = context.scene.world.plasma_age.age_name
+        if korlib.is_python_keyword(age_name):
+            self.report({"ERROR"}, "The Age name conflicts with the Python keyword '{}'".format(age_name))
+            return {"CANCELLED"}
+
+        # Bonus Fun: Implement Profile-mode here (later...)
+        e = exporter.PythonPackageExporter(filepath=self.filepath,
+                                           version=globals()[self.version])
+        try:
+            e.run()
+        except exporter.ExportError as error:
+            self.report({"ERROR"}, str(error))
+            return {"CANCELLED"}
+        except korlib.PythonNotAvailableError as error:
+            self.report({"ERROR"}, "Python Version {} not found".format(error))
+            return {"CANCELLED"}
+        except exporter.NonfatalExportError as error:
+            self.report({"WARNING"}, str(error))
+            return {"FINISHED"}
+        else:
+            return {"FINISHED"}
+
+    def invoke(self, context, event):
+        if not self.filepath:
+            bfp = self._get_default_path(context)
+            self.filepath = str(Path(bfp).with_suffix(".pak"))
+        context.window_manager.fileselect_add(self)
+        return {"RUNNING_MODAL"}
+
+
 # Add the export operator to the Export menu :)
 def menu_cb(self, context):
     if context.scene.render.engine == "PLASMA_GAME":
         self.layout.operator_context = "INVOKE_DEFAULT"
-        self.layout.operator(ExportOperator.bl_idname, text="Plasma Age (.age)")
+        self.layout.operator(PlasmaAgeExportOperator.bl_idname, text="Plasma Age (.age)")
+        self.layout.operator(PlasmaPythonExportOperator.bl_idname, text="Plasma Scripts (.pak)")
 
 
 def register():

korman/operators/op_nodes.py

@@ -26,6 +26,7 @@ class NodeOperator:
 class SelectFileOperator(NodeOperator, bpy.types.Operator):
     bl_idname = "file.plasma_file_picker"
     bl_label = "Select"
+    bl_description = "Load a file"
 
     filter_glob = StringProperty(options={"HIDDEN"})
     filepath = StringProperty(subtype="FILE_PATH")
@@ -41,6 +42,11 @@ class SelectFileOperator(NodeOperator, bpy.types.Operator):
             setattr(dest, self.filepath_property, self.filepath)
         if self.filename_property:
             setattr(dest, self.filename_property, self.filename)
+
+        if bpy.data.texts.get(self.filename, None) is None:
+            bpy.data.texts.load(self.filepath)
+        else:
+            self.report({"WARNING"}, "A file named '{}' is already loaded. It will be used.".format(self.filename))
         return {"FINISHED"}
 
     def invoke(self, context, event):
@@ -91,15 +97,17 @@ pyAttribArgMap= {
 
 class PlPyAttributeNodeOperator(NodeOperator, bpy.types.Operator):
     bl_idname = "node.plasma_attributes_to_node"
-    bl_label = "R"
+    bl_label = "Refresh Sockets"
+    bl_description = "Refresh the Python File node's attribute sockets"
     bl_options = {"INTERNAL"}
 
-    python_path = StringProperty(subtype="FILE_PATH")
+    text_path = StringProperty()
     node_path = StringProperty()
 
     def execute(self, context):
-        from ..plasma_attributes import get_attributes
+        from ..plasma_attributes import get_attributes_from_str
-        attribs = get_attributes(self.python_path)
+        text_id = bpy.data.texts[self.text_path]
+        attribs = get_attributes_from_str(text_id.as_string())
 
         node = eval(self.node_path)
         node_attrib_map = node.attribute_map

korman/operators/op_ui.py

@@ -13,6 +13,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with Korman.  If not, see <http://www.gnu.org/licenses/>.
 
+import addon_utils
 import bpy
 from bpy.props import *
 
@@ -87,3 +88,17 @@ class CollectionRemoveOperator(UIOperator, bpy.types.Operator):
             return {"FINISHED"}
         else:
             return {"CANCELLED"}
+
+
+class OpenAddonPrefs(UIOperator, bpy.types.Operator):
+    bl_idname = "ui.korman_open_prefs"
+    bl_label = "Open Korman Preferences"
+    bl_description = "Opens the Korman User Preferences"
+
+    def execute(self, context):
+        bpy.ops.screen.userpref_show("INVOKE_DEFAULT")
+        context.user_preferences.active_section = "ADDONS"
+        context.window_manager.addon_filter = "System"
+        korman_addon = addon_utils.addons_fake_modules["korman"]
+        addon_utils.module_bl_info(korman_addon)["show_expanded"] = True
+        return {"FINISHED"}

korman/operators/op_world.py

@@ -32,38 +32,35 @@ class GameAddOperator(AgeOperator, bpy.types.Operator):
     game_index = IntProperty(default=-1, options={"HIDDEN"})
 
     def execute(self, context):
-        w = context.world
+        prefs = context.user_preferences.addons["korman"].preferences
-        if w:
-            # First, verify this is a valid Uru directory...
-            path = Path(self.filepath)
-
-            # Blendsucks likes to tack filenames onto our doggone directories...
-            if not path.is_dir():
-                path = path.parent
-            if not ((path / "UruExplorer.exe").is_file() or (path / "plClient.exe").is_file()):
-                self.report({"ERROR"}, "The selected directory is not a copy of URU.")
-                return {"CANCELLED"}
 
-            # New game?
+        # First, verify this is a valid Uru directory...
-            games = w.plasma_games
+        path = Path(self.filepath)
-            new_game = self.game_index == -1
-            if new_game:
-                games.active_game_index = len(games.games)
-                game = games.games.add()
-            else:
-                game = games.games[self.game_index]
 
-            # Setup game...
+        # Blendsucks likes to tack filenames onto our doggone directories...
-            game.path = str(path)
+        if not path.is_dir():
-            if (path / "cypython22.dll").is_file():
+            path = path.parent
-                game.version = "pvPots"
+        if not ((path / "UruExplorer.exe").is_file() or (path / "plClient.exe").is_file()):
-            else:
+            self.report({"ERROR"}, "The selected directory is not a copy of URU.")
-                game.version = "pvMoul"
+            return {"CANCELLED"}
-            game.name = path.name
 
-            return {"FINISHED"}
+        # New game?
+        new_game = self.game_index == -1
+        if new_game:
+            prefs.active_game_index = len(prefs.games)
+            game = prefs.games.add()
         else:
-            return {"CANCELLED"}
+            game = prefs.games[self.game_index]
+
+        # Setup game...
+        game.path = str(path)
+        if (path / "cypython22.dll").is_file():
+            game.version = "pvPots"
+        else:
+            game.version = "pvMoul"
+        game.name = path.name
+
+        return {"FINISHED"}
 
 
     def invoke(self, context, event):
@@ -71,20 +68,53 @@ class GameAddOperator(AgeOperator, bpy.types.Operator):
         return {"RUNNING_MODAL"}
 
 
+class GameConvertOperator(AgeOperator, bpy.types.Operator):
+    bl_idname = "world.plasma_game_convert"
+    bl_label = "This will save your User Preferences file!"
+    bl_description = "Load old per-file Plasma Games into your user preferences"
+
+    def draw(self, context):
+        self.layout.label("test")
+
+    def execute(self, context):
+        prefs = context.user_preferences.addons["korman"].preferences
+        w = context.scene.world
+
+        for old_game in w.plasma_games.games:
+            # don't add dupe games
+            match = next((i for i in prefs.games if i.path == old_game.path), None)
+            if match is not None:
+                continue
+
+            new_game = prefs.games.add()
+            new_game.name = old_game.name
+            new_game.path = old_game.path
+            new_game.version = old_game.version
+
+        w.plasma_games.games.clear()
+        bpy.ops.wm.save_userpref()
+        return {"FINISHED"}
+
+    def invoke(self, context, event):
+        return context.window_manager.invoke_confirm(self, event)
+
+    @classmethod
+    def poll(cls, context):
+        return super().poll(context) and bool(context.scene.world.plasma_games.games)
+
+
 class GameRemoveOperator(AgeOperator, bpy.types.Operator):
     bl_idname = "world.plasma_game_remove"
     bl_label = "Remove Plasma Game"
 
     def execute(self, context):
-        w = context.world
+        prefs = context.user_preferences.addons["korman"].preferences
-        if w:
+
-            games = w.plasma_games
+        if prefs.active_game_index >= len(prefs.games):
-            if games.active_game_index >= len(games.games):
-                return {"CANCELLED"}
-            games.games.remove(games.active_game_index)
-            return {"FINISHED"}
-        else:
             return {"CANCELLED"}
+        prefs.games.remove(prefs.active_game_index)
+        prefs.active_game_index = max(prefs.active_game_index - 1, -1)
+        return {"FINISHED"}
 
 
 class PageAddOperator(AgeOperator, bpy.types.Operator):

korman/plasma_attributes.py

@@ -102,20 +102,22 @@ class PlasmaAttributeVisitor(ast.NodeVisitor):
         ast.NodeVisitor.generic_visit(self, node)
 
 
-def get_attributes(scriptFile):
+def get_attributes_from_file(filepath):
     """Scan the file for assignments matching our regex, let our visitor parse them, and return the
        file's ptAttribs, if any."""
-    attribs = None
+    with open(str(filepath)) as script:
-    with open(str(scriptFile)) as script:
+        return get_attributes_from_str(script.read())
-        results = funcregex.findall(script.read())
+
-        if results:
+def get_attributes_from_str(code):
-            # We'll fake the ptAttribs being all alone in a module...
+    results = funcregex.findall(code)
-            assigns = ast.parse("\n".join(results))
+    if results:
-            v = PlasmaAttributeVisitor()
+        # We'll fake the ptAttribs being all alone in a module...
-            v.visit(assigns)
+        assigns = ast.parse("\n".join(results))
-            if v._attributes:
+        v = PlasmaAttributeVisitor()
-                attribs = v._attributes
+        v.visit(assigns)
-    return attribs
+        if v._attributes:
+            return v._attributes
+    return {}
 
 if __name__ == "__main__":
     import json
@@ -129,7 +131,7 @@ if __name__ == "__main__":
         files = Path(readpath).glob("*.py")
         ptAttribs = {}
         for scriptFile in files:
-            attribs = get_attributes(scriptFile)
+            attribs = get_attributes_from_file(scriptFile)
             if attribs:
                 ptAttribs[scriptFile.stem] = attribs
 

korman/plasma_magic.py

@@ -0,0 +1,220 @@
+#    This file is part of Korman.
+#
+#    Korman is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    Korman is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with Korman.  If not, see <http://www.gnu.org/licenses/>.
+
+very_very_special_python = """
+from Plasma import *
+from PlasmaTypes import *
+
+class {age_name}(ptResponder):
+    pass
+"""
+
+very_very_special_sdl = """
+#==============================================================
+# This VeryVerySpecial SDL File was automatically generated
+# by Korman. Have a nice day!
+#
+# READ: When modifying an SDL record, do *not* modify the
+#       existing record. You must copy and paste a new version
+#       below the current one and make your changes there.
+#==============================================================
+
+STATEDESC {age_name}
+{{
+    VERSION 0
+}}
+
+"""
+
+# Copypasta (with small fixes for str.format) of glue.py from CWE's moul-scripts
+plasma_python_glue = """
+glue_cl = None
+glue_inst = None
+glue_params = None
+glue_paramKeys = None
+try:
+    x = glue_verbose
+except NameError:
+    glue_verbose = 0
+def glue_getClass():
+    global glue_cl
+    if glue_cl == None:
+        try:
+            cl = globals()[glue_name]
+            if issubclass(cl,ptModifier):
+                glue_cl = cl
+            else:
+                if glue_verbose:
+                    print "Class %s is not derived from modifier" % (cl.__name__)
+        except:
+            if glue_verbose:
+                try:
+                    print "Could not find class %s" % (glue_name)
+                except NameError:
+                    print "Filename/classname not set!"
+    return glue_cl
+def glue_getInst():
+    global glue_inst
+    if type(glue_inst) == type(None):
+        cl = glue_getClass()
+        if cl != None:
+            glue_inst = cl()
+    return glue_inst
+def glue_delInst():
+    global glue_inst
+    global glue_cl
+    global glue_params
+    global glue_paramKeys
+    if type(glue_inst) != type(None):
+        del glue_inst
+    glue_cl = None
+    glue_params = None
+    glue_paramKeys = None
+def glue_getVersion():
+    inst = glue_getInst()
+    ver = inst.version
+    glue_delInst()
+    return ver
+def glue_findAndAddAttribs(obj, glue_params):
+    if isinstance(obj,ptAttribute):
+        if glue_params.has_key(obj.id):
+            if glue_verbose:
+                print "WARNING: Duplicate attribute ids!"
+                print "%s has id %d which is already defined in %s" % (obj.name, obj.id, glue_params[obj.id].name)
+        else:
+            glue_params[obj.id] = obj
+    elif type(obj) == type([]):
+        for o in obj:
+            glue_findAndAddAttribs(o, glue_params)
+    elif type(obj) == type(dict()):
+        for o in obj.values():
+            glue_findAndAddAttribs(o, glue_params)
+    elif type(obj) == type( () ):
+        for o in obj:
+            glue_findAndAddAttribs(o, glue_params)
+            
+def glue_getParamDict():
+    global glue_params
+    global glue_paramKeys
+    if type(glue_params) == type(None):
+        glue_params = dict()
+        gd = globals()
+        for obj in gd.values():
+            glue_findAndAddAttribs(obj, glue_params)
+        # rebuild the parameter sorted key list
+        glue_paramKeys = glue_params.keys()
+        glue_paramKeys.sort()
+        glue_paramKeys.reverse()
+    return glue_params
+def glue_getClassName():
+    cl = glue_getClass()
+    if cl != None:
+        return cl.__name__
+    if glue_verbose:
+        print "Class not found in %s.py" % (glue_name)
+    return None
+def glue_getBlockID():
+    inst = glue_getInst()
+    if inst != None:
+        return inst.id
+    if glue_verbose:
+        print "Instance could not be created in %s.py" % (glue_name)
+    return None
+def glue_getNumParams():
+    pd = glue_getParamDict()
+    if pd != None:
+        return len(pd)
+    if glue_verbose:
+        print "No attributes found in %s.py" % (glue_name)
+    return 0
+def glue_getParam(number):
+    global glue_paramKeys
+    pd = glue_getParamDict()
+    if pd != None:
+        # see if there is a paramKey list
+        if type(glue_paramKeys) == type([]):
+            if number >= 0 and number < len(glue_paramKeys):
+                return pd[glue_paramKeys[number]].getdef()
+            else:
+                print "glue_getParam: Error! %d out of range of attribute list" % (number)
+        else:
+            pl = pd.values()
+            if number >= 0 and number < len(pl):
+                return pl[number].getdef()
+            else:
+                if glue_verbose:
+                    print "glue_getParam: Error! %d out of range of attribute list" % (number)
+    if glue_verbose:
+        print "GLUE: Attribute list error"
+    return None
+def glue_setParam(id,value):
+    pd = glue_getParamDict()
+    if pd != None:
+        if pd.has_key(id):
+            try:
+                pd[id].__setvalue__(value)
+            except AttributeError:
+                if isinstance(pd[id],ptAttributeList):
+                    try:
+                        if type(pd[id].value) != type([]):
+                            pd[id].value = []
+                    except AttributeError:
+                        pd[id].value = []
+                    pd[id].value.append(value)
+                else:
+                    pd[id].value = value
+        else:
+            if glue_verbose:
+                print "setParam: can't find id=",id
+    else:
+        print "setParma: Something terribly has gone wrong. Head for the cover."
+def glue_isNamedAttribute(id):
+    pd = glue_getParamDict()
+    if pd != None:
+        try:
+            if isinstance(pd[id],ptAttribNamedActivator):
+                return 1
+            if isinstance(pd[id],ptAttribNamedResponder):
+                return 2
+        except KeyError:
+            if glue_verbose:
+                print "Could not find id=%d attribute" % (id)
+    return 0
+def glue_isMultiModifier():
+    inst = glue_getInst()
+    if isinstance(inst,ptMultiModifier):
+        return 1
+    return 0
+def glue_getVisInfo(number):
+    global glue_paramKeys
+    pd = glue_getParamDict()
+    if pd != None:
+        # see if there is a paramKey list
+        if type(glue_paramKeys) == type([]):
+            if number >= 0 and number < len(glue_paramKeys):
+                return pd[glue_paramKeys[number]].getVisInfo()
+            else:
+                print "glue_getVisInfo: Error! %d out of range of attribute list" % (number)
+        else:
+            pl = pd.values()
+            if number >= 0 and number < len(pl):
+                return pl[number].getVisInfo()
+            else:
+                if glue_verbose:
+                    print "glue_getVisInfo: Error! %d out of range of attribute list" % (number)
+    if glue_verbose:
+        print "GLUE: Attribute list error"
+    return None
+"""

korman/properties/__init__.py

@@ -21,6 +21,7 @@ from .prop_lamp import *
 from . import modifiers
 from .prop_object import *
 from .prop_scene import *
+from .prop_text import *
 from .prop_texture import *
 from .prop_world import *
 
@@ -32,6 +33,7 @@ def register():
     bpy.types.Object.plasma_net = bpy.props.PointerProperty(type=PlasmaNet)
     bpy.types.Object.plasma_object = bpy.props.PointerProperty(type=PlasmaObject)
     bpy.types.Scene.plasma_scene = bpy.props.PointerProperty(type=PlasmaScene)
+    bpy.types.Text.plasma_text = bpy.props.PointerProperty(type=PlasmaText)
     bpy.types.Texture.plasma_layer = bpy.props.PointerProperty(type=PlasmaLayer)
     bpy.types.World.plasma_age = bpy.props.PointerProperty(type=PlasmaAge)
     bpy.types.World.plasma_fni = bpy.props.PointerProperty(type=PlasmaFni)

korman/properties/modifiers/gui.py

@@ -20,8 +20,8 @@ import mathutils
 from bpy.props import *
 from PyHSPlasma import *
 
+from ...addon_prefs import game_versions
 from .base import PlasmaModifierProperties, PlasmaModifierLogicWiz
-from .logic import game_versions
 from ... import idprops
 
 
@@ -180,15 +180,16 @@ class PlasmaJournalBookModifier(PlasmaModifierProperties, PlasmaModifierLogicWiz
         # Assign journal script based on target version
         journal_pfm = journal_pfms[version]
         journalnode = nodes.new("PlasmaPythonFileNode")
-        journalnode.filename = journal_pfm["filename"]
+        with journalnode.NoUpdate():
-
+            journalnode.filename = journal_pfm["filename"]
-        # Manually add required attributes to the PFM
+
-        journal_attribs = journal_pfm["attribs"]
+            # Manually add required attributes to the PFM
-        for attr in journal_attribs:
+            journal_attribs = journal_pfm["attribs"]
-            new_attr = journalnode.attributes.add()
+            for attr in journal_attribs:
-            new_attr.attribute_id = attr["id"]
+                new_attr = journalnode.attributes.add()
-            new_attr.attribute_type = attr["type"]
+                new_attr.attribute_id = attr["id"]
-            new_attr.attribute_name = attr["name"]
+                new_attr.attribute_type = attr["type"]
+                new_attr.attribute_name = attr["name"]
         journalnode.update()
 
         if version == pvPrime:

korman/properties/modifiers/logic.py

@@ -17,8 +17,8 @@ import bpy
 from bpy.props import *
 from PyHSPlasma import *
 
+from ...addon_prefs import game_versions
 from .base import PlasmaModifierProperties
-from ..prop_world import game_versions
 from ...exporter import ExportError
 from ... import idprops
 

korman/properties/modifiers/sound.py

@@ -166,6 +166,9 @@ class PlasmaSound(idprops.IDPropMixin, bpy.types.PropertyGroup):
         header, dataSize = self._get_sound_info()
         length = dataSize / header.avgBytesPerSec
 
+        # HAX: Ensure that the sound file is copied to game, if applicable.
+        exporter.output.add_sfx(self._sound)
+
         # There is some bug in the MOUL code that causes a crash if this does not match the expected
         # result. There's no sense in debugging that though--the user should never specify
         # streaming vs static. That's an implementation detail.

korman/properties/prop_text.py

@@ -0,0 +1,22 @@
+#    This file is part of Korman.
+#
+#    Korman is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    Korman is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with Korman.  If not, see <http://www.gnu.org/licenses/>.
+
+import bpy
+from bpy.props import *
+
+class PlasmaText(bpy.types.PropertyGroup):
+    package = BoolProperty(name="Export",
+                           description="Package this file in the age export",
+                           options=set())

korman/properties/prop_world.py

@@ -17,9 +17,7 @@ import bpy
 from bpy.props import *
 from PyHSPlasma import *
 
-game_versions = [("pvPrime", "Ages Beyond Myst (63.11)", "Targets the original Uru (Live) game"),
+from ..addon_prefs import game_versions
-                 ("pvPots", "Path of the Shell (63.12)", "Targets the most recent offline expansion pack"),
-                 ("pvMoul", "Myst Online: Uru Live (70)", "Targets the most recent online game")]
 
 class PlasmaFni(bpy.types.PropertyGroup):
     bl_idname = "world.plasma_fni"
@@ -58,23 +56,9 @@ class PlasmaFni(bpy.types.PropertyGroup):
                       min=1)
 
 
-class PlasmaGame(bpy.types.PropertyGroup):
-    name = StringProperty(name="Name",
-                          description="Name of the Plasma Game",
-                          options=set())
-    path = StringProperty(name="Path",
-                          description="Path to this Plasma Game",
-                          options=set())
-    version = EnumProperty(name="Version",
-                           description="Plasma version of this game",
-                           items=game_versions,
-                           options=set())
-
-
 class PlasmaGames(bpy.types.PropertyGroup):
     bl_idname = "world.plasma_games"
 
-    games = CollectionProperty(type=PlasmaGame)
     active_game_index = IntProperty(options={"HIDDEN"})
 
     @property

korman/ui/__init__.py

@@ -21,6 +21,7 @@ from .ui_menus import *
 from .ui_modifiers import *
 from .ui_object import *
 from .ui_render_layer import *
+from .ui_text import *
 from .ui_texture import *
 from .ui_toolbox import *
 from .ui_world import *

korman/ui/ui_text.py

@@ -0,0 +1,28 @@
+#    This file is part of Korman.
+#
+#    Korman is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    Korman is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with Korman.  If not, see <http://www.gnu.org/licenses/>.
+
+import bpy
+
+class PlasmaTextEditorHeader(bpy.types.Header):
+    bl_space_type = "TEXT_EDITOR"
+
+    def draw(self, context):
+        layout, text = self.layout, context.space_data.text
+
+        if text is not None:
+            is_py = text.name.endswith(".py")
+            row = layout.row(align=True)
+            row.enabled = is_py
+            row.prop(text.plasma_text, "package")

korman/ui/ui_world.py

@@ -16,7 +16,7 @@
 import bpy
 from pathlib import Path
 
-from ..korlib import ConsoleToggler
+from .. import korlib
 
 
 class AgeButtonsPanel:
@@ -34,41 +34,56 @@ class PlasmaGamePanel(AgeButtonsPanel, bpy.types.Panel):
 
     def draw(self, context):
         layout = self.layout
+        prefs = context.user_preferences.addons["korman"].preferences
         games = context.world.plasma_games
         age = context.world.plasma_age
 
         row = layout.row()
-        row.template_list("PlasmaGameList", "games", games, "games", games,
+        # Remember: game storage moved to addon preferences!
+        row.template_list("PlasmaGameListRO", "games", prefs, "games", games,
                           "active_game_index", rows=2)
-        col = row.column(align=True)
+        row.operator("ui.korman_open_prefs", icon="PREFERENCES", text="")
-        col.operator("world.plasma_game_add", icon="ZOOMIN", text="")
-        col.operator("world.plasma_game_remove", icon="ZOOMOUT", text="")
 
-        # Game Properties
+        # Game Tools
         active_game_index = games.active_game_index
-        if active_game_index < len(games.games):
+        if active_game_index < len(prefs.games):
-            active_game = games.games[active_game_index]
+            active_game = prefs.games[active_game_index]
-
+        else:
-            layout.separator()
+            active_game = None
-            box = layout.box()
 
-            box.prop(active_game, "path", emboss=False)
+        layout.separator()
-            box.prop(active_game, "version")
+        row = layout.row(align=True)
-            box.separator()
+        legal_game = bool(age.age_name.strip()) and active_game is not None
-
+
-            row = box.row(align=True)
+        row.operator_context = "EXEC_DEFAULT"
-            op = row.operator("world.plasma_game_add", icon="FILE_FOLDER", text="Change Path")
+        row.enabled = legal_game
-            op.filepath = active_game.path
+        op = row.operator("export.plasma_age", icon="EXPORT")
-            op.game_index = active_game_index
+        if active_game is not None:
-            row = row.row(align=True)
+            op.dat_only = False
-            row.operator_context = "EXEC_DEFAULT"
-            row.enabled = bool(age.age_name.strip())
-            op = row.operator("export.plasma_age", icon="EXPORT")
             op.filepath = str((Path(active_game.path) / "dat" / age.age_name).with_suffix(".age"))
             op.version = active_game.version
+        row = row.row(align=True)
+        row.enabled = legal_game
+        row.operator_context = "INVOKE_DEFAULT"
+        op = row.operator("export.plasma_age", icon="PACKAGE", text="Package Age")
+        if active_game is not None:
+            op.dat_only = False
+            op.filepath = "{}.zip".format(age.age_name)
+            op.version = active_game.version
+        row = row.row(align=True)
+        row.operator_context = "EXEC_DEFAULT"
+        row.enabled = legal_game and active_game.version != "pvMoul"
+        op = row.operator("export.plasma_pak", icon="FILE_SCRIPT")
+        if active_game is not None:
+            op.filepath = str((Path(active_game.path) / "Python" / age.age_name).with_suffix(".pak"))
+            op.version = active_game.version
+
 
+class PlasmaGameListRO(bpy.types.UIList):
+    def draw_item(self, context, layout, data, item, icon, active_data, active_property, index=0, flt_flag=0):
+        layout.label(item.name, icon="BOOKMARKS")
 
-class PlasmaGameList(bpy.types.UIList):
+class PlasmaGameListRW(bpy.types.UIList):
     def draw_item(self, context, layout, data, item, icon, active_data, active_property, index=0, flt_flag=0):
         layout.prop(item, "name", text="", emboss=False, icon="BOOKMARKS")
 
@@ -113,6 +128,9 @@ class PlasmaAgePanel(AgeButtonsPanel, bpy.types.Panel):
             col.prop(active_page, "seq_suffix")
             col.prop_menu_enum(active_page, "version")
 
+        # Age Names should really be legal Python 2.x identifiers for AgeSDLHooks
+        legal_identifier = korlib.is_legal_python2_identifier(age.age_name)
+
         # Core settings
         layout.separator()
         split = layout.split()
@@ -125,15 +143,23 @@ class PlasmaAgePanel(AgeButtonsPanel, bpy.types.Panel):
         col = split.column()
         col.label("Age Settings:")
         col.prop(age, "seq_prefix", text="ID")
-        col.alert = not age.age_name.strip()
+        col.alert = not legal_identifier or '_' in age.age_name
         col.prop(age, "age_name", text="")
 
+        # Display a hint if the identifier is illegal
+        if not legal_identifier:
+            if korlib.is_python_keyword(age.age_name):
+                layout.label(text="Ages should not be named the same as a Python keyword", icon="ERROR")
+            elif age.age_sdl:
+                fixed_identifier = korlib.replace_python2_identifier(age.age_name)
+                layout.label(text="Age's SDL will use the name '{}'".format(fixed_identifier), icon="ERROR")
+
         layout.separator()
         split = layout.split()
 
         col = split.column()
         col.label("Export Settings:")
-        col.enabled = ConsoleToggler.is_platform_supported()
+        col.enabled = korlib.ConsoleToggler.is_platform_supported()
         col.prop(age, "verbose")
         col.prop(age, "show_console")
 
@@ -145,6 +171,7 @@ class PlasmaAgePanel(AgeButtonsPanel, bpy.types.Panel):
         layout.separator()
         layout.prop(age, "envmap_method")
         layout.prop(age, "lighting_method")
+        layout.prop(age, "python_method")
         layout.prop(age, "texcache_method")