Pack the doggone Python in the exporter

No, I don't want to talk about it...

korman/exporter/convert.py

@@ -62,6 +62,7 @@ class Exporter:
             self.report.progress_add_step("Exporting Scene Objects")
             self.report.progress_add_step("Exporting Logic Nodes")
             self.report.progress_add_step("Finalizing Plasma Logic")
+            self.report.progress_add_step("Handling Snakes")
             self.report.progress_add_step("Exporting Textures")
             self.report.progress_add_step("Composing Geometry")
             self.report.progress_add_step("Saving Age Files")
@@ -94,6 +95,9 @@ class Exporter:
                 #          processing that needs to inspect those objects
                 self._post_process_scene_objects()
 
+                # Step 3.3: Ensure any helper Python files are packed
+                self._pack_ancillary_python()
+
                 # Step 4: Finalize...
                 self.mesh.material.finalize()
                 self.mesh.finalize()
@@ -346,6 +350,17 @@ class Exporter:
                     proc(self, bl_obj, sceneobject)
             inc_progress()
 
+    def _pack_ancillary_python(self):
+        texts = bpy.data.texts
+        self.report.progress_advance()
+        self.report.progress_range = len(texts)
+        inc_progress = self.report.progress_increment
+
+        for i in texts:
+            if i.name.endswith(".py") and self.output.want_py_text(i):
+                self.output.add_python_code(i.name, text_id=i)
+            inc_progress()
+
     def _save_age(self):
         self.report.progress_advance()
 
@@ -369,6 +384,10 @@ class Exporter:
     def envmap_method(self):
         return bpy.context.scene.world.plasma_age.envmap_method
 
+    @property
+    def python_method(self):
+        return bpy.context.scene.world.plasma_age.python_method
+
     @property
     def texcache_path(self):
         age = bpy.context.scene.world.plasma_age

korman/exporter/manager.py

@@ -19,6 +19,7 @@ from PyHSPlasma import *
 import weakref
 
 from . import explosions
+from ..plasma_magic import *
 
 # These objects have to be in the plSceneNode pool in order to be loaded...
 # NOTE: We are using Factory indices because I doubt all of these classes are implemented.
@@ -136,15 +137,19 @@ class ExportManager:
     def create_builtins(self, age, textures):
         # BuiltIn.prp
         if bpy.context.scene.world.plasma_age.age_sdl:
-            builtin = self.create_page(age, "BuiltIn", -2, True)
-            sdl = self.add_object(plSceneObject, name="AgeSDLHook", loc=builtin)
-            pfm = self.add_object(plPythonFileMod, name="VeryVerySpecialPythonFileMod", so=sdl)
-            pfm.filename = age
+            self._create_builtin_pages(age)
+            self._pack_agesdl_hook(age)
 
         # Textures.prp
         if textures:
             self.create_page(age, "Textures", -1, True)
 
+    def _create_builtin_pages(self, age):
+        builtin = self.create_page(age, "BuiltIn", -2, True)
+        sdl = self.add_object(plSceneObject, name="AgeSDLHook", loc=builtin)
+        pfm = self.add_object(plPythonFileMod, name="VeryVerySpecialPythonFileMod", so=sdl)
+        pfm.filename = age
+
     def create_page(self, age, name, id, builtin=False):
         location = plLocation(self.mgr.getVer())
         location.prefix = bpy.context.scene.world.plasma_age.seq_prefix
@@ -233,6 +238,28 @@ class ExportManager:
         else:
             return key.location
 
+    def _pack_agesdl_hook(self, age):
+        get_text = bpy.data.texts.get
+        output = self._exporter().output
+
+        # AgeSDL Hook Python
+        py_filename = "{}.py".format(age)
+        age_py = get_text(py_filename, None)
+        if output.want_py_text(age_py):
+            py_code = age_py.as_string()
+        else:
+            py_code = very_very_special_python.format(age_name=age).lstrip()
+        output.add_python_mod(py_filename, text_id=age_py, str_data=py_code)
+
+        # AgeSDL
+        sdl_filename = "{}.sdl".format(age)
+        age_sdl = get_text(sdl_filename)
+        if age_sdl is not None:
+            sdl_code = None
+        else:
+            sdl_code = very_very_special_sdl.format(age_name=age).lstrip()
+        output.add_sdl(sdl_filename, text_id=age_sdl, str_data=sdl_code)
+
     def save_age(self):
         self._write_age()
         self._write_fni()

korman/exporter/outfile.py

@@ -16,9 +16,11 @@
 from contextlib import contextmanager
 import enum
 from hashlib import md5
+from .. import korlib
 import locale
 import os
 from pathlib import Path
+from ..plasma_magic import plasma_python_glue
 from PyHSPlasma import *
 import shutil
 import time
@@ -40,6 +42,9 @@ def _hashfile(filename, hasher, block=0xFFFF):
 class _FileType(enum.Enum):
     generated_dat = 0
     sfx = 1
+    sdl = 2
+    python_code = 3
+    generated_ancillary = 4
 
 
 class _OutputFile:
@@ -48,8 +53,9 @@ class _OutputFile:
         self.dirname = kwargs.get("dirname")
         self.filename = kwargs.get("filename")
         self.skip_hash = kwargs.get("skip_hash", False)
+        self.internal = kwargs.get("internal", False)
 
-        if self.file_type == _FileType.generated_dat:
+        if self.file_type in (_FileType.generated_dat, _FileType.generated_ancillary):
             self.file_data = kwargs.get("file_data", None)
             self.file_path = kwargs.get("file_path", None)
             self.mod_time = Path(self.file_path).stat().st_mtime if self.file_path else None
@@ -69,6 +75,23 @@ class _OutputFile:
             if self.id_data.packed_file is not None:
                 self.file_data = self.id_data.packed_file.data
 
+        if self.file_type in (_FileType.sdl, _FileType.python_code):
+            self.id_data = kwargs.get("id_data")
+            self.file_data = kwargs.get("file_data")
+            self.needs_glue = kwargs.get("needs_glue", True)
+            assert bool(self.id_data) or bool(self.file_data)
+
+            self.mod_time = None
+            self.file_path = None
+            if self.id_data is not None:
+                path = Path(self.id_data.filepath)
+                if path.exists():
+                    self.mod_time = path.stat().st_mtime
+                    self.file_path = self.id_data.filepath
+
+            if self.file_data is None:
+                self.file_data = self.id_data.as_string()
+
     def __eq__(self, rhs):
         return str(self) == str(rhs)
 
@@ -76,7 +99,7 @@ class _OutputFile:
         return hash(str(self))
 
     def hash_md5(self):
-        if self.file_path is not None:
+        if self.file_path:
             with open(self.file_path, "rb") as handle:
                 h = md5()
                 data = handle.read(0xFFFF)
@@ -106,28 +129,36 @@ class OutputFiles:
             self._export_path = self._export_file.parent.parent
         self._files = set()
         self._is_zip = self._export_file.suffix.lower() == ".zip"
+        self._py_files = set()
         self._time = time.time()
 
-    def add_python(self, filename, text_id=None, str_data=None):
+    def add_python_code(self, filename, text_id=None, str_data=None):
+        assert filename not in self._py_files
         of = _OutputFile(file_type=_FileType.python_code,
                          dirname="Python", filename=filename,
                          id_data=text_id, file_data=str_data,
-                         skip_hash=True)
+                         skip_hash=True,
+                         internal=(self._version != pvMoul),
+                         needs_glue=False)
         self._files.add(of)
+        self._py_files.add(filename)
 
-    def add_sdl(self, filename, text_id=None, str_data=None):
-        version = self._version
-        if version == pvEoa:
-            enc = plEncryptedStream.kEncAes
-        elif version == pvMoul:
-            enc = None
-        else:
-            enc = plEncryptedStream.kEncXtea
+    def add_python_mod(self, filename, text_id=None, str_data=None):
+        assert filename not in self._py_files
+        of = _OutputFile(file_type=_FileType.python_code,
+                         dirname="Python", filename=filename,
+                         id_data=text_id, file_data=str_data,
+                         skip_hash=True,
+                         internal=(self._version != pvMoul),
+                         needs_glue=True)
+        self._files.add(of)
+        self._py_files.add(filename)
 
+    def add_sdl(self, filename, text_id=None, str_data=None):
         of = _OutputFile(file_type=_FileType.sdl,
                          dirname="SDL", filename=filename,
                          id_data=text_id, file_data=str_data,
-                         enc=enc)
+                         enc=self.super_secure_encryption)
         self._files.add(of)
 
 
@@ -165,11 +196,14 @@ class OutputFiles:
             if not stream is backing_stream:
                 backing_stream.close()
 
+            dirname = kwargs.get("dirname", "dat")
             kwargs = {
-                "file_type": _FileType.generated_dat,
-                "dirname": "dat",
+                "file_type": _FileType.generated_dat if dirname == "dat" else
+                             _FileType.generated_ancillary,
+                "dirname": dirname,
                 "filename": filename,
-                "skip_hash": skip_hash,
+                "skip_hash": kwargs.get("skip_hash", False),
+                "internal": kwargs.get("internal", False),
             }
             if isinstance(backing_stream, hsRAMStream):
                 kwargs["file_data"] = backing_stream.buffer
@@ -188,13 +222,46 @@ class OutputFiles:
             else:
                 yield i
 
+    def _package_compyled_python(self):
+        func = lambda x: x.file_type == _FileType.python_code
+        report = self._exporter().report
+        version = self._version
+
+        # There can be some debate about what the correct Python version for pvMoul is.
+        # I, quite frankly, don't give a rat's ass at the moment because CWE will only
+        # load Python.pak and no ancillary packages. Maybe someone should fix that, mm?
+        if version <= pvPots:
+            py_version = (2, 2)
+        else:
+            py_version = (2, 3)
+
+        try:
+            pyc_objects = []
+            for i in self._generate_files(func):
+                if i.needs_glue:
+                    py_code = "{}\n\n{}\n".format(i.file_data, plasma_python_glue)
+                else:
+                    py_code = i.file_data
+                result, pyc = korlib.compyle(i.filename, py_code, py_version, report, indent=1)
+                if result:
+                    pyc_objects.append((i.filename, pyc))
+        except korlib.PythonNotAvailableError as error:
+            report.warn("Python {} is not available. Your Age scripts were not packaged.", error, indent=1)
+        else:
+            if pyc_objects:
+                with self.generate_dat_file("{}.pak".format(self._exporter().age_name),
+                                            dirname="Python", enc=self.super_secure_encryption) as stream:
+                    korlib.package_python(stream, pyc_objects)
+
     def save(self):
         # At this stage, all Plasma data has been generated from whatever crap is in
         # Blender. The only remaining part is to make sure any external dependencies are
         # copied or packed into the appropriate format OR the asset hashes are generated.
+        version = self._version
 
         # Step 1: Handle Python
-        # ... todo ...
+        if self._exporter().python_method != "none" and version != pvMoul:
+            self._package_compyled_python()
 
         # Step 2: Generate sumfile
         if self._version != pvMoul:
@@ -206,9 +273,31 @@ class OutputFiles:
         else:
             self._write_deps()
 
+    @property
+    def super_secure_encryption(self):
+        version = self._version
+        if version == pvEoa:
+            return plEncryptedStream.kEncAes
+        elif version == pvMoul:
+            # trollface.jpg
+            return None
+        else:
+            return plEncryptedStream.kEncXtea
+
+    def want_py_text(self, text_id):
+        if text_id is None:
+            return False
+        method = self._exporter().python_method
+        if method == "none":
+            return False
+        elif method == "all":
+            return text_id.name not in self._py_files
+        else:
+            return text_id.plasma_text.package and text_id.name not in self._py_files
+
     def _write_deps(self):
-        func = lambda x: x.file_type == _FileType.sfx
         times = (self._time, self._time)
+        func = lambda x: not x.internal and x.file_type not in (_FileType.generated_ancillary, _FileType.generated_dat)
 
         for i in self._generate_files(func):
             # Will only ever run for non-"dat" directories.
@@ -229,9 +318,9 @@ class OutputFiles:
         enc = plEncryptedStream.kEncAes if version >= pvEoa else plEncryptedStream.kEncXtea
         filename = "{}.sum".format(self._exporter().age_name)
         if dat_only:
-            func = lambda x: not x.skip_hash and x.dirname == "dat"
+            func = lambda x: (not x.skip_hash and not x.internal) and x.dirname == "dat"
         else:
-            func = lambda x: not x.skip_hash
+            func = lambda x: not x.skip_hash and not x.internal
 
         with self.generate_dat_file(filename, enc=enc, skip_hash=True) as stream:
             files = list(self._generate_files(func))
@@ -256,9 +345,9 @@ class OutputFiles:
         dat_only = self._exporter().dat_only
         export_time = time.localtime(self._time)[:6]
         if dat_only:
-            func = lambda x: x.dirname == "dat"
+            func = lambda x: x.dirname == "dat" and not x.internal
         else:
-            func = None
+            func = lambda x: not x.internal
 
         with zipfile.ZipFile(str(self._export_file), 'w', zipfile.ZIP_DEFLATED) as zf:
             for i in self._generate_files(func):

korman/korlib/python.py

@@ -98,10 +98,10 @@ def _find_python(py_version):
         except ImportError:
             pass
         else:
-            py_executable = _find_python_reg(winreg.HKEY_LOCAL_MACHINE, py_version)
+            py_executable = _find_python_reg(winreg.HKEY_CURRENT_USER, py_version)
             if _verify_python(py_version, py_executable):
                 return py_executable
-            py_executable = _find_python_reg(winreg.HKEY_CURRENT_USER, py_version)
+            py_executable = _find_python_reg(winreg.HKEY_LOCAL_MACHINE, py_version)
             if _verify_python(py_version, py_executable):
                 return py_executable
 

korman/nodes/node_python.py

@@ -246,7 +246,20 @@ class PlasmaPythonFileNode(PlasmaVersionedNode, bpy.types.Node):
 
     def export(self, exporter, bo, so):
         pfm = self.get_key(exporter, so).object
-        pfm.filename = Path(self.filename).stem
+        py_name = Path(self.filename).stem
+        pfm.filename = py_name
+
+        # Check to see if we should pack this file
+        if exporter.output.want_py_text(self.text_id):
+            exporter.report.msg("Including Python '{}' for package", self.filename, indent=3)
+            exporter.output.add_python_mod(self.filename, text_id=self.text_id)
+            # PFMs can have their own SDL...
+            sdl_text = bpy.data.texts.get("{}.sdl".format(py_name), None)
+            if sdl_text is not None:
+                exporter.report.msg("Including corresponding SDL '{}'", sdl_text.name, indent=3)
+                exporter.output.add_sdl(sdl_text.name, text_id=sdl_text)
+
+        # Handle exporting the Python Parameters
         attrib_sockets = (i for i in self.inputs if i.is_linked)
         for socket in attrib_sockets:
             attrib = socket.attribute_type