From 4d7c1cb58be6c8259ad7ab0c56aced93bf943689 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Sat, 29 Dec 2018 11:03:04 -0500 Subject: [PATCH 01/16] Move Plasma Games to addon User Preferences Plasma game installs are a per-user config item and should not be stored in a blend file. Considering that we will be adding more per-user configs, namely Python 2.[2|3|7] install directories, it seems like a good move to go ahead and move the games over. --- korman/__init__.py | 4 +- korman/addon_prefs.py | 76 ++++++++++++++++++++ korman/operators/op_export.py | 3 +- korman/operators/op_ui.py | 15 ++++ korman/operators/op_world.py | 102 +++++++++++++++++---------- korman/properties/modifiers/gui.py | 2 +- korman/properties/modifiers/logic.py | 2 +- korman/properties/prop_world.py | 18 +---- korman/ui/ui_world.py | 42 ++++++----- 9 files changed, 184 insertions(+), 80 deletions(-) create mode 100644 korman/addon_prefs.py diff --git a/korman/__init__.py b/korman/__init__.py index f7b0df8..824cfd8 100644 --- a/korman/__init__.py +++ b/korman/__init__.py @@ -14,6 +14,7 @@ # along with Korman. If not, see . import bpy +from . import addon_prefs from . import exporter, render from . import properties, ui from . import nodes @@ -26,8 +27,7 @@ bl_info = { "location": "File > Import-Export", "description": "Exporter for Cyan Worlds' Plasma Engine", "warning": "beta", - "category": "System", # Eventually, we will hide some of the default - # Blender panels (think materials) + "category": "System", } diff --git a/korman/addon_prefs.py b/korman/addon_prefs.py new file mode 100644 index 0000000..94a3011 --- /dev/null +++ b/korman/addon_prefs.py @@ -0,0 +1,76 @@ +# This file is part of Korman. +# +# Korman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Korman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Korman. If not, see . + +import bpy +from bpy.props import * + +game_versions = [("pvPrime", "Ages Beyond Myst (63.11)", "Targets the original Uru (Live) game"), + ("pvPots", "Path of the Shell (63.12)", "Targets the most recent offline expansion pack"), + ("pvMoul", "Myst Online: Uru Live (70)", "Targets the most recent online game")] + +class PlasmaGame(bpy.types.PropertyGroup): + name = StringProperty(name="Name", + description="Name of the Plasma Game", + options=set()) + path = StringProperty(name="Path", + description="Path to this Plasma Game", + options=set()) + version = EnumProperty(name="Version", + description="Plasma version of this game", + items=game_versions, + options=set()) + + +class KormanAddonPreferences(bpy.types.AddonPreferences): + bl_idname = __package__ + + games = CollectionProperty(type=PlasmaGame) + active_game_index = IntProperty(options={"SKIP_SAVE"}) + + def draw(self, context): + layout = self.layout + + layout.label("Plasma Games:") + row = layout.row() + row.template_list("PlasmaGameListRW", "games", self, "games", self, + "active_game_index", rows=2) + col = row.column(align=True) + col.operator("world.plasma_game_add", icon="ZOOMIN", text="") + col.operator("world.plasma_game_remove", icon="ZOOMOUT", text="") + col.operator("world.plasma_game_convert", icon="IMPORT", text="") + + # Game Properties + active_game_index = self.active_game_index + if bool(self.games) and active_game_index < len(self.games): + active_game = self.games[active_game_index] + + layout.separator() + box = layout.box() + + box.prop(active_game, "path", emboss=False) + box.prop(active_game, "version") + box.separator() + + row = box.row(align=True) + op = row.operator("world.plasma_game_add", icon="FILE_FOLDER", text="Change Path") + op.filepath = active_game.path + op.game_index = active_game_index + + @classmethod + def register(cls): + # Register the old-timey per-world Plasma Games for use in the conversion + # operator. What fun. I guess.... + from .properties.prop_world import PlasmaGames + PlasmaGames.games = CollectionProperty(type=PlasmaGame) diff --git a/korman/operators/op_export.py b/korman/operators/op_export.py index 3a098f9..adec5bd 100644 --- a/korman/operators/op_export.py +++ b/korman/operators/op_export.py @@ -19,9 +19,10 @@ import cProfile from pathlib import Path import pstats +from ..addon_prefs import game_versions from .. import exporter from ..helpers import UiHelper -from ..properties.prop_world import PlasmaAge, game_versions +from ..properties.prop_world import PlasmaAge from ..korlib import ConsoleToggler class ExportOperator(bpy.types.Operator): diff --git a/korman/operators/op_ui.py b/korman/operators/op_ui.py index 0f23da4..ffa3b17 100644 --- a/korman/operators/op_ui.py +++ b/korman/operators/op_ui.py @@ -13,6 +13,7 @@ # You should have received a copy of the GNU General Public License # along with Korman. If not, see . +import addon_utils import bpy from bpy.props import * @@ -87,3 +88,17 @@ class CollectionRemoveOperator(UIOperator, bpy.types.Operator): return {"FINISHED"} else: return {"CANCELLED"} + + +class OpenAddonPrefs(UIOperator, bpy.types.Operator): + bl_idname = "ui.korman_open_prefs" + bl_label = "Open Korman Preferences" + bl_description = "Opens the Korman User Preferences" + + def execute(self, context): + bpy.ops.screen.userpref_show("INVOKE_DEFAULT") + context.user_preferences.active_section = "ADDONS" + context.window_manager.addon_filter = "System" + korman_addon = addon_utils.addons_fake_modules["korman"] + addon_utils.module_bl_info(korman_addon)["show_expanded"] = True + return {"FINISHED"} diff --git a/korman/operators/op_world.py b/korman/operators/op_world.py index 7affe66..866a580 100644 --- a/korman/operators/op_world.py +++ b/korman/operators/op_world.py @@ -32,38 +32,35 @@ class GameAddOperator(AgeOperator, bpy.types.Operator): game_index = IntProperty(default=-1, options={"HIDDEN"}) def execute(self, context): - w = context.world - if w: - # First, verify this is a valid Uru directory... - path = Path(self.filepath) - - # Blendsucks likes to tack filenames onto our doggone directories... - if not path.is_dir(): - path = path.parent - if not ((path / "UruExplorer.exe").is_file() or (path / "plClient.exe").is_file()): - self.report({"ERROR"}, "The selected directory is not a copy of URU.") - return {"CANCELLED"} + prefs = context.user_preferences.addons["korman"].preferences - # New game? - games = w.plasma_games - new_game = self.game_index == -1 - if new_game: - games.active_game_index = len(games.games) - game = games.games.add() - else: - game = games.games[self.game_index] + # First, verify this is a valid Uru directory... + path = Path(self.filepath) - # Setup game... - game.path = str(path) - if (path / "cypython22.dll").is_file(): - game.version = "pvPots" - else: - game.version = "pvMoul" - game.name = path.name + # Blendsucks likes to tack filenames onto our doggone directories... + if not path.is_dir(): + path = path.parent + if not ((path / "UruExplorer.exe").is_file() or (path / "plClient.exe").is_file()): + self.report({"ERROR"}, "The selected directory is not a copy of URU.") + return {"CANCELLED"} - return {"FINISHED"} + # New game? + new_game = self.game_index == -1 + if new_game: + prefs.active_game_index = len(prefs.games) + game = prefs.games.add() else: - return {"CANCELLED"} + game = prefs.games[self.game_index] + + # Setup game... + game.path = str(path) + if (path / "cypython22.dll").is_file(): + game.version = "pvPots" + else: + game.version = "pvMoul" + game.name = path.name + + return {"FINISHED"} def invoke(self, context, event): @@ -71,20 +68,53 @@ class GameAddOperator(AgeOperator, bpy.types.Operator): return {"RUNNING_MODAL"} +class GameConvertOperator(AgeOperator, bpy.types.Operator): + bl_idname = "world.plasma_game_convert" + bl_label = "This will save your User Preferences file!" + bl_description = "Load old per-file Plasma Games into your user preferences" + + def draw(self, context): + self.layout.label("test") + + def execute(self, context): + prefs = context.user_preferences.addons["korman"].preferences + w = context.scene.world + + for old_game in w.plasma_games.games: + # don't add dupe games + match = next((i for i in prefs.games if i.path == old_game.path), None) + if match is not None: + continue + + new_game = prefs.games.add() + new_game.name = old_game.name + new_game.path = old_game.path + new_game.version = old_game.version + + w.plasma_games.games.clear() + bpy.ops.wm.save_userpref() + return {"FINISHED"} + + def invoke(self, context, event): + return context.window_manager.invoke_confirm(self, event) + + @classmethod + def poll(cls, context): + return super().poll(context) and bool(context.scene.world.plasma_games.games) + + class GameRemoveOperator(AgeOperator, bpy.types.Operator): bl_idname = "world.plasma_game_remove" bl_label = "Remove Plasma Game" def execute(self, context): - w = context.world - if w: - games = w.plasma_games - if games.active_game_index >= len(games.games): - return {"CANCELLED"} - games.games.remove(games.active_game_index) - return {"FINISHED"} - else: + prefs = context.user_preferences.addons["korman"].preferences + + if prefs.active_game_index >= len(prefs.games): return {"CANCELLED"} + prefs.games.remove(prefs.active_game_index) + prefs.active_game_index = max(prefs.active_game_index - 1, -1) + return {"FINISHED"} class PageAddOperator(AgeOperator, bpy.types.Operator): diff --git a/korman/properties/modifiers/gui.py b/korman/properties/modifiers/gui.py index 257c3bb..62cecdf 100644 --- a/korman/properties/modifiers/gui.py +++ b/korman/properties/modifiers/gui.py @@ -20,8 +20,8 @@ import mathutils from bpy.props import * from PyHSPlasma import * +from ...addon_prefs import game_versions from .base import PlasmaModifierProperties, PlasmaModifierLogicWiz -from .logic import game_versions from ... import idprops diff --git a/korman/properties/modifiers/logic.py b/korman/properties/modifiers/logic.py index 2dea10e..261bc02 100644 --- a/korman/properties/modifiers/logic.py +++ b/korman/properties/modifiers/logic.py @@ -17,8 +17,8 @@ import bpy from bpy.props import * from PyHSPlasma import * +from ...addon_prefs import game_versions from .base import PlasmaModifierProperties -from ..prop_world import game_versions from ...exporter import ExportError from ... import idprops diff --git a/korman/properties/prop_world.py b/korman/properties/prop_world.py index 9f32b43..1e159a5 100644 --- a/korman/properties/prop_world.py +++ b/korman/properties/prop_world.py @@ -17,9 +17,7 @@ import bpy from bpy.props import * from PyHSPlasma import * -game_versions = [("pvPrime", "Ages Beyond Myst (63.11)", "Targets the original Uru (Live) game"), - ("pvPots", "Path of the Shell (63.12)", "Targets the most recent offline expansion pack"), - ("pvMoul", "Myst Online: Uru Live (70)", "Targets the most recent online game")] +from ..addon_prefs import game_versions class PlasmaFni(bpy.types.PropertyGroup): bl_idname = "world.plasma_fni" @@ -58,23 +56,9 @@ class PlasmaFni(bpy.types.PropertyGroup): min=1) -class PlasmaGame(bpy.types.PropertyGroup): - name = StringProperty(name="Name", - description="Name of the Plasma Game", - options=set()) - path = StringProperty(name="Path", - description="Path to this Plasma Game", - options=set()) - version = EnumProperty(name="Version", - description="Plasma version of this game", - items=game_versions, - options=set()) - - class PlasmaGames(bpy.types.PropertyGroup): bl_idname = "world.plasma_games" - games = CollectionProperty(type=PlasmaGame) active_game_index = IntProperty(options={"HIDDEN"}) @property diff --git a/korman/ui/ui_world.py b/korman/ui/ui_world.py index c5580c4..543cc26 100644 --- a/korman/ui/ui_world.py +++ b/korman/ui/ui_world.py @@ -34,41 +34,39 @@ class PlasmaGamePanel(AgeButtonsPanel, bpy.types.Panel): def draw(self, context): layout = self.layout + prefs = context.user_preferences.addons["korman"].preferences games = context.world.plasma_games age = context.world.plasma_age row = layout.row() - row.template_list("PlasmaGameList", "games", games, "games", games, + # Remember: game storage moved to addon preferences! + row.template_list("PlasmaGameListRO", "games", prefs, "games", games, "active_game_index", rows=2) - col = row.column(align=True) - col.operator("world.plasma_game_add", icon="ZOOMIN", text="") - col.operator("world.plasma_game_remove", icon="ZOOMOUT", text="") + row.operator("ui.korman_open_prefs", icon="PREFERENCES", text="") - # Game Properties + # Game Tools active_game_index = games.active_game_index - if active_game_index < len(games.games): - active_game = games.games[active_game_index] + if active_game_index < len(prefs.games): + active_game = prefs.games[active_game_index] + else: + active_game = None - layout.separator() - box = layout.box() + layout.separator() + row = layout.row(align=True) - box.prop(active_game, "path", emboss=False) - box.prop(active_game, "version") - box.separator() - - row = box.row(align=True) - op = row.operator("world.plasma_game_add", icon="FILE_FOLDER", text="Change Path") - op.filepath = active_game.path - op.game_index = active_game_index - row = row.row(align=True) - row.operator_context = "EXEC_DEFAULT" - row.enabled = bool(age.age_name.strip()) - op = row.operator("export.plasma_age", icon="EXPORT") + row.operator_context = "EXEC_DEFAULT" + row.enabled = bool(age.age_name.strip()) and active_game is not None + op = row.operator("export.plasma_age", icon="EXPORT") + if active_game is not None: op.filepath = str((Path(active_game.path) / "dat" / age.age_name).with_suffix(".age")) op.version = active_game.version -class PlasmaGameList(bpy.types.UIList): +class PlasmaGameListRO(bpy.types.UIList): + def draw_item(self, context, layout, data, item, icon, active_data, active_property, index=0, flt_flag=0): + layout.label(item.name, icon="BOOKMARKS") + +class PlasmaGameListRW(bpy.types.UIList): def draw_item(self, context, layout, data, item, icon, active_data, active_property, index=0, flt_flag=0): layout.prop(item, "name", text="", emboss=False, icon="BOOKMARKS") From 288058aa389e288e0222c7487470c39073ff63ac Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Wed, 2 Jan 2019 16:39:51 -0500 Subject: [PATCH 02/16] Refactor output file generation Age output files are now handled in all aspects by a singleton manager. This allows us to track all generated files and external dependency files and ensure they are correctly copied over to the target game... Or not, in the case of an age/prp export from the File > Export menu. Currently only SFX files are handled as an external dependency. TODO are python and SDL files. Further, because we have an output file manager, we can bundle all the files into a zip archive for releasing the age in one step. Wow such amazing. ^_^ --- korman/exporter/convert.py | 26 ++- korman/exporter/manager.py | 55 +++--- korman/exporter/outfile.py | 267 +++++++++++++++++++++++++++ korman/exporter/sumfile.py | 73 -------- korman/operators/op_export.py | 8 +- korman/properties/modifiers/sound.py | 3 + korman/ui/ui_world.py | 8 + 7 files changed, 327 insertions(+), 113 deletions(-) create mode 100644 korman/exporter/outfile.py delete mode 100644 korman/exporter/sumfile.py diff --git a/korman/exporter/convert.py b/korman/exporter/convert.py index 7dc9ae8..81ffc08 100644 --- a/korman/exporter/convert.py +++ b/korman/exporter/convert.py @@ -27,9 +27,9 @@ from . import image from . import logger from . import manager from . import mesh +from . import outfile from . import physics from . import rtlight -from . import sumfile from . import utils class Exporter: @@ -40,10 +40,6 @@ class Exporter: self.node_trees_exported = set() self.want_node_trees = {} - @property - def age_name(self): - return Path(self._op.filepath).stem - def run(self): log = logger.ExportVerboseLogger if self._op.verbose else logger.ExportProgressLogger with ConsoleToggler(self._op.show_console), log(self._op.filepath) as self.report: @@ -53,7 +49,7 @@ class Exporter: self.physics = physics.PhysicsConverter(self) self.light = rtlight.LightConverter(self) self.animation = animation.AnimationConverter(self) - self.sumfile = sumfile.SumFile() + self.output = outfile.OutputFiles(self, self._op.filepath) self.camera = camera.CameraConverter(self) self.image = image.ImageCache(self) @@ -347,8 +343,22 @@ class Exporter: def _save_age(self): self.report.progress_advance() - self.mgr.save_age(Path(self._op.filepath)) - self.image.save() + + # If something bad happens in the final flush, it would be a shame to + # simply toss away the potentially freshly regenerated texture cache. + try: + self.mgr.save_age() + self.output.save() + finally: + self.image.save() + + @property + def age_name(self): + return Path(self._op.filepath).stem + + @property + def dat_only(self): + return self._op.dat_only @property def envmap_method(self): diff --git a/korman/exporter/manager.py b/korman/exporter/manager.py index 6ff74b3..dc7775e 100644 --- a/korman/exporter/manager.py +++ b/korman/exporter/manager.py @@ -233,27 +233,23 @@ class ExportManager: else: return key.location - def save_age(self, path): - ageName = path.stem - sumfile = self._exporter().sumfile - - sumfile.append(path) - self.mgr.WriteAge(str(path), self._age_info) - self._write_fni(path) - self._write_pages(path) - - if self.getVer() != pvMoul: - sumpath = path.with_suffix(".sum") - sumfile.write(sumpath, self.getVer()) - - def _write_fni(self, path): - if self.mgr.getVer() <= pvMoul: - enc = plEncryptedStream.kEncXtea - else: - enc = plEncryptedStream.kEncAES - fname = path.with_suffix(".fni") + def save_age(self): + self._write_age() + self._write_fni() + self._write_pages() + + def _write_age(self): + f = "{}.age".format(self._age_info.name) + output = self._exporter().output + + with output.generate_dat_file(f, enc=plEncryptedStream.kEncAuto) as stream: + self._age_info.writeToStream(stream) - with plEncryptedStream(self.mgr.getVer()).open(str(fname), fmWrite, enc) as stream: + def _write_fni(self): + f = "{}.fni".format(self._age_info.name) + output = self._exporter().output + + with output.generate_dat_file(f, enc=plEncryptedStream.kEncAuto) as stream: fni = bpy.context.scene.world.plasma_fni stream.writeLine("Graphics.Renderer.SetClearColor {} {} {}".format(*fni.clear_color)) if fni.fog_method != "none": @@ -263,17 +259,14 @@ class ExportManager: elif fni.fog_method == "exp2": stream.writeLine("Graphics.Renderer.Fog.SetDefExp2 {} {}".format(fni.fog_end, fni.fog_density)) stream.writeLine("Graphics.Renderer.SetYon {}".format(fni.yon)) - self._exporter().sumfile.append(fname) - def _write_pages(self, path): + def _write_pages(self): + age_name = self._age_info.name + output = self._exporter().output for loc in self._pages.values(): page = self.mgr.FindPage(loc) # not cached because it's C++ owned - # I know that plAgeInfo has its own way of doing this, but we'd have - # to do some looping and stuff. This is easier. - if self.mgr.getVer() <= pvMoul: - chapter = "_District_" - else: - chapter = "_" - f = path.with_name("{}{}{}".format(path.stem, chapter, page.page)).with_suffix(".prp") - self.mgr.WritePage(str(f), page) - self._exporter().sumfile.append(f) + chapter = "_District_" if self.mgr.getVer() <= pvMoul else "_" + f = "{}{}{}.prp".format(age_name, chapter, page.page) + + with output.generate_dat_file(f) as stream: + self.mgr.WritePage(stream, page) diff --git a/korman/exporter/outfile.py b/korman/exporter/outfile.py new file mode 100644 index 0000000..b98f1f1 --- /dev/null +++ b/korman/exporter/outfile.py @@ -0,0 +1,267 @@ +# This file is part of Korman. +# +# Korman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Korman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Korman. If not, see . + +from contextlib import contextmanager +import enum +from hashlib import md5 +import os +from pathlib import Path +from PyHSPlasma import * +import shutil +import time +import weakref +import zipfile + +def _hashfile(filename, hasher, block=0xFFFF): + with open(str(filename), "rb") as handle: + h = hasher() + data = handle.read(block) + while data: + h.update(data) + data = handle.read(block) + return h.digest() + +@enum.unique +class _FileType(enum.Enum): + generated_dat = 0 + sfx = 1 + + +class _OutputFile: + def __init__(self, **kwargs): + self.file_type = kwargs.get("file_type") + self.dirname = kwargs.get("dirname") + self.filename = kwargs.get("filename") + self.skip_hash = kwargs.get("skip_hash", False) + + if self.file_type == _FileType.generated_dat: + self.file_data = kwargs.get("file_data", None) + self.file_path = kwargs.get("file_path", None) + self.mod_time = Path(self.file_path).stat().st_mtime if self.file_path else None + + # need either a data buffer OR a file path + assert bool(self.file_data) ^ bool(self.file_path) + + if self.file_type == _FileType.sfx: + self.id_data = kwargs.get("id_data") + path = Path(self.id_data.filepath).resolve() + if path.exists(): + self.file_path = str(path) + self.mod_time = path.stat().st_mtime + else: + self.file_path = None + self.mod_time = None + if self.id_data.packed_file is not None: + self.file_data = self.id_data.packed_file.data + + def __eq__(self, rhs): + return str(self) == str(rhs) + + def __hash__(self): + return hash(str(self)) + + def hash_md5(self): + if self.file_path is not None: + with open(self.file_path, "rb") as handle: + h = md5() + data = handle.read(0xFFFF) + while data: + h.update(data) + data = handle.read(0xFFFF) + return h.digest() + elif self.file_data is not None: + return md5(self.file_data).digest() + else: + raise RuntimeError() + + def __str__(self): + return "{}/{}".format(self.dirname, self.filename) + + +class OutputFiles: + def __init__(self, exporter, path): + self._exporter = weakref.ref(exporter) + self._export_file = Path(path).resolve() + if exporter.dat_only: + self._export_path = self._export_file.parent + else: + self._export_path = self._export_file.parent.parent + self._files = set() + self._is_zip = self._export_file.suffix.lower() == ".zip" + self._time = time.time() + + def add_python(self, filename, text_id=None, str_data=None): + of = _OutputFile(file_type=_FileType.python_code, + dirname="Python", filename=filename, + id_data=text_id, file_data=str_data, + skip_hash=True) + self._files.add(of) + + def add_sdl(self, filename, text_id=None, str_data=None): + version = self._version + if version == pvEoa: + enc = plEncryptedStream.kEncAes + elif version == pvMoul: + enc = None + else: + enc = plEncryptedStream.kEncXtea + + of = _OutputFile(file_type=_FileType.sdl, + dirname="SDL", filename=filename, + id_data=text_id, file_data=str_data, + enc=enc) + self._files.add(of) + + + def add_sfx(self, sound_id): + of = _OutputFile(file_type=_FileType.sfx, + dirname="sfx", filename=sound_id.name, + id_data=sound_id) + self._files.add(of) + + @contextmanager + def generate_dat_file(self, filename, **kwargs): + if self._is_zip: + stream = hsRAMStream(self._version) + else: + file_path = str(self._export_file.parent / filename) + stream = hsFileStream(self._version) + stream.open(file_path, fmCreate) + backing_stream = stream + + enc = kwargs.get("enc", None) + if enc is not None: + stream = plEncryptedStream(self._version) + stream.open(backing_stream, fmCreate, enc) + + # The actual export code is run at the "yield" statement. If an error occurs, we + # do not want to track this file. Note that the except block is required for the + # else block to be legal. ^_^ + try: + yield stream + except: + raise + else: + # Must call the EncryptedStream close to actually encrypt the data + stream.close() + if not stream is backing_stream: + backing_stream.close() + + kwargs = { + "file_type": _FileType.generated_dat, + "dirname": "dat", + "filename": filename, + "skip_hash": skip_hash, + } + if isinstance(backing_stream, hsRAMStream): + kwargs["file_data"] = backing_stream.buffer + else: + kwargs["file_path"] = file_path + self._files.add(_OutputFile(**kwargs)) + + def _generate_files(self, func=None): + dat_only = self._exporter().dat_only + for i in self._files: + if dat_only and i.dirname != "dat": + continue + if func is not None: + if func(i): + yield i + else: + yield i + + def save(self): + # At this stage, all Plasma data has been generated from whatever crap is in + # Blender. The only remaining part is to make sure any external dependencies are + # copied or packed into the appropriate format OR the asset hashes are generated. + + # Step 1: Handle Python + # ... todo ... + + # Step 2: Generate sumfile + if self._version != pvMoul: + self._write_sumfile() + + # Step 3: Ensure errbody is gut + if self._is_zip: + self._write_zipfile() + else: + self._write_deps() + + def _write_deps(self): + func = lambda x: x.file_type == _FileType.sfx + times = (self._time, self._time) + + for i in self._generate_files(func): + # Will only ever run for non-"dat" directories. + dst_path = str(self._export_path / i.dirname / i.filename) + if i.file_path: + shutil.copy2(i.file_path, dst_path) + elif i.file_data: + with open(dst_path, "wb") as handle: + handle.write(i.file_data) + os.utime(dst_path, times) + else: + raise RuntimeError() + + def _write_sumfile(self): + version = self._version + dat_only = self._exporter().dat_only + enc = plEncryptedStream.kEncAes if version >= pvEoa else plEncryptedStream.kEncXtea + filename = "{}.sum".format(self._exporter().age_name) + if dat_only: + func = lambda x: not x.skip_hash and x.dirname == "dat" + else: + func = lambda x: not x.skip_hash + + with self.generate_dat_file(filename, enc=enc, skip_hash=True) as stream: + files = list(self._generate_files(func)) + stream.writeInt(len(files)) + stream.writeInt(0) + for i in files: + # ABM and UU don't want the directory for PRPs... Bug? + extension = Path(i.filename).suffix.lower() + if extension == ".prp" and version < pvPots: + filename = i.filename + else: + filename = "{}\\{}".format(i.dirname, i.filename) + mod_time = i.mod_time if i.mod_time else self._time + hash_md5 = i.hash_md5() + + stream.writeSafeStr(filename) + stream.write(hash_md5) + stream.writeInt(int(mod_time)) + stream.writeInt(0) + + def _write_zipfile(self): + dat_only = self._exporter().dat_only + export_time = time.localtime(self._time)[:6] + if dat_only: + func = lambda x: x.dirname == "dat" + else: + func = None + + with zipfile.ZipFile(str(self._export_file), 'w', zipfile.ZIP_DEFLATED) as zf: + for i in self._generate_files(func): + arcpath = i.filename if dat_only else "{}/{}".format(i.dirname, i.filename) + if i.file_path: + zf.write(i.file_path, arcpath) + elif i.file_data: + zi = zipfile.ZipInfo(arcpath, export_time) + zf.writestr(zi, i.file_data) + + @property + def _version(self): + return self._exporter().mgr.getVer() diff --git a/korman/exporter/sumfile.py b/korman/exporter/sumfile.py deleted file mode 100644 index 0bd84c8..0000000 --- a/korman/exporter/sumfile.py +++ /dev/null @@ -1,73 +0,0 @@ -# This file is part of Korman. -# -# Korman is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Korman is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Korman. If not, see . - -import hashlib -from pathlib import Path -from PyHSPlasma import * - -def _hashfile(filename, hasher, block=0xFFFF): - with open(str(filename), "rb") as handle: - h = hasher() - data = handle.read(block) - while data: - h.update(data) - data = handle.read(block) - return h.digest() - -class SumFile: - def __init__(self): - self._files = set() - - def append(self, filename): - self._files.add(filename) - - def _collect_files(self, version): - files = [] - for file in self._files: - filename, extension = file.name, file.suffix.lower() - if extension in {".age", ".csv", ".fni", ".loc", ".node", ".p2f", ".pfp", ".sub"}: - filename = Path("dat") / filename - elif extension == ".prp" and version > pvPrime: - # ABM and UU don't want the directory for PRPs... Bug? - filename = Path("dat") / filename - elif extension in {".pak", ".py"}: - filename = Path("Python") / filename - elif extension in {".avi", ".bik", ".oggv", ".webm"}: - filename = Path("avi") / filename - elif extension in {".ogg", ".opus", ".wav"}: - filename = Path("sfx") / filename - elif extension == ".sdl": - filename = Path("SDL") / filename - # else the filename has no directory prefix... oh well - - md5 = _hashfile(file, hashlib.md5) - timestamp = file.stat().st_mtime - files.append((str(filename), md5, int(timestamp))) - return files - - - def write(self, sumpath, version): - """Writes a .sum file for Uru ABM, PotS, Myst 5, etc.""" - files = self._collect_files(version) - enc = plEncryptedStream.kEncAes if version >= pvEoa else plEncryptedStream.kEncXtea - - with plEncryptedStream(version).open(str(sumpath), fmWrite, enc) as stream: - stream.writeInt(len(files)) - stream.writeInt(0) - for file in files: - stream.writeSafeStr(str(file[0])) - stream.write(file[1]) - stream.writeInt(file[2]) - stream.writeInt(0) diff --git a/korman/operators/op_export.py b/korman/operators/op_export.py index adec5bd..d246542 100644 --- a/korman/operators/op_export.py +++ b/korman/operators/op_export.py @@ -79,7 +79,7 @@ class ExportOperator(bpy.types.Operator): # This wigs out and very bad things happen if it's not directly on the operator... filepath = StringProperty(subtype="FILE_PATH") - filter_glob = StringProperty(default="*.age", options={'HIDDEN'}) + filter_glob = StringProperty(default="*.age;*.zip", options={'HIDDEN'}) version = EnumProperty(name="Version", description="Plasma version to export this age for", @@ -87,6 +87,11 @@ class ExportOperator(bpy.types.Operator): default="pvPots", options=set()) + dat_only = BoolProperty(name="Export Only PRPs", + description="Only the Age PRPs should be exported", + default=True, + options={"HIDDEN"}) + def draw(self, context): layout = self.layout age = context.scene.world.plasma_age @@ -133,6 +138,7 @@ class ExportOperator(bpy.types.Operator): except: self.report({"ERROR"}, "Failed to create export directory") return {"CANCELLED"} + path.touch() # We need to back out of edit mode--this ensures that all changes are committed if context.mode != "OBJECT": diff --git a/korman/properties/modifiers/sound.py b/korman/properties/modifiers/sound.py index db3a297..49d5893 100644 --- a/korman/properties/modifiers/sound.py +++ b/korman/properties/modifiers/sound.py @@ -166,6 +166,9 @@ class PlasmaSound(idprops.IDPropMixin, bpy.types.PropertyGroup): header, dataSize = self._get_sound_info() length = dataSize / header.avgBytesPerSec + # HAX: Ensure that the sound file is copied to game, if applicable. + exporter.output.add_sfx(self._sound) + # There is some bug in the MOUL code that causes a crash if this does not match the expected # result. There's no sense in debugging that though--the user should never specify # streaming vs static. That's an implementation detail. diff --git a/korman/ui/ui_world.py b/korman/ui/ui_world.py index 543cc26..a02d388 100644 --- a/korman/ui/ui_world.py +++ b/korman/ui/ui_world.py @@ -58,8 +58,16 @@ class PlasmaGamePanel(AgeButtonsPanel, bpy.types.Panel): row.enabled = bool(age.age_name.strip()) and active_game is not None op = row.operator("export.plasma_age", icon="EXPORT") if active_game is not None: + op.dat_only = False op.filepath = str((Path(active_game.path) / "dat" / age.age_name).with_suffix(".age")) op.version = active_game.version + row = row.row(align=True) + row.operator_context = "INVOKE_DEFAULT" + op = row.operator("export.plasma_age", icon="PACKAGE", text="Package Age") + if active_game is not None: + op.dat_only = False + op.filepath = "{}.zip".format(age.age_name) + op.version = active_game.version class PlasmaGameListRO(bpy.types.UIList): From e5b79bf6b5871c048c8c623de3f5570506db2c51 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Sat, 5 Jan 2019 23:17:13 -0500 Subject: [PATCH 03/16] Add Python packing operator Implements the boilerplate code for compiling Python code in arbitrary python versions and packing the marshalled data into Cyan's Python.pak format. Since this is a lot of bp, a separate operator has been added to both test the resulting mayhem and provide age creators an easy way to export only their needed Python. The only python that is packed currently is the age sdl hook file, if any. In order for that part to happen, Python File nodes need to be upgraded from having a string path to actually using the new text_id field. --- korman/addon_prefs.py | 30 ++++- korman/exporter/__init__.py | 1 + korman/exporter/python.py | 173 ++++++++++++++++++++++++++ korman/korlib/__init__.py | 1 + korman/korlib/python.py | 200 ++++++++++++++++++++++++++++++ korman/nodes/node_python.py | 5 +- korman/operators/op_export.py | 116 ++++++++++++++--- korman/plasma_magic.py | 219 +++++++++++++++++++++++++++++++++ korman/properties/__init__.py | 2 + korman/properties/prop_text.py | 22 ++++ korman/ui/__init__.py | 1 + korman/ui/ui_text.py | 28 +++++ korman/ui/ui_world.py | 12 +- 13 files changed, 786 insertions(+), 24 deletions(-) create mode 100644 korman/exporter/python.py create mode 100644 korman/korlib/python.py create mode 100644 korman/plasma_magic.py create mode 100644 korman/properties/prop_text.py create mode 100644 korman/ui/ui_text.py diff --git a/korman/addon_prefs.py b/korman/addon_prefs.py index 94a3011..1155e57 100644 --- a/korman/addon_prefs.py +++ b/korman/addon_prefs.py @@ -39,11 +39,26 @@ class KormanAddonPreferences(bpy.types.AddonPreferences): games = CollectionProperty(type=PlasmaGame) active_game_index = IntProperty(options={"SKIP_SAVE"}) + python22_executable = StringProperty(name="Python 2.2", + description="Path to the Python 2.2 executable", + options=set(), + subtype="FILE_PATH") + python23_executable = StringProperty(name="Python 2.3", + description="Path to the Python 2.3 executable", + options=set(), + subtype="FILE_PATH") + python27_executable = StringProperty(name="Python 2.7", + description="Path to the Python 2.7 executable", + options=set(), + subtype="FILE_PATH") + def draw(self, context): layout = self.layout + split = layout.split() + main_col = split.column() - layout.label("Plasma Games:") - row = layout.row() + main_col.label("Plasma Games:") + row = main_col.row() row.template_list("PlasmaGameListRW", "games", self, "games", self, "active_game_index", rows=2) col = row.column(align=True) @@ -51,13 +66,20 @@ class KormanAddonPreferences(bpy.types.AddonPreferences): col.operator("world.plasma_game_remove", icon="ZOOMOUT", text="") col.operator("world.plasma_game_convert", icon="IMPORT", text="") + # Python Installs + main_col = split.column() + main_col.label("Python Executables:") + main_col.prop(self, "python22_executable") + main_col.prop(self, "python23_executable") + main_col.prop(self, "python27_executable") + # Game Properties active_game_index = self.active_game_index if bool(self.games) and active_game_index < len(self.games): active_game = self.games[active_game_index] - layout.separator() - box = layout.box() + layout.label("Game Configuration:") + box = layout.box().column() box.prop(active_game, "path", emboss=False) box.prop(active_game, "version") diff --git a/korman/exporter/__init__.py b/korman/exporter/__init__.py index 0ec72c5..e8652a8 100644 --- a/korman/exporter/__init__.py +++ b/korman/exporter/__init__.py @@ -18,4 +18,5 @@ from PyHSPlasma import * from .convert import * from .explosions import * +from .python import * from . import utils diff --git a/korman/exporter/python.py b/korman/exporter/python.py new file mode 100644 index 0000000..e263b9f --- /dev/null +++ b/korman/exporter/python.py @@ -0,0 +1,173 @@ +# This file is part of Korman. +# +# Korman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Korman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Korman. If not, see . + +import bpy +from pathlib import Path +from PyHSPlasma import * + +from .explosions import ExportError +from . import logger +from .. import korlib +from ..plasma_magic import plasma_python_glue, very_very_special_python + +class PythonPackageExporter: + def __init__(self, filepath, version): + self._filepath = filepath + self._modules = {} + self._pfms = {} + self._version = version + + def _compyle(self, report): + report.progress_advance() + report.progress_range = len(self._modules) + len(self._pfms) + inc_progress = report.progress_increment + + age = bpy.context.scene.world.plasma_age + Text = bpy.types.Text + if self._version <= pvPots: + py_version = (2, 2) + else: + py_version = (2, 3) + py_code = [] + + for filename, source in self._pfms.items(): + if isinstance(source, Text): + if not source.plasma_text.package and age.python_method != "all": + inc_progress() + continue + code = source.as_string() + else: + code = source + + code = "{}\n\n{}\n".format(code, plasma_python_glue) + success, result = korlib.compyle(filename, code, py_version, report, indent=1) + if not success: + raise ExportError("Failed to compyle '{}':\n{}".format(filename, result)) + py_code.append((filename, result)) + inc_progress() + + for filename, source in self._modules.items(): + if isinstance(source, Text): + if not source.plasma_text.package and age.python_method != "all": + inc_progress() + continue + code = source.as_string() + else: + code = source + + # no glue needed here, ma! + success, result = korlib.compyle(filename, code, py_version, report, indent=1) + if not success: + raise ExportError("Failed to compyle '{}':\n{}".format(filename, result)) + py_code.append((filename, result)) + inc_progress() + + # man that was ugly... + return py_code + + def _ensure_age_sdl_hook(self, report): + age_props = bpy.context.scene.world.plasma_age + if age_props.age_sdl: + py_filename = "{}.py".format(age_props.age_name) + age_py = self._modules.get(py_filename) + if age_py is not None: + del self._modules[py_filename] + if age_py.plasma_text.package or age.python_method == "all": + self._pfms[py_filename] = age_py + else: + report.warn("AgeSDL Python Script provided, but not requested for packing... Using default Python.", indent=1) + self._pfms[py_filename] = very_very_special_python.format(age_name=age_props.age_name) + else: + report.msg("Packing default AgeSDL Python", indent=1) + very_very_special_python.format(age_name=age_props.age_name) + self._pfms[py_filename] = very_very_special_python.format(age_name=age_props.age_name) + + def _harvest_pfms(self, report): + objects = bpy.context.scene.objects + report.progress_advance() + report.progress_range = len(objects) + inc_progress = report.progress_increment + + for i in objects: + logic = i.plasma_modifiers.advanced_logic + if i.plasma_object.enabled and logic.enabled: + for j in logic.logic_groups: + tree_versions = (globals()[version] for version in j.version) + if self._version in tree_versions: + self._harvest_tree(j.node_tree) + inc_progress() + + def _harvest_modules(self, report): + texts = bpy.data.texts + report.progress_advance() + report.progress_range = len(texts) + inc_progress = report.progress_increment + + for i in texts: + if i.name.endswith(".py") and i.name not in self._pfms: + self._modules.setdefault(i.name, i) + inc_progress() + + def _harvest_tree(self, tree): + # Search the node tree for any python file nodes. Any that we find are PFMs + for i in tree.nodes: + if i.bl_idname == "PlasmaPythonFileNode": + if i.filename and i.text_id: + self._pfms.setdefault(i.filename, i.text_id) + + def run(self): + """Runs a stripped-down version of the Exporter that only handles Python files""" + age_props = bpy.context.scene.world.plasma_age + log = logger.ExportVerboseLogger if age_props.verbose else logger.ExportProgressLogger + with korlib.ConsoleToggler(age_props.show_console), log(self._filepath) as report: + report.progress_add_step("Harvesting Plasma PythonFileMods") + report.progress_add_step("Harvesting Helper Python Modules") + report.progress_add_step("Compyling Python Code") + report.progress_add_step("Packing Compyled Code") + report.progress_start("PACKING PYTHON") + + # Harvest the Python code + self._harvest_pfms(report) + self._harvest_modules(report) + self._ensure_age_sdl_hook(report) + + # Compyle and package the Python + self._package_python(report) + + # DONE + report.progress_end() + + def _package_python(self, report): + py_code = self._compyle(report) + self._write_python_pak(py_code, report) + + def _write_python_pak(self, py_code, report): + report.progress_advance() + + if self._version == pvEoa: + enc = plEncryptedStream.kEncAes + elif self._version == pvMoul: + enc = None + else: + enc = plEncryptedStream.kEncXtea + + if enc is None: + stream = hsFileStream(self._version).open(self._filepath, fmCreate) + else: + stream = plEncryptedStream(self._version).open(self._filepath, fmCreate, enc) + try: + korlib.package_python(stream, py_code) + finally: + stream.close() diff --git a/korman/korlib/__init__.py b/korman/korlib/__init__.py index 89006c0..3e03574 100644 --- a/korman/korlib/__init__.py +++ b/korman/korlib/__init__.py @@ -73,6 +73,7 @@ else: finally: from .console import ConsoleToggler + from .python import * from .texture import TEX_DETAIL_ALPHA, TEX_DETAIL_ADD, TEX_DETAIL_MULTIPLY def _wave_chunks(stream): diff --git a/korman/korlib/python.py b/korman/korlib/python.py new file mode 100644 index 0000000..0a7e219 --- /dev/null +++ b/korman/korlib/python.py @@ -0,0 +1,200 @@ +# This file is part of Korman. +# +# Korman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Korman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Korman. If not, see . + +from __future__ import generators # Python 2.2 +import marshal +import os.path +import sys + +_python_executables = {} + +class PythonNotAvailableError(Exception): + pass + + +def compyle(file_name, py_code, py_version, report=None, indent=0): + # NOTE: Should never run under Python 2.x + my_version = sys.version_info[:2] + assert my_version == (2, 7) or my_version[0] > 2 + + # Remember: Python 2.2 file, so no single line if statements... + idx = file_name.find('.') + if idx == -1: + module_name = file_name + else: + module_name = file_name[:idx] + + if report is not None: + report.msg("Compyling {}", file_name, indent=indent) + + if my_version != py_version: + import subprocess + + py_executable = _find_python(py_version) + args = (py_executable, __file__, module_name) + try: + py_code = py_code.encode("utf-8") + except UnicodeError: + if report is not None: + report.error("Could not encode '{}'", file_name, indent=indent+1) + return (False, "Could not encode file") + result = subprocess.run(args, input=py_code, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if result.returncode != 0: + try: + error = result.stdout.decode("utf-8").replace('\r\n', '\n') + except UnicodeError: + error = result.stdout + if report is not None: + report.error("Compylation Error in '{}'\n{}", file_name, error, indent=indent+1) + return (result.returncode == 0, result.stdout) + else: + raise NotImplementedError() + +def _compyle(module_name, py_code): + # Old python versions have major issues with Windows style newlines. + # Also, bad things happen if there is no newline at the end. + py_code += '\n' # sigh, this is slow on old Python... + py_code = py_code.replace('\r\n', '\n') + py_code = py_code.replace('\r', '\n') + code_object = compile(py_code, module_name, "exec") + + # The difference between us and the py_compile module is twofold: + # 1) py_compile compyles to a file. We might be exporting to memory, so that's + # not what we want. + # 2) py_compile saves a *pyc format file containing information such as compyle + # time and marshal format version. These items are not included in Cyan's + # Python.pak format. + # Therefore, we simply return the marshalled data as a string. + return marshal.dumps(code_object) + +def _find_python(py_version): + def find_executable(py_version): + # First, try to use Blender to find the Python executable + try: + import bpy + except ImportError: + pass + else: + userprefs = bpy.context.user_preferences.addons["korman"].preferences + py_executable = getattr(userprefs, "python{}{}_executable".format(*py_version), None) + if _verify_python(py_version, py_executable): + return py_executable + + # Second, try looking Python up in the registry. + try: + import winreg + except ImportError: + pass + else: + py_executable = _find_python_reg(winreg.HKEY_LOCAL_MACHINE, py_version) + if _verify_python(py_version, py_executable): + return py_executable + py_executable = _find_python_reg(winreg.HKEY_CURRENT_USER, py_version) + if _verify_python(py_version, py_executable): + return py_executable + + # I give up, you win. + return None + + py_executable = _python_executables.setdefault(py_version, find_executable(py_version)) + if py_executable: + return py_executable + else: + raise PythonNotAvailableError("{}.{}".format(*py_version)) + +def _find_python_reg(reg_key, py_version): + import winreg + subkey_name = "Software\\Python\\PythonCore\\{}.{}\\InstallPath".format(*py_version) + try: + python_dir = winreg.QueryValue(reg_key, subkey_name) + except FileNotFoundError: + return None + else: + return os.path.join(python_dir, "python.exe") + +def package_python(stream, pyc_objects): + # Python.pak format: + # uint32_t numFiles + # - safeStr filename + # - uint32_t offset + # ~~~~~ + # uint32_t filesz + # uint8_t data[filesz] + assert bool(pyc_objects) + + # `stream` might be a plEncryptedStream, which doesn't seek very well at all. + # Therefore, we will go ahead and calculate the size of the index block so + # there is no need to seek around to write offset values + base_offset = 4 # uint32_t numFiles + data_offset = 0 + pyc_info = [] # sad, but makes life easier... + for module_name, compyled_code in pyc_objects: + pyc_info.append((module_name, data_offset, compyled_code)) + + # index offset overall + base_offset += 2 # writeSafeStr length + # NOTE: This assumes that libHSPlasma's hsStream::writeSafeStr converts + # the Python unicode/string object to UTF-8. Currently, this is true. + base_offset += len(module_name.encode("utf-8")) # writeSafeStr + base_offset += 4 + + # current file data offset + data_offset += 4 # uint32_t filesz + data_offset += len(compyled_code) + + stream.writeInt(len(pyc_info)) + for module_name, data_offset, compyled_code in pyc_info: + stream.writeSafeStr(module_name) + # offset of data == index size (base_offset) + offset to data blob (data_offset) + stream.writeInt(base_offset + data_offset) + for module_name, data_offset, compyled_code in pyc_info: + stream.writeInt(len(compyled_code)) + stream.write(compyled_code) + +def _verify_python(py_version, py_exe): + if not py_exe: + return False + + import subprocess + try: + args = (py_exe, "-V") + result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=5) + except OSError: + return False + else: + output = result.stdout.decode() + try: + py_str, py_check = output[:6], output[7:10] + except IndexError: + return False + else: + if py_str != "Python": + return False + return "{}.{}".format(*py_version) == py_check + +if __name__ == "__main__": + # Python tries to be "helpful" on Windows by converting \n to \r\n. + # Therefore we must change the mode of stdout. + if sys.platform == "win32": + import os, msvcrt + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + + try: + module_name = sys.argv[1] + except IndexError: + module_name = "" + py_code_source = sys.stdin.read() + py_code_object = _compyle(module_name, py_code_source) + sys.stdout.write(py_code_object) diff --git a/korman/nodes/node_python.py b/korman/nodes/node_python.py index 60f3583..96c9dcf 100644 --- a/korman/nodes/node_python.py +++ b/korman/nodes/node_python.py @@ -186,10 +186,13 @@ class PlasmaPythonFileNode(PlasmaNodeBase, bpy.types.Node): self.inputs.clear() bpy.ops.node.plasma_attributes_to_node(node_path=self.node_path, python_path=self.filepath) - filename = StringProperty(name="File", + filename = StringProperty(name="File Name", description="Python Filename") filepath = StringProperty(update=_update_pyfile, options={"HIDDEN"}) + text_id = PointerProperty(name="Script File", + description="Script file datablock", + type=bpy.types.Text) attributes = CollectionProperty(type=PlasmaAttribute, options={"HIDDEN"}) no_update = BoolProperty(default=False, options={"HIDDEN", "SKIP_SAVE"}) diff --git a/korman/operators/op_export.py b/korman/operators/op_export.py index d246542..a1944d0 100644 --- a/korman/operators/op_export.py +++ b/korman/operators/op_export.py @@ -17,15 +17,34 @@ import bpy from bpy.props import * import cProfile from pathlib import Path +from PyHSPlasma import * import pstats from ..addon_prefs import game_versions from .. import exporter from ..helpers import UiHelper +from .. import korlib from ..properties.prop_world import PlasmaAge -from ..korlib import ConsoleToggler -class ExportOperator(bpy.types.Operator): +class ExportOperator: + def _get_default_path(self, context): + blend_filepath = context.blend_data.filepath + if not blend_filepath: + blend_filepath = context.scene.world.plasma_age.age_name + if not blend_filepath: + blend_filepath = "Korman" + return blend_filepath + + @property + def has_reports(self): + return hasattr(self.report) + + @classmethod + def poll(cls, context): + return context.scene.render.engine == "PLASMA_GAME" + + +class PlasmaAgeExportOperator(ExportOperator, bpy.types.Operator): """Exports ages for Cyan Worlds' Plasma Engine""" bl_idname = "export.plasma_age" @@ -72,6 +91,14 @@ class ExportOperator(bpy.types.Operator): ("perengine", "Export Supported EnvMaps", "Only environment maps supported by the selected game engine are exported")], "default": "dcm2dem"}), + "python_method": (EnumProperty, {"name": "Python", + "description": "Specifies how Python should be packed", + "items": [("none", "Pack Nothing", "Don't pack any Python files."), + ("as_requested", "Pack Requested Scripts", "Packs any script both linked as a Text file and requested for packaging."), + ("all", "Pack All Scripts", "Packs all Python files linked as a Text file.")], + "default": "as_requested", + "options": set()}), + "export_active": (BoolProperty, {"name": "INTERNAL: Export currently running", "default": False, "options": {"SKIP_SAVE"}}), @@ -101,7 +128,7 @@ class ExportOperator(bpy.types.Operator): layout.prop(age, "texcache_method", text="") layout.prop(age, "lighting_method") row = layout.row() - row.enabled = ConsoleToggler.is_platform_supported() + row.enabled = korlib.ConsoleToggler.is_platform_supported() row.prop(age, "show_console") layout.prop(age, "verbose") layout.prop(age, "profile_export") @@ -117,14 +144,6 @@ class ExportOperator(bpy.types.Operator): else: super().__setattr__(attr, value) - @property - def has_reports(self): - return hasattr(self.report) - - @classmethod - def poll(cls, context): - return context.scene.render.engine == "PLASMA_GAME" - def execute(self, context): # Before we begin, do some basic sanity checking... path = Path(self.filepath) @@ -173,12 +192,8 @@ class ExportOperator(bpy.types.Operator): # Called when a user hits "export" from the menu # We will prompt them for the export info, then call execute() if not self.filepath: - blend_filepath = context.blend_data.filepath - if not blend_filepath: - blend_filepath = context.scene.world.plasma_age.age_name - if not blend_filepath: - blend_filepath = "Korman" - self.filepath = str(Path(blend_filepath).with_suffix(".age")) + bfp = self._get_default_path(context) + self.filepath = str(Path(bfp).with_suffix(".age")) context.window_manager.fileselect_add(self) return {"RUNNING_MODAL"} @@ -196,11 +211,76 @@ class ExportOperator(bpy.types.Operator): setattr(PlasmaAge, name, prop(**age_options)) +class PlasmaPythonExportOperator(ExportOperator, bpy.types.Operator): + bl_idname = "export.plasma_pak" + bl_label = "Package Scripts" + bl_description = "Package Age Python scripts" + + filepath = StringProperty(subtype="FILE_PATH") + filter_glob = StringProperty(default="*.pak", options={'HIDDEN'}) + + version = EnumProperty(name="Version", + description="Plasma version to export this age for", + items=game_versions, + default="pvPots", + options=set()) + + def draw(self, context): + layout = self.layout + age = context.scene.world.plasma_age + + # The crazy mess we're doing with props on the fly means we have to explicitly draw them :( + row = layout.row() + row.alert = age.python_method == "none" + row.prop(age, "python_method") + layout.prop(self, "version") + row = layout.row() + row.enabled = korlib.ConsoleToggler.is_platform_supported() + row.prop(age, "show_console") + layout.prop(age, "verbose") + + def execute(self, context): + path = Path(self.filepath) + if not self.filepath: + self.report({"ERROR"}, "No file specified") + return {"CANCELLED"} + else: + if not path.exists: + try: + path.mkdir(parents=True) + except OSError: + self.report({"ERROR"}, "Failed to create export directory") + return {"CANCELLED"} + path.touch() + + # Bonus Fun: Implement Profile-mode here (later...) + e = exporter.PythonPackageExporter(filepath=self.filepath, + version=globals()[self.version]) + try: + e.run() + except exporter.ExportError as error: + self.report({"ERROR"}, str(error)) + return {"CANCELLED"} + except korlib.PythonNotAvailableError as error: + self.report({"ERROR"}, "Python Version {} not found".format(error)) + return {"CANCELLED"} + else: + return {"FINISHED"} + + def invoke(self, context, event): + if not self.filepath: + bfp = self._get_default_path(context) + self.filepath = str(Path(bfp).with_suffix(".pak")) + context.window_manager.fileselect_add(self) + return {"RUNNING_MODAL"} + + # Add the export operator to the Export menu :) def menu_cb(self, context): if context.scene.render.engine == "PLASMA_GAME": self.layout.operator_context = "INVOKE_DEFAULT" - self.layout.operator(ExportOperator.bl_idname, text="Plasma Age (.age)") + self.layout.operator(PlasmaAgeExportOperator.bl_idname, text="Plasma Age (.age)") + self.layout.operator(PlasmaPythonExportOperator.bl_idname, text="Plasma Scripts (.pak)") def register(): diff --git a/korman/plasma_magic.py b/korman/plasma_magic.py new file mode 100644 index 0000000..b67d438 --- /dev/null +++ b/korman/plasma_magic.py @@ -0,0 +1,219 @@ +# This file is part of Korman. +# +# Korman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Korman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Korman. If not, see . + +very_very_special_python = """ +from Plasma import * +from PlasmaTypes import * + +globals()["{age_name}"] = type("{age_name}", (ptResponder,), dict()) +""" + +very_very_special_sdl = """ +#============================================================== +# This VeryVerySpecial SDL File was automatically generated +# by Korman. Have a nice day! +# +# READ: When modifying an SDL record, do *not* modify the +# existing record. You must copy and paste a new version +# below the current one and make your changes there. +#============================================================== + +STATEDESC {age_name} +{{ + VERSION 0 +}} + +""" + +# Copypasta (with small fixes for str.format) of glue.py from CWE's moul-scripts +plasma_python_glue = """ +glue_cl = None +glue_inst = None +glue_params = None +glue_paramKeys = None +try: + x = glue_verbose +except NameError: + glue_verbose = 0 +def glue_getClass(): + global glue_cl + if glue_cl == None: + try: + cl = globals()[glue_name] + if issubclass(cl,ptModifier): + glue_cl = cl + else: + if glue_verbose: + print "Class %s is not derived from modifier" % (cl.__name__) + except: + if glue_verbose: + try: + print "Could not find class %s" % (glue_name) + except NameError: + print "Filename/classname not set!" + return glue_cl +def glue_getInst(): + global glue_inst + if type(glue_inst) == type(None): + cl = glue_getClass() + if cl != None: + glue_inst = cl() + return glue_inst +def glue_delInst(): + global glue_inst + global glue_cl + global glue_params + global glue_paramKeys + if type(glue_inst) != type(None): + del glue_inst + glue_cl = None + glue_params = None + glue_paramKeys = None +def glue_getVersion(): + inst = glue_getInst() + ver = inst.version + glue_delInst() + return ver +def glue_findAndAddAttribs(obj, glue_params): + if isinstance(obj,ptAttribute): + if glue_params.has_key(obj.id): + if glue_verbose: + print "WARNING: Duplicate attribute ids!" + print "%s has id %d which is already defined in %s" % (obj.name, obj.id, glue_params[obj.id].name) + else: + glue_params[obj.id] = obj + elif type(obj) == type([]): + for o in obj: + glue_findAndAddAttribs(o, glue_params) + elif type(obj) == type(dict()): + for o in obj.values(): + glue_findAndAddAttribs(o, glue_params) + elif type(obj) == type( () ): + for o in obj: + glue_findAndAddAttribs(o, glue_params) + +def glue_getParamDict(): + global glue_params + global glue_paramKeys + if type(glue_params) == type(None): + glue_params = dict() + gd = globals() + for obj in gd.values(): + glue_findAndAddAttribs(obj, glue_params) + # rebuild the parameter sorted key list + glue_paramKeys = glue_params.keys() + glue_paramKeys.sort() + glue_paramKeys.reverse() + return glue_params +def glue_getClassName(): + cl = glue_getClass() + if cl != None: + return cl.__name__ + if glue_verbose: + print "Class not found in %s.py" % (glue_name) + return None +def glue_getBlockID(): + inst = glue_getInst() + if inst != None: + return inst.id + if glue_verbose: + print "Instance could not be created in %s.py" % (glue_name) + return None +def glue_getNumParams(): + pd = glue_getParamDict() + if pd != None: + return len(pd) + if glue_verbose: + print "No attributes found in %s.py" % (glue_name) + return 0 +def glue_getParam(number): + global glue_paramKeys + pd = glue_getParamDict() + if pd != None: + # see if there is a paramKey list + if type(glue_paramKeys) == type([]): + if number >= 0 and number < len(glue_paramKeys): + return pd[glue_paramKeys[number]].getdef() + else: + print "glue_getParam: Error! %d out of range of attribute list" % (number) + else: + pl = pd.values() + if number >= 0 and number < len(pl): + return pl[number].getdef() + else: + if glue_verbose: + print "glue_getParam: Error! %d out of range of attribute list" % (number) + if glue_verbose: + print "GLUE: Attribute list error" + return None +def glue_setParam(id,value): + pd = glue_getParamDict() + if pd != None: + if pd.has_key(id): + try: + pd[id].__setvalue__(value) + except AttributeError: + if isinstance(pd[id],ptAttributeList): + try: + if type(pd[id].value) != type([]): + pd[id].value = [] + except AttributeError: + pd[id].value = [] + pd[id].value.append(value) + else: + pd[id].value = value + else: + if glue_verbose: + print "setParam: can't find id=",id + else: + print "setParma: Something terribly has gone wrong. Head for the cover." +def glue_isNamedAttribute(id): + pd = glue_getParamDict() + if pd != None: + try: + if isinstance(pd[id],ptAttribNamedActivator): + return 1 + if isinstance(pd[id],ptAttribNamedResponder): + return 2 + except KeyError: + if glue_verbose: + print "Could not find id=%d attribute" % (id) + return 0 +def glue_isMultiModifier(): + inst = glue_getInst() + if isinstance(inst,ptMultiModifier): + return 1 + return 0 +def glue_getVisInfo(number): + global glue_paramKeys + pd = glue_getParamDict() + if pd != None: + # see if there is a paramKey list + if type(glue_paramKeys) == type([]): + if number >= 0 and number < len(glue_paramKeys): + return pd[glue_paramKeys[number]].getVisInfo() + else: + print "glue_getVisInfo: Error! %d out of range of attribute list" % (number) + else: + pl = pd.values() + if number >= 0 and number < len(pl): + return pl[number].getVisInfo() + else: + if glue_verbose: + print "glue_getVisInfo: Error! %d out of range of attribute list" % (number) + if glue_verbose: + print "GLUE: Attribute list error" + return None +""" diff --git a/korman/properties/__init__.py b/korman/properties/__init__.py index d8ea5e6..741a519 100644 --- a/korman/properties/__init__.py +++ b/korman/properties/__init__.py @@ -21,6 +21,7 @@ from .prop_lamp import * from . import modifiers from .prop_object import * from .prop_scene import * +from .prop_text import * from .prop_texture import * from .prop_world import * @@ -32,6 +33,7 @@ def register(): bpy.types.Object.plasma_net = bpy.props.PointerProperty(type=PlasmaNet) bpy.types.Object.plasma_object = bpy.props.PointerProperty(type=PlasmaObject) bpy.types.Scene.plasma_scene = bpy.props.PointerProperty(type=PlasmaScene) + bpy.types.Text.plasma_text = bpy.props.PointerProperty(type=PlasmaText) bpy.types.Texture.plasma_layer = bpy.props.PointerProperty(type=PlasmaLayer) bpy.types.World.plasma_age = bpy.props.PointerProperty(type=PlasmaAge) bpy.types.World.plasma_fni = bpy.props.PointerProperty(type=PlasmaFni) diff --git a/korman/properties/prop_text.py b/korman/properties/prop_text.py new file mode 100644 index 0000000..3746cf7 --- /dev/null +++ b/korman/properties/prop_text.py @@ -0,0 +1,22 @@ +# This file is part of Korman. +# +# Korman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Korman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Korman. If not, see . + +import bpy +from bpy.props import * + +class PlasmaText(bpy.types.PropertyGroup): + package = BoolProperty(name="Export", + description="Package this file in the age export", + options=set()) diff --git a/korman/ui/__init__.py b/korman/ui/__init__.py index 3369bb7..a292060 100644 --- a/korman/ui/__init__.py +++ b/korman/ui/__init__.py @@ -21,6 +21,7 @@ from .ui_menus import * from .ui_modifiers import * from .ui_object import * from .ui_render_layer import * +from .ui_text import * from .ui_texture import * from .ui_toolbox import * from .ui_world import * diff --git a/korman/ui/ui_text.py b/korman/ui/ui_text.py new file mode 100644 index 0000000..4d3fd12 --- /dev/null +++ b/korman/ui/ui_text.py @@ -0,0 +1,28 @@ +# This file is part of Korman. +# +# Korman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Korman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Korman. If not, see . + +import bpy + +class PlasmaTextEditorHeader(bpy.types.Header): + bl_space_type = "TEXT_EDITOR" + + def draw(self, context): + layout, text = self.layout, context.space_data.text + + if text is not None: + is_py = text.name.endswith(".py") + row = layout.row(align=True) + row.enabled = is_py + row.prop(text.plasma_text, "package") diff --git a/korman/ui/ui_world.py b/korman/ui/ui_world.py index a02d388..938f0f3 100644 --- a/korman/ui/ui_world.py +++ b/korman/ui/ui_world.py @@ -53,21 +53,30 @@ class PlasmaGamePanel(AgeButtonsPanel, bpy.types.Panel): layout.separator() row = layout.row(align=True) + legal_game = bool(age.age_name.strip()) and active_game is not None row.operator_context = "EXEC_DEFAULT" - row.enabled = bool(age.age_name.strip()) and active_game is not None + row.enabled = legal_game op = row.operator("export.plasma_age", icon="EXPORT") if active_game is not None: op.dat_only = False op.filepath = str((Path(active_game.path) / "dat" / age.age_name).with_suffix(".age")) op.version = active_game.version row = row.row(align=True) + row.enabled = legal_game row.operator_context = "INVOKE_DEFAULT" op = row.operator("export.plasma_age", icon="PACKAGE", text="Package Age") if active_game is not None: op.dat_only = False op.filepath = "{}.zip".format(age.age_name) op.version = active_game.version + row = row.row(align=True) + row.operator_context = "EXEC_DEFAULT" + row.enabled = legal_game and active_game.version != "pvMoul" + op = row.operator("export.plasma_pak", icon="FILE_SCRIPT") + if active_game is not None: + op.filepath = str((Path(active_game.path) / "Python" / age.age_name).with_suffix(".pak")) + op.version = active_game.version class PlasmaGameListRO(bpy.types.UIList): @@ -151,6 +160,7 @@ class PlasmaAgePanel(AgeButtonsPanel, bpy.types.Panel): layout.separator() layout.prop(age, "envmap_method") layout.prop(age, "lighting_method") + layout.prop(age, "python_method") layout.prop(age, "texcache_method") From b6c9551883f828ec0843a517525da85d92ed1a8a Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Sun, 6 Jan 2019 20:27:14 -0500 Subject: [PATCH 04/16] PythonFile Node v2 Version 2 of the python file node is now backed by a `bpy.types.Text` datablock in the case of a file whose attributes are updated from a backing file. --- korman/nodes/node_python.py | 111 +++++++++++++++++++++-------- korman/operators/op_nodes.py | 16 +++-- korman/plasma_attributes.py | 28 ++++---- korman/properties/modifiers/gui.py | 19 ++--- 4 files changed, 117 insertions(+), 57 deletions(-) diff --git a/korman/nodes/node_python.py b/korman/nodes/node_python.py index 96c9dcf..8d9b646 100644 --- a/korman/nodes/node_python.py +++ b/korman/nodes/node_python.py @@ -15,10 +15,12 @@ import bpy from bpy.props import * +from contextlib import contextmanager from pathlib import Path from PyHSPlasma import * from .node_core import * +from .node_deprecated import PlasmaVersionedNode from .. import idprops _single_user_attribs = { @@ -166,33 +168,40 @@ class PlasmaAttribute(bpy.types.PropertyGroup): simple_value = property(_get_simple_value, _set_simple_value) -class PlasmaPythonFileNode(PlasmaNodeBase, bpy.types.Node): +class PlasmaPythonFileNode(PlasmaVersionedNode, bpy.types.Node): bl_category = "PYTHON" bl_idname = "PlasmaPythonFileNode" bl_label = "Python File" - bl_width_default = 210 - - class _NoUpdate: - def __init__(self, node): - self._node = node - def __enter__(self): - self._node.no_update = True - def __exit__(self, type, value, traceback): - self._node.no_update = False + bl_width_default = 290 def _update_pyfile(self, context): - with self._NoUpdate(self) as _hack: + if self.no_update: + return + text_id = bpy.data.texts.get(self.filename, None) + if text_id: + self.text_id = text_id + + def _update_pytext(self, context): + if self.no_update: + return + with self.NoUpdate(): + self.filename = self.text_id.name self.attributes.clear() self.inputs.clear() - bpy.ops.node.plasma_attributes_to_node(node_path=self.node_path, python_path=self.filepath) + if self.text_id is not None: + bpy.ops.node.plasma_attributes_to_node(node_path=self.node_path, text_path=self.text_id.name) filename = StringProperty(name="File Name", - description="Python Filename") - filepath = StringProperty(update=_update_pyfile, - options={"HIDDEN"}) + description="Python Filename", + update=_update_pyfile) + filepath = StringProperty(options={"HIDDEN"}) text_id = PointerProperty(name="Script File", description="Script file datablock", - type=bpy.types.Text) + type=bpy.types.Text, + update=_update_pytext) + + # This property exists for UI purposes ONLY + package = BoolProperty(options={"HIDDEN", "SKIP_SAVE"}) attributes = CollectionProperty(type=PlasmaAttribute, options={"HIDDEN"}) no_update = BoolProperty(default=False, options={"HIDDEN", "SKIP_SAVE"}) @@ -202,23 +211,35 @@ class PlasmaPythonFileNode(PlasmaNodeBase, bpy.types.Node): return { i.attribute_id: i for i in self.attributes } def draw_buttons(self, context, layout): - row = layout.row(align=True) - if self.filename: - row.prop(self, "filename") - try: - if Path(self.filepath).exists(): - operator = row.operator("node.plasma_attributes_to_node", icon="FILE_REFRESH", text="") - operator.python_path = self.filepath - operator.node_path = self.node_path - except OSError: - pass - - op_text = "" if self.filename else "Select" - operator = row.operator("file.plasma_file_picker", icon="SCRIPT", text=op_text) + main_row = layout.row(align=True) + row = main_row.row(align=True) + row.alert = self.text_id is None and bool(self.filename) + row.prop(self, "text_id", text="Script") + # open operator + operator = main_row.operator("file.plasma_file_picker", icon="FILESEL", text="") operator.filter_glob = "*.py" operator.data_path = self.node_path - operator.filepath_property = "filepath" operator.filename_property = "filename" + # package button + row = main_row.row(align=True) + if self.text_id is not None: + row.enabled = True + icon = "PACKAGE" if self.text_id.plasma_text.package else "UGLYPACKAGE" + row.prop(self.text_id.plasma_text, "package", icon=icon, text="") + else: + row.enabled = False + row.prop(self, "package", text="", icon="UGLYPACKAGE") + # rescan operator + row = main_row.row(align=True) + row.enabled = self.text_id is not None + operator = row.operator("node.plasma_attributes_to_node", icon="FILE_REFRESH", text="") + if self.text_id is not None: + operator.text_path = self.text_id.name + operator.node_path = self.node_path + + # This could happen on an upgrade + if self.text_id is None and self.filename: + layout.label(text="Script '{}' is not loaded in Blender".format(self.filename), icon="ERROR") def get_key(self, exporter, so): return exporter.mgr.find_create_key(plPythonFileMod, name=self.key_name, so=so) @@ -276,10 +297,18 @@ class PlasmaPythonFileNode(PlasmaNodeBase, bpy.types.Node): if not is_init and new_pos != old_pos: self.inputs.move(old_pos, new_pos) + @contextmanager + def NoUpdate(self): + self.no_update = True + try: + yield self + finally: + self.no_update = False + def update(self): if self.no_update: return - with self._NoUpdate(self) as _no_recurse: + with self.NoUpdate(): # First, we really want to make sure our junk matches up. Yes, this does dupe what # happens in PlasmaAttribNodeBase, but we can link much more than those node types... toasty_sockets = [] @@ -315,6 +344,26 @@ class PlasmaPythonFileNode(PlasmaNodeBase, bpy.types.Node): while len(unconnected) > 1: self.inputs.remove(unconnected.pop()) + @property + def latest_version(self): + return 2 + + def upgrade(self): + # In version 1 of this node, Python scripts were referenced by their filename in the + # python package and by their path on the local machine. This created an undue dependency + # on the artist's environment. In version 2, we will use Blender's text data blocks to back + # Python scripts. It is still legal to export Python File nodes that are not backed by a script. + if self.version == 1: + text_id = bpy.data.texts.get(self.filename, None) + if text_id is None: + path = Path(self.filepath) + if path.exists(): + text_id = bpy.data.texts.load(self.filepath) + with self.NoUpdate(): + self.text_id = text_id + self.property_unset("filepath") + self.version = 2 + class PlasmaPythonFileNodeSocket(PlasmaNodeSocketBase, bpy.types.NodeSocket): attribute_id = IntProperty(options={"HIDDEN"}) diff --git a/korman/operators/op_nodes.py b/korman/operators/op_nodes.py index 64e0391..6fa1e96 100644 --- a/korman/operators/op_nodes.py +++ b/korman/operators/op_nodes.py @@ -26,6 +26,7 @@ class NodeOperator: class SelectFileOperator(NodeOperator, bpy.types.Operator): bl_idname = "file.plasma_file_picker" bl_label = "Select" + bl_description = "Load a file" filter_glob = StringProperty(options={"HIDDEN"}) filepath = StringProperty(subtype="FILE_PATH") @@ -41,6 +42,11 @@ class SelectFileOperator(NodeOperator, bpy.types.Operator): setattr(dest, self.filepath_property, self.filepath) if self.filename_property: setattr(dest, self.filename_property, self.filename) + + if bpy.data.texts.get(self.filename, None) is None: + bpy.data.texts.load(self.filepath) + else: + self.report({"WARNING"}, "A file named '{}' is already loaded. It will be used.".format(self.filename)) return {"FINISHED"} def invoke(self, context, event): @@ -91,15 +97,17 @@ pyAttribArgMap= { class PlPyAttributeNodeOperator(NodeOperator, bpy.types.Operator): bl_idname = "node.plasma_attributes_to_node" - bl_label = "R" + bl_label = "Refresh Sockets" + bl_description = "Refresh the Python File node's attribute sockets" bl_options = {"INTERNAL"} - python_path = StringProperty(subtype="FILE_PATH") + text_path = StringProperty() node_path = StringProperty() def execute(self, context): - from ..plasma_attributes import get_attributes - attribs = get_attributes(self.python_path) + from ..plasma_attributes import get_attributes_from_str + text_id = bpy.data.texts[self.text_path] + attribs = get_attributes_from_str(text_id.as_string()) node = eval(self.node_path) node_attrib_map = node.attribute_map diff --git a/korman/plasma_attributes.py b/korman/plasma_attributes.py index b2dfd34..70408fb 100644 --- a/korman/plasma_attributes.py +++ b/korman/plasma_attributes.py @@ -102,20 +102,22 @@ class PlasmaAttributeVisitor(ast.NodeVisitor): ast.NodeVisitor.generic_visit(self, node) -def get_attributes(scriptFile): +def get_attributes_from_file(filepath): """Scan the file for assignments matching our regex, let our visitor parse them, and return the file's ptAttribs, if any.""" - attribs = None - with open(str(scriptFile)) as script: - results = funcregex.findall(script.read()) - if results: - # We'll fake the ptAttribs being all alone in a module... - assigns = ast.parse("\n".join(results)) - v = PlasmaAttributeVisitor() - v.visit(assigns) - if v._attributes: - attribs = v._attributes - return attribs + with open(str(filepath)) as script: + return get_attributes_from_str(script.read()) + +def get_attributes_from_str(code): + results = funcregex.findall(code) + if results: + # We'll fake the ptAttribs being all alone in a module... + assigns = ast.parse("\n".join(results)) + v = PlasmaAttributeVisitor() + v.visit(assigns) + if v._attributes: + return v._attributes + return {} if __name__ == "__main__": import json @@ -129,7 +131,7 @@ if __name__ == "__main__": files = Path(readpath).glob("*.py") ptAttribs = {} for scriptFile in files: - attribs = get_attributes(scriptFile) + attribs = get_attributes_from_file(scriptFile) if attribs: ptAttribs[scriptFile.stem] = attribs diff --git a/korman/properties/modifiers/gui.py b/korman/properties/modifiers/gui.py index 62cecdf..2c42f9d 100644 --- a/korman/properties/modifiers/gui.py +++ b/korman/properties/modifiers/gui.py @@ -180,15 +180,16 @@ class PlasmaJournalBookModifier(PlasmaModifierProperties, PlasmaModifierLogicWiz # Assign journal script based on target version journal_pfm = journal_pfms[version] journalnode = nodes.new("PlasmaPythonFileNode") - journalnode.filename = journal_pfm["filename"] - - # Manually add required attributes to the PFM - journal_attribs = journal_pfm["attribs"] - for attr in journal_attribs: - new_attr = journalnode.attributes.add() - new_attr.attribute_id = attr["id"] - new_attr.attribute_type = attr["type"] - new_attr.attribute_name = attr["name"] + with journalnode.NoUpdate(): + journalnode.filename = journal_pfm["filename"] + + # Manually add required attributes to the PFM + journal_attribs = journal_pfm["attribs"] + for attr in journal_attribs: + new_attr = journalnode.attributes.add() + new_attr.attribute_id = attr["id"] + new_attr.attribute_type = attr["type"] + new_attr.attribute_name = attr["name"] journalnode.update() if version == pvPrime: From b2120aba10c90aa393806fe9e7074c0173dd6b25 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Mon, 7 Jan 2019 00:47:14 -0500 Subject: [PATCH 05/16] Add new "nonfatal error" handling for exporter There are some cases where errors, while bad, are not the end of the world. I'm thinking namely about compyling the age python. The age still exports just fine, but the ancillary data is flawed. This new system collects nonfatal errors until the export is done, then raises them all at once. --- korman/exporter/convert.py | 5 +++++ korman/exporter/explosions.py | 9 +++++++++ korman/exporter/logger.py | 31 ++++++++++++++++++++++++++++--- korman/operators/op_export.py | 3 +++ 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/korman/exporter/convert.py b/korman/exporter/convert.py index 81ffc08..36d424f 100644 --- a/korman/exporter/convert.py +++ b/korman/exporter/convert.py @@ -107,6 +107,11 @@ class Exporter: self.report.progress_end() self.report.save() + # Step 5.2: If any nonfatal errors were encountered during the export, we will + # raise them here, now that everything is finished, to draw attention + # to whatever the problem might be. + self.report.raise_errors() + def _bake_static_lighting(self): lighting_method = self._op.lighting_method if lighting_method != "skip": diff --git a/korman/exporter/explosions.py b/korman/exporter/explosions.py index 331c173..48151e3 100644 --- a/korman/exporter/explosions.py +++ b/korman/exporter/explosions.py @@ -13,6 +13,15 @@ # You should have received a copy of the GNU General Public License # along with Korman. If not, see . +class NonfatalExportError(Exception): + def __init__(self, *args, **kwargs): + assert args + if len(args) > 1: + super(Exception, self).__init__(args[0].format(*args[1:], **kwargs)) + else: + super(Exception, self).__init__(args[0]) + + class ExportError(Exception): def __init__(self, value="Undefined Export Error"): super(Exception, self).__init__(value) diff --git a/korman/exporter/logger.py b/korman/exporter/logger.py index 7d4e232..98f96ec 100644 --- a/korman/exporter/logger.py +++ b/korman/exporter/logger.py @@ -14,6 +14,7 @@ # along with Korman. If not, see . from ..korlib import ConsoleToggler +from .explosions import NonfatalExportError from pathlib import Path import threading import time @@ -24,6 +25,7 @@ _MAX_TIME_UNTIL_ELIPSES = 2.0 class _ExportLogger: def __init__(self, print_logs, age_path=None): + self._errors = [] self._porting = [] self._warnings = [] self._age_path = Path(age_path) if age_path is not None else None @@ -42,10 +44,23 @@ class _ExportLogger: def __exit__(self, type, value, traceback): if value is not None: - ConsoleToggler().keep_console = True + ConsoleToggler().keep_console = not isinstance(value, NonfatalExportError) self._file.close() return False + def error(self, *args, **kwargs): + assert args + indent = kwargs.get("indent", 0) + msg = "{}ERROR: {}".format(" " * indent, args[0]) + if len(args) > 1: + msg = msg.format(*args[1:], **kwargs) + if self._file is not None: + self._file.writelines((msg, "\n")) + if self._print_logs: + print(msg) + cache = args[0] if len(args) == 1 else args[0].format(*args[1:]) + self._errors.append(cache) + def msg(self, *args, **kwargs): assert args indent = kwargs.get("indent", 0) @@ -67,7 +82,8 @@ class _ExportLogger: self._file.writelines((msg, "\n")) if self._print_logs: print(msg) - self._porting.append(args[0]) + cache = args[0] if len(args) == 1 else args[0].format(*args[1:]) + self._porting.append(cache) def progress_add_step(self, name): @@ -92,6 +108,14 @@ class _ExportLogger: self.msg("Exporting '{}'", self._age_path.name) self._time_start_overall = time.perf_counter() + def raise_errors(self): + num_errors = len(self._errors) + if num_errors == 1: + raise NonfatalExportError(self._errors[0]) + elif num_errors: + raise NonfatalExportError("""{} errors were encountered during export. Check the export log for more details: + {}""", num_errors, self._file.name) + def save(self): # TODO pass @@ -106,7 +130,8 @@ class _ExportLogger: self._file.writelines((msg, "\n")) if self._print_logs: print(msg) - self._warnings.append(args[0]) + cache = args[0] if len(args) == 1 else args[0].format(*args[1:]) + self._warnings.append(cache) class ExportProgressLogger(_ExportLogger): diff --git a/korman/operators/op_export.py b/korman/operators/op_export.py index a1944d0..fa8040b 100644 --- a/korman/operators/op_export.py +++ b/korman/operators/op_export.py @@ -177,6 +177,9 @@ class PlasmaAgeExportOperator(ExportOperator, bpy.types.Operator): except exporter.ExportError as error: self.report({"ERROR"}, str(error)) return {"CANCELLED"} + except exporter.NonfatalExportError as error: + self.report({"ERROR"}, str(error)) + return {"FINISHED"} else: if self.profile_export: stats_out = path.with_name("{}_profile.log".format(ageName)) From e78978c21a38002f69ab90b7fc3348e92372eec7 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Mon, 7 Jan 2019 00:54:37 -0500 Subject: [PATCH 06/16] Ensure text file data is encoded correctly --- korman/exporter/outfile.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/korman/exporter/outfile.py b/korman/exporter/outfile.py index b98f1f1..5994a7a 100644 --- a/korman/exporter/outfile.py +++ b/korman/exporter/outfile.py @@ -16,6 +16,7 @@ from contextlib import contextmanager import enum from hashlib import md5 +import locale import os from pathlib import Path from PyHSPlasma import * @@ -24,6 +25,8 @@ import time import weakref import zipfile +_encoding = locale.getpreferredencoding(False) + def _hashfile(filename, hasher, block=0xFFFF): with open(str(filename), "rb") as handle: h = hasher() @@ -82,7 +85,10 @@ class _OutputFile: data = handle.read(0xFFFF) return h.digest() elif self.file_data is not None: - return md5(self.file_data).digest() + if isinstance(self.file_data, str): + return md5(self.file_data.encode(_encoding)).digest() + else: + return md5(self.file_data).digest() else: raise RuntimeError() @@ -210,7 +216,8 @@ class OutputFiles: if i.file_path: shutil.copy2(i.file_path, dst_path) elif i.file_data: - with open(dst_path, "wb") as handle: + mode = "w" if isinstance(i.file_data, str) else "wb" + with open(dst_path, mode) as handle: handle.write(i.file_data) os.utime(dst_path, times) else: @@ -259,8 +266,12 @@ class OutputFiles: if i.file_path: zf.write(i.file_path, arcpath) elif i.file_data: + if isinstance(i.file_data, str): + data = i.file_data.encode(_encoding) + else: + data = i.file_data zi = zipfile.ZipInfo(arcpath, export_time) - zf.writestr(zi, i.file_data) + zf.writestr(zi, data) @property def _version(self): From ec2070a0000c38dc5d26f189eaf81121346424c0 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Mon, 7 Jan 2019 01:16:59 -0500 Subject: [PATCH 07/16] Pack the doggone Python in the exporter No, I don't want to talk about it... --- korman/exporter/convert.py | 19 ++++++ korman/exporter/manager.py | 35 ++++++++-- korman/exporter/outfile.py | 133 ++++++++++++++++++++++++++++++------ korman/korlib/python.py | 4 +- korman/nodes/node_python.py | 15 +++- 5 files changed, 177 insertions(+), 29 deletions(-) diff --git a/korman/exporter/convert.py b/korman/exporter/convert.py index 36d424f..cebc788 100644 --- a/korman/exporter/convert.py +++ b/korman/exporter/convert.py @@ -62,6 +62,7 @@ class Exporter: self.report.progress_add_step("Exporting Scene Objects") self.report.progress_add_step("Exporting Logic Nodes") self.report.progress_add_step("Finalizing Plasma Logic") + self.report.progress_add_step("Handling Snakes") self.report.progress_add_step("Exporting Textures") self.report.progress_add_step("Composing Geometry") self.report.progress_add_step("Saving Age Files") @@ -94,6 +95,9 @@ class Exporter: # processing that needs to inspect those objects self._post_process_scene_objects() + # Step 3.3: Ensure any helper Python files are packed + self._pack_ancillary_python() + # Step 4: Finalize... self.mesh.material.finalize() self.mesh.finalize() @@ -346,6 +350,17 @@ class Exporter: proc(self, bl_obj, sceneobject) inc_progress() + def _pack_ancillary_python(self): + texts = bpy.data.texts + self.report.progress_advance() + self.report.progress_range = len(texts) + inc_progress = self.report.progress_increment + + for i in texts: + if i.name.endswith(".py") and self.output.want_py_text(i): + self.output.add_python_code(i.name, text_id=i) + inc_progress() + def _save_age(self): self.report.progress_advance() @@ -369,6 +384,10 @@ class Exporter: def envmap_method(self): return bpy.context.scene.world.plasma_age.envmap_method + @property + def python_method(self): + return bpy.context.scene.world.plasma_age.python_method + @property def texcache_path(self): age = bpy.context.scene.world.plasma_age diff --git a/korman/exporter/manager.py b/korman/exporter/manager.py index dc7775e..c3906c7 100644 --- a/korman/exporter/manager.py +++ b/korman/exporter/manager.py @@ -19,6 +19,7 @@ from PyHSPlasma import * import weakref from . import explosions +from ..plasma_magic import * # These objects have to be in the plSceneNode pool in order to be loaded... # NOTE: We are using Factory indices because I doubt all of these classes are implemented. @@ -136,15 +137,19 @@ class ExportManager: def create_builtins(self, age, textures): # BuiltIn.prp if bpy.context.scene.world.plasma_age.age_sdl: - builtin = self.create_page(age, "BuiltIn", -2, True) - sdl = self.add_object(plSceneObject, name="AgeSDLHook", loc=builtin) - pfm = self.add_object(plPythonFileMod, name="VeryVerySpecialPythonFileMod", so=sdl) - pfm.filename = age + self._create_builtin_pages(age) + self._pack_agesdl_hook(age) # Textures.prp if textures: self.create_page(age, "Textures", -1, True) + def _create_builtin_pages(self, age): + builtin = self.create_page(age, "BuiltIn", -2, True) + sdl = self.add_object(plSceneObject, name="AgeSDLHook", loc=builtin) + pfm = self.add_object(plPythonFileMod, name="VeryVerySpecialPythonFileMod", so=sdl) + pfm.filename = age + def create_page(self, age, name, id, builtin=False): location = plLocation(self.mgr.getVer()) location.prefix = bpy.context.scene.world.plasma_age.seq_prefix @@ -233,6 +238,28 @@ class ExportManager: else: return key.location + def _pack_agesdl_hook(self, age): + get_text = bpy.data.texts.get + output = self._exporter().output + + # AgeSDL Hook Python + py_filename = "{}.py".format(age) + age_py = get_text(py_filename, None) + if output.want_py_text(age_py): + py_code = age_py.as_string() + else: + py_code = very_very_special_python.format(age_name=age).lstrip() + output.add_python_mod(py_filename, text_id=age_py, str_data=py_code) + + # AgeSDL + sdl_filename = "{}.sdl".format(age) + age_sdl = get_text(sdl_filename) + if age_sdl is not None: + sdl_code = None + else: + sdl_code = very_very_special_sdl.format(age_name=age).lstrip() + output.add_sdl(sdl_filename, text_id=age_sdl, str_data=sdl_code) + def save_age(self): self._write_age() self._write_fni() diff --git a/korman/exporter/outfile.py b/korman/exporter/outfile.py index 5994a7a..1a8639f 100644 --- a/korman/exporter/outfile.py +++ b/korman/exporter/outfile.py @@ -16,9 +16,11 @@ from contextlib import contextmanager import enum from hashlib import md5 +from .. import korlib import locale import os from pathlib import Path +from ..plasma_magic import plasma_python_glue from PyHSPlasma import * import shutil import time @@ -40,6 +42,9 @@ def _hashfile(filename, hasher, block=0xFFFF): class _FileType(enum.Enum): generated_dat = 0 sfx = 1 + sdl = 2 + python_code = 3 + generated_ancillary = 4 class _OutputFile: @@ -48,8 +53,9 @@ class _OutputFile: self.dirname = kwargs.get("dirname") self.filename = kwargs.get("filename") self.skip_hash = kwargs.get("skip_hash", False) + self.internal = kwargs.get("internal", False) - if self.file_type == _FileType.generated_dat: + if self.file_type in (_FileType.generated_dat, _FileType.generated_ancillary): self.file_data = kwargs.get("file_data", None) self.file_path = kwargs.get("file_path", None) self.mod_time = Path(self.file_path).stat().st_mtime if self.file_path else None @@ -69,6 +75,23 @@ class _OutputFile: if self.id_data.packed_file is not None: self.file_data = self.id_data.packed_file.data + if self.file_type in (_FileType.sdl, _FileType.python_code): + self.id_data = kwargs.get("id_data") + self.file_data = kwargs.get("file_data") + self.needs_glue = kwargs.get("needs_glue", True) + assert bool(self.id_data) or bool(self.file_data) + + self.mod_time = None + self.file_path = None + if self.id_data is not None: + path = Path(self.id_data.filepath) + if path.exists(): + self.mod_time = path.stat().st_mtime + self.file_path = self.id_data.filepath + + if self.file_data is None: + self.file_data = self.id_data.as_string() + def __eq__(self, rhs): return str(self) == str(rhs) @@ -76,7 +99,7 @@ class _OutputFile: return hash(str(self)) def hash_md5(self): - if self.file_path is not None: + if self.file_path: with open(self.file_path, "rb") as handle: h = md5() data = handle.read(0xFFFF) @@ -106,28 +129,36 @@ class OutputFiles: self._export_path = self._export_file.parent.parent self._files = set() self._is_zip = self._export_file.suffix.lower() == ".zip" + self._py_files = set() self._time = time.time() - def add_python(self, filename, text_id=None, str_data=None): + def add_python_code(self, filename, text_id=None, str_data=None): + assert filename not in self._py_files of = _OutputFile(file_type=_FileType.python_code, dirname="Python", filename=filename, id_data=text_id, file_data=str_data, - skip_hash=True) + skip_hash=True, + internal=(self._version != pvMoul), + needs_glue=False) self._files.add(of) + self._py_files.add(filename) - def add_sdl(self, filename, text_id=None, str_data=None): - version = self._version - if version == pvEoa: - enc = plEncryptedStream.kEncAes - elif version == pvMoul: - enc = None - else: - enc = plEncryptedStream.kEncXtea + def add_python_mod(self, filename, text_id=None, str_data=None): + assert filename not in self._py_files + of = _OutputFile(file_type=_FileType.python_code, + dirname="Python", filename=filename, + id_data=text_id, file_data=str_data, + skip_hash=True, + internal=(self._version != pvMoul), + needs_glue=True) + self._files.add(of) + self._py_files.add(filename) + def add_sdl(self, filename, text_id=None, str_data=None): of = _OutputFile(file_type=_FileType.sdl, dirname="SDL", filename=filename, id_data=text_id, file_data=str_data, - enc=enc) + enc=self.super_secure_encryption) self._files.add(of) @@ -165,11 +196,14 @@ class OutputFiles: if not stream is backing_stream: backing_stream.close() + dirname = kwargs.get("dirname", "dat") kwargs = { - "file_type": _FileType.generated_dat, - "dirname": "dat", + "file_type": _FileType.generated_dat if dirname == "dat" else + _FileType.generated_ancillary, + "dirname": dirname, "filename": filename, - "skip_hash": skip_hash, + "skip_hash": kwargs.get("skip_hash", False), + "internal": kwargs.get("internal", False), } if isinstance(backing_stream, hsRAMStream): kwargs["file_data"] = backing_stream.buffer @@ -188,13 +222,46 @@ class OutputFiles: else: yield i + def _package_compyled_python(self): + func = lambda x: x.file_type == _FileType.python_code + report = self._exporter().report + version = self._version + + # There can be some debate about what the correct Python version for pvMoul is. + # I, quite frankly, don't give a rat's ass at the moment because CWE will only + # load Python.pak and no ancillary packages. Maybe someone should fix that, mm? + if version <= pvPots: + py_version = (2, 2) + else: + py_version = (2, 3) + + try: + pyc_objects = [] + for i in self._generate_files(func): + if i.needs_glue: + py_code = "{}\n\n{}\n".format(i.file_data, plasma_python_glue) + else: + py_code = i.file_data + result, pyc = korlib.compyle(i.filename, py_code, py_version, report, indent=1) + if result: + pyc_objects.append((i.filename, pyc)) + except korlib.PythonNotAvailableError as error: + report.warn("Python {} is not available. Your Age scripts were not packaged.", error, indent=1) + else: + if pyc_objects: + with self.generate_dat_file("{}.pak".format(self._exporter().age_name), + dirname="Python", enc=self.super_secure_encryption) as stream: + korlib.package_python(stream, pyc_objects) + def save(self): # At this stage, all Plasma data has been generated from whatever crap is in # Blender. The only remaining part is to make sure any external dependencies are # copied or packed into the appropriate format OR the asset hashes are generated. + version = self._version # Step 1: Handle Python - # ... todo ... + if self._exporter().python_method != "none" and version != pvMoul: + self._package_compyled_python() # Step 2: Generate sumfile if self._version != pvMoul: @@ -206,9 +273,31 @@ class OutputFiles: else: self._write_deps() + @property + def super_secure_encryption(self): + version = self._version + if version == pvEoa: + return plEncryptedStream.kEncAes + elif version == pvMoul: + # trollface.jpg + return None + else: + return plEncryptedStream.kEncXtea + + def want_py_text(self, text_id): + if text_id is None: + return False + method = self._exporter().python_method + if method == "none": + return False + elif method == "all": + return text_id.name not in self._py_files + else: + return text_id.plasma_text.package and text_id.name not in self._py_files + def _write_deps(self): - func = lambda x: x.file_type == _FileType.sfx times = (self._time, self._time) + func = lambda x: not x.internal and x.file_type not in (_FileType.generated_ancillary, _FileType.generated_dat) for i in self._generate_files(func): # Will only ever run for non-"dat" directories. @@ -229,9 +318,9 @@ class OutputFiles: enc = plEncryptedStream.kEncAes if version >= pvEoa else plEncryptedStream.kEncXtea filename = "{}.sum".format(self._exporter().age_name) if dat_only: - func = lambda x: not x.skip_hash and x.dirname == "dat" + func = lambda x: (not x.skip_hash and not x.internal) and x.dirname == "dat" else: - func = lambda x: not x.skip_hash + func = lambda x: not x.skip_hash and not x.internal with self.generate_dat_file(filename, enc=enc, skip_hash=True) as stream: files = list(self._generate_files(func)) @@ -256,9 +345,9 @@ class OutputFiles: dat_only = self._exporter().dat_only export_time = time.localtime(self._time)[:6] if dat_only: - func = lambda x: x.dirname == "dat" + func = lambda x: x.dirname == "dat" and not x.internal else: - func = None + func = lambda x: not x.internal with zipfile.ZipFile(str(self._export_file), 'w', zipfile.ZIP_DEFLATED) as zf: for i in self._generate_files(func): diff --git a/korman/korlib/python.py b/korman/korlib/python.py index 0a7e219..7d3ae70 100644 --- a/korman/korlib/python.py +++ b/korman/korlib/python.py @@ -98,10 +98,10 @@ def _find_python(py_version): except ImportError: pass else: - py_executable = _find_python_reg(winreg.HKEY_LOCAL_MACHINE, py_version) + py_executable = _find_python_reg(winreg.HKEY_CURRENT_USER, py_version) if _verify_python(py_version, py_executable): return py_executable - py_executable = _find_python_reg(winreg.HKEY_CURRENT_USER, py_version) + py_executable = _find_python_reg(winreg.HKEY_LOCAL_MACHINE, py_version) if _verify_python(py_version, py_executable): return py_executable diff --git a/korman/nodes/node_python.py b/korman/nodes/node_python.py index 8d9b646..26ef9e1 100644 --- a/korman/nodes/node_python.py +++ b/korman/nodes/node_python.py @@ -246,7 +246,20 @@ class PlasmaPythonFileNode(PlasmaVersionedNode, bpy.types.Node): def export(self, exporter, bo, so): pfm = self.get_key(exporter, so).object - pfm.filename = Path(self.filename).stem + py_name = Path(self.filename).stem + pfm.filename = py_name + + # Check to see if we should pack this file + if exporter.output.want_py_text(self.text_id): + exporter.report.msg("Including Python '{}' for package", self.filename, indent=3) + exporter.output.add_python_mod(self.filename, text_id=self.text_id) + # PFMs can have their own SDL... + sdl_text = bpy.data.texts.get("{}.sdl".format(py_name), None) + if sdl_text is not None: + exporter.report.msg("Including corresponding SDL '{}'", sdl_text.name, indent=3) + exporter.output.add_sdl(sdl_text.name, text_id=sdl_text) + + # Handle exporting the Python Parameters attrib_sockets = (i for i in self.inputs if i.is_linked) for socket in attrib_sockets: attrib = socket.attribute_type From 46889aca36a1d03886c07787b1a61fabd19b6952 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Mon, 7 Jan 2019 14:00:37 -0500 Subject: [PATCH 08/16] Add Python 2.2 to the installer --- installer/Installer.nsi | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/installer/Installer.nsi b/installer/Installer.nsi index 8b96969..c172dcd 100644 --- a/installer/Installer.nsi +++ b/installer/Installer.nsi @@ -211,6 +211,14 @@ SectionGroup /e "Korman" SectionEnd SectionGroupEnd +Section "Python 2.2" + SectionIn 1 2 + + SetOutPath "$TEMP\Korman" + File "Files\x86\Python-2.2.3.exe" + ExecWait "$TEMP\Korman\Python-2.2.3.exe /S" +SectionEnd + Section #TheRemover WriteRegStr HKLM "Software\Korman" "" $INSTDIR WriteUninstaller "$INSTDIR\korman_uninstall.exe" From 09d5fbdeb74608bf9a58c71a2f2cd13e01d83ccb Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Mon, 7 Jan 2019 14:16:17 -0500 Subject: [PATCH 09/16] Validate user specified Python installs in the UI This adds a user visible hint to whether or not the Python executable (or outright junk) they specify is even valid. --- korman/addon_prefs.py | 64 +++++++++++++++++++++++++++++++++++++---- korman/korlib/python.py | 8 +++--- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/korman/addon_prefs.py b/korman/addon_prefs.py index 1155e57..cd6b3be 100644 --- a/korman/addon_prefs.py +++ b/korman/addon_prefs.py @@ -15,6 +15,7 @@ import bpy from bpy.props import * +from . import korlib game_versions = [("pvPrime", "Ages Beyond Myst (63.11)", "Targets the original Uru (Live) game"), ("pvPots", "Path of the Shell (63.12)", "Targets the most recent offline expansion pack"), @@ -39,18 +40,62 @@ class KormanAddonPreferences(bpy.types.AddonPreferences): games = CollectionProperty(type=PlasmaGame) active_game_index = IntProperty(options={"SKIP_SAVE"}) + def _check_py22_exe(self, context): + if self._ensure_abspath((2, 2)): + self._check_python((2, 2)) + def _check_py23_exe(self, context): + if self._ensure_abspath((2, 3)): + self._check_python((2, 3)) + def _check_py27_exe(self, context): + if self._ensure_abspath((2, 7)): + self._check_python((2, 7)) + python22_executable = StringProperty(name="Python 2.2", description="Path to the Python 2.2 executable", options=set(), - subtype="FILE_PATH") + subtype="FILE_PATH", + update=_check_py22_exe) python23_executable = StringProperty(name="Python 2.3", description="Path to the Python 2.3 executable", options=set(), - subtype="FILE_PATH") + subtype="FILE_PATH", + update=_check_py23_exe) python27_executable = StringProperty(name="Python 2.7", description="Path to the Python 2.7 executable", options=set(), - subtype="FILE_PATH") + subtype="FILE_PATH", + update=_check_py27_exe) + + def _validate_py_exes(self): + if not self.is_property_set("python22_valid"): + self._check_python((2, 2)) + if not self.is_property_set("python23_valid"): + self._check_python((2, 3)) + if not self.is_property_set("python27_valid"): + self._check_python((2, 7)) + return True + + # Internal error states + python22_valid = BoolProperty(options={"HIDDEN", "SKIP_SAVE"}) + python23_valid = BoolProperty(options={"HIDDEN", "SKIP_SAVE"}) + python27_valid = BoolProperty(options={"HIDDEN", "SKIP_SAVE"}) + python_validated = BoolProperty(get=_validate_py_exes, options={"HIDDEN", "SKIP_SAVE"}) + + def _check_python(self, py_version): + py_exe = getattr(self, "python{}{}_executable".format(*py_version)) + if py_exe: + valid = korlib.verify_python(py_version, py_exe) + else: + valid = True + setattr(self, "python{}{}_valid".format(*py_version), valid) + + def _ensure_abspath(self, py_version): + attr = "python{}{}_executable".format(*py_version) + path = getattr(self, attr) + if path.startswith("//"): + setattr(self, attr, bpy.path.abspath(path)) + return False + return True def draw(self, context): layout = self.layout @@ -67,11 +112,18 @@ class KormanAddonPreferences(bpy.types.AddonPreferences): col.operator("world.plasma_game_convert", icon="IMPORT", text="") # Python Installs + assert self.python_validated main_col = split.column() main_col.label("Python Executables:") - main_col.prop(self, "python22_executable") - main_col.prop(self, "python23_executable") - main_col.prop(self, "python27_executable") + col = main_col.column() + col.alert = not self.python22_valid + col.prop(self, "python22_executable") + col = main_col.column() + col.alert = not self.python23_valid + col.prop(self, "python23_executable") + col = main_col.column() + col.alert = not self.python27_valid + col.prop(self, "python27_executable") # Game Properties active_game_index = self.active_game_index diff --git a/korman/korlib/python.py b/korman/korlib/python.py index 7d3ae70..f2033a9 100644 --- a/korman/korlib/python.py +++ b/korman/korlib/python.py @@ -89,7 +89,7 @@ def _find_python(py_version): else: userprefs = bpy.context.user_preferences.addons["korman"].preferences py_executable = getattr(userprefs, "python{}{}_executable".format(*py_version), None) - if _verify_python(py_version, py_executable): + if verify_python(py_version, py_executable): return py_executable # Second, try looking Python up in the registry. @@ -99,10 +99,10 @@ def _find_python(py_version): pass else: py_executable = _find_python_reg(winreg.HKEY_CURRENT_USER, py_version) - if _verify_python(py_version, py_executable): + if verify_python(py_version, py_executable): return py_executable py_executable = _find_python_reg(winreg.HKEY_LOCAL_MACHINE, py_version) - if _verify_python(py_version, py_executable): + if verify_python(py_version, py_executable): return py_executable # I give up, you win. @@ -163,7 +163,7 @@ def package_python(stream, pyc_objects): stream.writeInt(len(compyled_code)) stream.write(compyled_code) -def _verify_python(py_version, py_exe): +def verify_python(py_version, py_exe): if not py_exe: return False From 1bee0b17366c3de363ca9b8710b63bd9f64a445d Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Mon, 7 Jan 2019 15:12:47 -0500 Subject: [PATCH 10/16] Ensure "secure" files are encrypted --- korman/exporter/outfile.py | 44 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/korman/exporter/outfile.py b/korman/exporter/outfile.py index 1a8639f..78ff054 100644 --- a/korman/exporter/outfile.py +++ b/korman/exporter/outfile.py @@ -27,6 +27,7 @@ import time import weakref import zipfile +_CHUNK_SIZE = 0xA00000 _encoding = locale.getpreferredencoding(False) def _hashfile(filename, hasher, block=0xFFFF): @@ -92,6 +93,42 @@ class _OutputFile: if self.file_data is None: self.file_data = self.id_data.as_string() + # Last chance for encryption... + enc = kwargs.get("enc", None) + if enc is not None: + self._encrypt(enc) + + def _encrypt(self, enc): + backing_stream = hsRAMStream() + with plEncryptedStream().open(backing_stream, fmCreate, enc) as enc_stream: + if self.file_path: + if plEncryptedStream.IsFileEncrypted(self.file_path): + with plEncryptedStream().open(self.file_path, fmRead, plEncryptedStream.kEncAuto) as dec_stream: + self._enc_spin_wash(enc_stream, dec_stream) + else: + with hsFileStream().open(self.file_path, fmRead) as dec_stream: + self._enc_spin_wash(enc_stream, dec_stream) + elif self.file_data: + if isinstance(self.file_data, str): + enc_stream.write(self.file_data.encode(_encoding)) + else: + enc_stream.write(self.file_data) + else: + raise RuntimeError() + + self.file_data = backing_stream.buffer + # do NOT copy over an unencrypted file... + self.file_path = None + + def _enc_spin_wash(self, enc_stream, dec_stream): + while True: + size_rem = dec_stream.size - dec_stream.pos + readsz = min(size_rem, _CHUNK_SIZE) + if readsz == 0: + break + data = dec_stream.read(readsz) + enc_stream.write(data) + def __eq__(self, rhs): return str(self) == str(rhs) @@ -102,10 +139,10 @@ class _OutputFile: if self.file_path: with open(self.file_path, "rb") as handle: h = md5() - data = handle.read(0xFFFF) + data = handle.read(_CHUNK_SIZE) while data: h.update(data) - data = handle.read(0xFFFF) + data = handle.read(_CHUNK_SIZE) return h.digest() elif self.file_data is not None: if isinstance(self.file_data, str): @@ -196,6 +233,9 @@ class OutputFiles: if not stream is backing_stream: backing_stream.close() + # Not passing enc as a keyword argument to the output file definition. It makes more + # sense to yield an encrypted stream from this context manager and encrypt as we go + # instead of doing lots of buffer copying to encrypt as a post step. dirname = kwargs.get("dirname", "dat") kwargs = { "file_type": _FileType.generated_dat if dirname == "dat" else From 4b5f061ac6f5332306b761deedf0a93054ff0cce Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Tue, 8 Jan 2019 17:55:53 -0500 Subject: [PATCH 11/16] Fix Python paks being written into "dat" --- korman/exporter/outfile.py | 49 ++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/korman/exporter/outfile.py b/korman/exporter/outfile.py index 78ff054..4d39cb4 100644 --- a/korman/exporter/outfile.py +++ b/korman/exporter/outfile.py @@ -207,18 +207,27 @@ class OutputFiles: @contextmanager def generate_dat_file(self, filename, **kwargs): - if self._is_zip: + dat_only = self._exporter().dat_only + dirname = kwargs.get("dirname", "dat") + bogus = dat_only and dirname != "dat" + + if self._is_zip or bogus: stream = hsRAMStream(self._version) else: - file_path = str(self._export_file.parent / filename) + if dat_only: + file_path = str(self._export_file.parent / filename) + else: + file_path = str(self._export_path / dirname / filename) stream = hsFileStream(self._version) stream.open(file_path, fmCreate) backing_stream = stream - enc = kwargs.get("enc", None) - if enc is not None: - stream = plEncryptedStream(self._version) - stream.open(backing_stream, fmCreate, enc) + # No sense in wasting time encrypting data that isn't going to be used in the export + if not bogus: + enc = kwargs.get("enc", None) + if enc is not None: + stream = plEncryptedStream(self._version) + stream.open(backing_stream, fmCreate, enc) # The actual export code is run at the "yield" statement. If an error occurs, we # do not want to track this file. Note that the except block is required for the @@ -236,20 +245,20 @@ class OutputFiles: # Not passing enc as a keyword argument to the output file definition. It makes more # sense to yield an encrypted stream from this context manager and encrypt as we go # instead of doing lots of buffer copying to encrypt as a post step. - dirname = kwargs.get("dirname", "dat") - kwargs = { - "file_type": _FileType.generated_dat if dirname == "dat" else - _FileType.generated_ancillary, - "dirname": dirname, - "filename": filename, - "skip_hash": kwargs.get("skip_hash", False), - "internal": kwargs.get("internal", False), - } - if isinstance(backing_stream, hsRAMStream): - kwargs["file_data"] = backing_stream.buffer - else: - kwargs["file_path"] = file_path - self._files.add(_OutputFile(**kwargs)) + if not bogus: + kwargs = { + "file_type": _FileType.generated_dat if dirname == "dat" else + _FileType.generated_ancillary, + "dirname": dirname, + "filename": filename, + "skip_hash": kwargs.get("skip_hash", False), + "internal": kwargs.get("internal", False), + } + if isinstance(backing_stream, hsRAMStream): + kwargs["file_data"] = backing_stream.buffer + else: + kwargs["file_path"] = file_path + self._files.add(_OutputFile(**kwargs)) def _generate_files(self, func=None): dat_only = self._exporter().dat_only From 6929c37ed7e1d4e64baed80b39c57406286ac840 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Tue, 8 Jan 2019 19:29:20 -0500 Subject: [PATCH 12/16] Don't crash if a dependency file is missing --- korman/exporter/convert.py | 1 + korman/exporter/outfile.py | 33 ++++++++++++++++++++++----------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/korman/exporter/convert.py b/korman/exporter/convert.py index cebc788..a55e18d 100644 --- a/korman/exporter/convert.py +++ b/korman/exporter/convert.py @@ -363,6 +363,7 @@ class Exporter: def _save_age(self): self.report.progress_advance() + self.report.msg("\nWriting Age data...") # If something bad happens in the final flush, it would be a shame to # simply toss away the potentially freshly regenerated texture cache. diff --git a/korman/exporter/outfile.py b/korman/exporter/outfile.py index 4d39cb4..c77f697 100644 --- a/korman/exporter/outfile.py +++ b/korman/exporter/outfile.py @@ -55,6 +55,8 @@ class _OutputFile: self.filename = kwargs.get("filename") self.skip_hash = kwargs.get("skip_hash", False) self.internal = kwargs.get("internal", False) + self.file_path = None + self.mod_time = None if self.file_type in (_FileType.generated_dat, _FileType.generated_ancillary): self.file_data = kwargs.get("file_data", None) @@ -66,13 +68,14 @@ class _OutputFile: if self.file_type == _FileType.sfx: self.id_data = kwargs.get("id_data") - path = Path(self.id_data.filepath).resolve() - if path.exists(): - self.file_path = str(path) - self.mod_time = path.stat().st_mtime - else: - self.file_path = None - self.mod_time = None + path = Path(self.id_data.filepath) + try: + if path.exists(): + self.file_path = str(path.resolve()) + self.mod_time = path.stat().st_mtime + except OSError: + pass + if self.id_data.packed_file is not None: self.file_data = self.id_data.packed_file.data @@ -86,9 +89,12 @@ class _OutputFile: self.file_path = None if self.id_data is not None: path = Path(self.id_data.filepath) - if path.exists(): - self.mod_time = path.stat().st_mtime - self.file_path = self.id_data.filepath + try: + if path.exists(): + self.file_path = str(path.resolve()) + self.mod_time = path.stat().st_mtime + except OSError: + pass if self.file_data is None: self.file_data = self.id_data.as_string() @@ -347,6 +353,7 @@ class OutputFiles: def _write_deps(self): times = (self._time, self._time) func = lambda x: not x.internal and x.file_type not in (_FileType.generated_ancillary, _FileType.generated_dat) + report = self._exporter().report for i in self._generate_files(func): # Will only ever run for non-"dat" directories. @@ -359,7 +366,8 @@ class OutputFiles: handle.write(i.file_data) os.utime(dst_path, times) else: - raise RuntimeError() + report.warn("No data found for dependency file '{}'. It will not be copied into the export directory.", + str(i.dirname / i.filename), indent=1) def _write_sumfile(self): version = self._version @@ -397,6 +405,7 @@ class OutputFiles: func = lambda x: x.dirname == "dat" and not x.internal else: func = lambda x: not x.internal + report = self._exporter().report with zipfile.ZipFile(str(self._export_file), 'w', zipfile.ZIP_DEFLATED) as zf: for i in self._generate_files(func): @@ -410,6 +419,8 @@ class OutputFiles: data = i.file_data zi = zipfile.ZipInfo(arcpath, export_time) zf.writestr(zi, data) + else: + report.warn("No data found for dependency file '{}'. It will not be archived.", arcpath, indent=1) @property def _version(self): From 6c4aedb17af540de9e96967c21cbdc5781da1e11 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Tue, 8 Jan 2019 20:55:20 -0500 Subject: [PATCH 13/16] Replace fancy AgeSDL Python meta hacks Path of the Shell did not like my fancy metaprogramming tricks for defining an AgeSDL Python class that contained characters that are illegal in Python identifiers. So, now, we revert to just using a standard class declaration. That means that we need to strip out any illegal identifiers from the age name first. A legal Python 2.x identifier is constrained to the ASCII alphanumeric characters and the underscore with the stipulation that the first character cannot be a number. To illustrate this to the artist, we alert the age name property field if an illegal character is found in the age name. We also alert on the underscore, which is now used as a very very special replacement character. In the case of an illegal character, an error message is shown in the UI with the correct AgeSDL name. Of course, I hope no one really uses those illegal characters and this is just more fulmination on my part... --- korman/exporter/manager.py | 10 ++++++---- korman/exporter/python.py | 7 ++++--- korman/korlib/__init__.py | 35 +++++++++++++++++++++++++++++++++++ korman/operators/op_export.py | 12 +++++++++++- korman/plasma_magic.py | 3 ++- korman/ui/ui_world.py | 17 ++++++++++++++--- 6 files changed, 72 insertions(+), 12 deletions(-) diff --git a/korman/exporter/manager.py b/korman/exporter/manager.py index c3906c7..380caa6 100644 --- a/korman/exporter/manager.py +++ b/korman/exporter/manager.py @@ -19,6 +19,7 @@ from PyHSPlasma import * import weakref from . import explosions +from .. import korlib from ..plasma_magic import * # These objects have to be in the plSceneNode pool in order to be loaded... @@ -243,21 +244,22 @@ class ExportManager: output = self._exporter().output # AgeSDL Hook Python - py_filename = "{}.py".format(age) + fixed_agename = korlib.replace_python2_identifier(age) + py_filename = "{}.py".format(fixed_agename) age_py = get_text(py_filename, None) if output.want_py_text(age_py): py_code = age_py.as_string() else: - py_code = very_very_special_python.format(age_name=age).lstrip() + py_code = very_very_special_python.format(age_name=fixed_agename).lstrip() output.add_python_mod(py_filename, text_id=age_py, str_data=py_code) # AgeSDL - sdl_filename = "{}.sdl".format(age) + sdl_filename = "{}.sdl".format(fixed_agename) age_sdl = get_text(sdl_filename) if age_sdl is not None: sdl_code = None else: - sdl_code = very_very_special_sdl.format(age_name=age).lstrip() + sdl_code = very_very_special_sdl.format(age_name=fixed_agename).lstrip() output.add_sdl(sdl_filename, text_id=age_sdl, str_data=sdl_code) def save_age(self): diff --git a/korman/exporter/python.py b/korman/exporter/python.py index e263b9f..ef87718 100644 --- a/korman/exporter/python.py +++ b/korman/exporter/python.py @@ -80,7 +80,8 @@ class PythonPackageExporter: def _ensure_age_sdl_hook(self, report): age_props = bpy.context.scene.world.plasma_age if age_props.age_sdl: - py_filename = "{}.py".format(age_props.age_name) + fixed_agename = korlib.replace_python2_identifier(age_props.age_name) + py_filename = "{}.py".format(fixed_agename) age_py = self._modules.get(py_filename) if age_py is not None: del self._modules[py_filename] @@ -88,11 +89,11 @@ class PythonPackageExporter: self._pfms[py_filename] = age_py else: report.warn("AgeSDL Python Script provided, but not requested for packing... Using default Python.", indent=1) - self._pfms[py_filename] = very_very_special_python.format(age_name=age_props.age_name) + self._pfms[py_filename] = very_very_special_python.format(age_name=fixed_agename) else: report.msg("Packing default AgeSDL Python", indent=1) very_very_special_python.format(age_name=age_props.age_name) - self._pfms[py_filename] = very_very_special_python.format(age_name=age_props.age_name) + self._pfms[py_filename] = very_very_special_python.format(age_name=fixed_agename) def _harvest_pfms(self, report): objects = bpy.context.scene.objects diff --git a/korman/korlib/__init__.py b/korman/korlib/__init__.py index 3e03574..78a0fe9 100644 --- a/korman/korlib/__init__.py +++ b/korman/korlib/__init__.py @@ -76,6 +76,13 @@ finally: from .python import * from .texture import TEX_DETAIL_ALPHA, TEX_DETAIL_ADD, TEX_DETAIL_MULTIPLY + _IDENTIFIER_RANGES = ((ord('0'), ord('9')), (ord('A'), ord('Z')), (ord('a'), ord('z'))) + from keyword import kwlist as _kwlist + _KEYWORDS = set(_kwlist) + # Python 2.x keywords + _KEYWORDS.add("exec") + _KEYWORDS.add("print") + def _wave_chunks(stream): while not stream.eof(): chunk_name = stream.read(4) @@ -101,3 +108,31 @@ finally: header.read(stream) return chunks[b"data"]["size"] + + def is_legal_python2_identifier(identifier): + if not identifier: + return False + + # FIXME: str.isascii in Python 3.7 + if any(ord(i) > 0x7F for i in identifier): + return False + if is_python_keyword(identifier): + return False + return identifier.isidentifier() + + is_python_keyword = _KEYWORDS.__contains__ + + def replace_python2_identifier(identifier): + """Replaces illegal characters in a Python identifier with a replacement character""" + + def process(identifier): + # No leading digits in identifiers, so skip the first range element (0...9) + yield next((identifier[0] for low, high in _IDENTIFIER_RANGES[1:] + if low <= ord(identifier[0]) <= high), '_') + for i in identifier[1:]: + yield next((i for low, high in _IDENTIFIER_RANGES if low <= ord(i) <= high), '_') + + if identifier: + return "".join(process(identifier)) + else: + return "" diff --git a/korman/operators/op_export.py b/korman/operators/op_export.py index fa8040b..b83ea5c 100644 --- a/korman/operators/op_export.py +++ b/korman/operators/op_export.py @@ -163,8 +163,12 @@ class PlasmaAgeExportOperator(ExportOperator, bpy.types.Operator): if context.mode != "OBJECT": bpy.ops.object.mode_set(mode="OBJECT") - # Separate blender operator and actual export logic for my sanity ageName = path.stem + if korlib.is_python_keyword(ageName): + self.report({"ERROR"}, "The Age name conflicts with the Python keyword '{}'".format(ageName)) + return {"CANCELLED"} + + # Separate blender operator and actual export logic for my sanity with UiHelper(context) as _ui: e = exporter.Exporter(self) try: @@ -256,6 +260,12 @@ class PlasmaPythonExportOperator(ExportOperator, bpy.types.Operator): return {"CANCELLED"} path.touch() + # Age names cannot be python keywords + age_name = context.scene.world.plasma_age.age_name + if korlib.is_python_keyword(age_name): + self.report({"ERROR"}, "The Age name conflicts with the Python keyword '{}'".format(age_name)) + return {"CANCELLED"} + # Bonus Fun: Implement Profile-mode here (later...) e = exporter.PythonPackageExporter(filepath=self.filepath, version=globals()[self.version]) diff --git a/korman/plasma_magic.py b/korman/plasma_magic.py index b67d438..ed89987 100644 --- a/korman/plasma_magic.py +++ b/korman/plasma_magic.py @@ -17,7 +17,8 @@ very_very_special_python = """ from Plasma import * from PlasmaTypes import * -globals()["{age_name}"] = type("{age_name}", (ptResponder,), dict()) +class {age_name}(ptResponder): + pass """ very_very_special_sdl = """ diff --git a/korman/ui/ui_world.py b/korman/ui/ui_world.py index 938f0f3..6618bfb 100644 --- a/korman/ui/ui_world.py +++ b/korman/ui/ui_world.py @@ -16,7 +16,7 @@ import bpy from pathlib import Path -from ..korlib import ConsoleToggler +from .. import korlib class AgeButtonsPanel: @@ -128,6 +128,9 @@ class PlasmaAgePanel(AgeButtonsPanel, bpy.types.Panel): col.prop(active_page, "seq_suffix") col.prop_menu_enum(active_page, "version") + # Age Names should really be legal Python 2.x identifiers for AgeSDLHooks + legal_identifier = korlib.is_legal_python2_identifier(age.age_name) + # Core settings layout.separator() split = layout.split() @@ -140,15 +143,23 @@ class PlasmaAgePanel(AgeButtonsPanel, bpy.types.Panel): col = split.column() col.label("Age Settings:") col.prop(age, "seq_prefix", text="ID") - col.alert = not age.age_name.strip() + col.alert = not legal_identifier or '_' in age.age_name col.prop(age, "age_name", text="") + # Display a hint if the identifier is illegal + if not legal_identifier: + if korlib.is_python_keyword(age.age_name): + layout.label(text="Ages should not be named the same as a Python keyword", icon="ERROR") + elif age.age_sdl: + fixed_identifier = korlib.replace_python2_identifier(age.age_name) + layout.label(text="Age's SDL will use the name '{}'".format(fixed_identifier), icon="ERROR") + layout.separator() split = layout.split() col = split.column() col.label("Export Settings:") - col.enabled = ConsoleToggler.is_platform_supported() + col.enabled = korlib.ConsoleToggler.is_platform_supported() col.prop(age, "verbose") col.prop(age, "show_console") From e4e1e97eb4aebdac8c7836cab317c67e87efcfc6 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Thu, 10 Jan 2019 19:19:13 -0500 Subject: [PATCH 14/16] Allow generation of empty Python paks --- korman/exporter/python.py | 3 +++ korman/korlib/python.py | 4 +++- korman/operators/op_export.py | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/korman/exporter/python.py b/korman/exporter/python.py index ef87718..e397d95 100644 --- a/korman/exporter/python.py +++ b/korman/exporter/python.py @@ -149,9 +149,12 @@ class PythonPackageExporter: # DONE report.progress_end() + report.raise_errors() def _package_python(self, report): py_code = self._compyle(report) + if not py_code: + report.error("No Python files were packaged.") self._write_python_pak(py_code, report) def _write_python_pak(self, py_code, report): diff --git a/korman/korlib/python.py b/korman/korlib/python.py index f2033a9..5869539 100644 --- a/korman/korlib/python.py +++ b/korman/korlib/python.py @@ -132,7 +132,9 @@ def package_python(stream, pyc_objects): # ~~~~~ # uint32_t filesz # uint8_t data[filesz] - assert bool(pyc_objects) + if not pyc_objects: + stream.writeInt(0) + return # `stream` might be a plEncryptedStream, which doesn't seek very well at all. # Therefore, we will go ahead and calculate the size of the index block so diff --git a/korman/operators/op_export.py b/korman/operators/op_export.py index b83ea5c..e4df68e 100644 --- a/korman/operators/op_export.py +++ b/korman/operators/op_export.py @@ -277,6 +277,9 @@ class PlasmaPythonExportOperator(ExportOperator, bpy.types.Operator): except korlib.PythonNotAvailableError as error: self.report({"ERROR"}, "Python Version {} not found".format(error)) return {"CANCELLED"} + except exporter.NonfatalExportError as error: + self.report({"WARNING"}, str(error)) + return {"FINISHED"} else: return {"FINISHED"} From 574aa8230172770ca5d841ac110648c7a082748d Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Thu, 10 Jan 2019 19:32:21 -0500 Subject: [PATCH 15/16] Group game properties in Korman Preferences --- korman/addon_prefs.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/korman/addon_prefs.py b/korman/addon_prefs.py index cd6b3be..5d9252b 100644 --- a/korman/addon_prefs.py +++ b/korman/addon_prefs.py @@ -105,33 +105,20 @@ class KormanAddonPreferences(bpy.types.AddonPreferences): main_col.label("Plasma Games:") row = main_col.row() row.template_list("PlasmaGameListRW", "games", self, "games", self, - "active_game_index", rows=2) + "active_game_index", rows=3) col = row.column(align=True) col.operator("world.plasma_game_add", icon="ZOOMIN", text="") col.operator("world.plasma_game_remove", icon="ZOOMOUT", text="") col.operator("world.plasma_game_convert", icon="IMPORT", text="") - # Python Installs - assert self.python_validated - main_col = split.column() - main_col.label("Python Executables:") - col = main_col.column() - col.alert = not self.python22_valid - col.prop(self, "python22_executable") - col = main_col.column() - col.alert = not self.python23_valid - col.prop(self, "python23_executable") - col = main_col.column() - col.alert = not self.python27_valid - col.prop(self, "python27_executable") - # Game Properties active_game_index = self.active_game_index if bool(self.games) and active_game_index < len(self.games): active_game = self.games[active_game_index] - layout.label("Game Configuration:") - box = layout.box().column() + col = split.column() + col.label("Game Configuration:") + box = col.box().column() box.prop(active_game, "path", emboss=False) box.prop(active_game, "version") @@ -142,6 +129,17 @@ class KormanAddonPreferences(bpy.types.AddonPreferences): op.filepath = active_game.path op.game_index = active_game_index + # Python Installs + assert self.python_validated + col = layout.column() + col.label("Python Executables:") + col.alert = not self.python22_valid + col.prop(self, "python22_executable") + col.alert = not self.python23_valid + col.prop(self, "python23_executable") + col.alert = not self.python27_valid + col.prop(self, "python27_executable") + @classmethod def register(cls): # Register the old-timey per-world Plasma Games for use in the conversion From e650b4eef4cec8386fe1457ba7a0b8d43dc86602 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Thu, 10 Jan 2019 20:30:37 -0500 Subject: [PATCH 16/16] Fix improper usage of files on disk in export If a file's data is already available in Blender, it might be changed. For example, an internal text datablock or a changed text file. We need to use those overrides. --- korman/exporter/outfile.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/korman/exporter/outfile.py b/korman/exporter/outfile.py index c77f697..1842850 100644 --- a/korman/exporter/outfile.py +++ b/korman/exporter/outfile.py @@ -78,6 +78,8 @@ class _OutputFile: if self.id_data.packed_file is not None: self.file_data = self.id_data.packed_file.data + else: + self.file_data = None if self.file_type in (_FileType.sdl, _FileType.python_code): self.id_data = kwargs.get("id_data") @@ -358,13 +360,13 @@ class OutputFiles: for i in self._generate_files(func): # Will only ever run for non-"dat" directories. dst_path = str(self._export_path / i.dirname / i.filename) - if i.file_path: - shutil.copy2(i.file_path, dst_path) - elif i.file_data: + if i.file_data: mode = "w" if isinstance(i.file_data, str) else "wb" with open(dst_path, mode) as handle: handle.write(i.file_data) os.utime(dst_path, times) + elif i.file_path: + shutil.copy2(i.file_path, dst_path) else: report.warn("No data found for dependency file '{}'. It will not be copied into the export directory.", str(i.dirname / i.filename), indent=1) @@ -410,15 +412,15 @@ class OutputFiles: with zipfile.ZipFile(str(self._export_file), 'w', zipfile.ZIP_DEFLATED) as zf: for i in self._generate_files(func): arcpath = i.filename if dat_only else "{}/{}".format(i.dirname, i.filename) - if i.file_path: - zf.write(i.file_path, arcpath) - elif i.file_data: + if i.file_data: if isinstance(i.file_data, str): data = i.file_data.encode(_encoding) else: data = i.file_data zi = zipfile.ZipInfo(arcpath, export_time) zf.writestr(zi, data) + elif i.file_path: + zf.write(i.file_path, arcpath) else: report.warn("No data found for dependency file '{}'. It will not be archived.", arcpath, indent=1)