From 288058aa389e288e0222c7487470c39073ff63ac Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Wed, 2 Jan 2019 16:39:51 -0500 Subject: [PATCH] Refactor output file generation Age output files are now handled in all aspects by a singleton manager. This allows us to track all generated files and external dependency files and ensure they are correctly copied over to the target game... Or not, in the case of an age/prp export from the File > Export menu. Currently only SFX files are handled as an external dependency. TODO are python and SDL files. Further, because we have an output file manager, we can bundle all the files into a zip archive for releasing the age in one step. Wow such amazing. ^_^ --- korman/exporter/convert.py | 26 ++- korman/exporter/manager.py | 55 +++--- korman/exporter/outfile.py | 267 +++++++++++++++++++++++++++ korman/exporter/sumfile.py | 73 -------- korman/operators/op_export.py | 8 +- korman/properties/modifiers/sound.py | 3 + korman/ui/ui_world.py | 8 + 7 files changed, 327 insertions(+), 113 deletions(-) create mode 100644 korman/exporter/outfile.py delete mode 100644 korman/exporter/sumfile.py diff --git a/korman/exporter/convert.py b/korman/exporter/convert.py index 7dc9ae8..81ffc08 100644 --- a/korman/exporter/convert.py +++ b/korman/exporter/convert.py @@ -27,9 +27,9 @@ from . import image from . import logger from . import manager from . import mesh +from . import outfile from . import physics from . import rtlight -from . import sumfile from . import utils class Exporter: @@ -40,10 +40,6 @@ class Exporter: self.node_trees_exported = set() self.want_node_trees = {} - @property - def age_name(self): - return Path(self._op.filepath).stem - def run(self): log = logger.ExportVerboseLogger if self._op.verbose else logger.ExportProgressLogger with ConsoleToggler(self._op.show_console), log(self._op.filepath) as self.report: @@ -53,7 +49,7 @@ class Exporter: self.physics = physics.PhysicsConverter(self) self.light = rtlight.LightConverter(self) self.animation = animation.AnimationConverter(self) - self.sumfile = sumfile.SumFile() + self.output = outfile.OutputFiles(self, self._op.filepath) self.camera = camera.CameraConverter(self) self.image = image.ImageCache(self) @@ -347,8 +343,22 @@ class Exporter: def _save_age(self): self.report.progress_advance() - self.mgr.save_age(Path(self._op.filepath)) - self.image.save() + + # If something bad happens in the final flush, it would be a shame to + # simply toss away the potentially freshly regenerated texture cache. + try: + self.mgr.save_age() + self.output.save() + finally: + self.image.save() + + @property + def age_name(self): + return Path(self._op.filepath).stem + + @property + def dat_only(self): + return self._op.dat_only @property def envmap_method(self): diff --git a/korman/exporter/manager.py b/korman/exporter/manager.py index 6ff74b3..dc7775e 100644 --- a/korman/exporter/manager.py +++ b/korman/exporter/manager.py @@ -233,27 +233,23 @@ class ExportManager: else: return key.location - def save_age(self, path): - ageName = path.stem - sumfile = self._exporter().sumfile - - sumfile.append(path) - self.mgr.WriteAge(str(path), self._age_info) - self._write_fni(path) - self._write_pages(path) - - if self.getVer() != pvMoul: - sumpath = path.with_suffix(".sum") - sumfile.write(sumpath, self.getVer()) - - def _write_fni(self, path): - if self.mgr.getVer() <= pvMoul: - enc = plEncryptedStream.kEncXtea - else: - enc = plEncryptedStream.kEncAES - fname = path.with_suffix(".fni") + def save_age(self): + self._write_age() + self._write_fni() + self._write_pages() + + def _write_age(self): + f = "{}.age".format(self._age_info.name) + output = self._exporter().output + + with output.generate_dat_file(f, enc=plEncryptedStream.kEncAuto) as stream: + self._age_info.writeToStream(stream) - with plEncryptedStream(self.mgr.getVer()).open(str(fname), fmWrite, enc) as stream: + def _write_fni(self): + f = "{}.fni".format(self._age_info.name) + output = self._exporter().output + + with output.generate_dat_file(f, enc=plEncryptedStream.kEncAuto) as stream: fni = bpy.context.scene.world.plasma_fni stream.writeLine("Graphics.Renderer.SetClearColor {} {} {}".format(*fni.clear_color)) if fni.fog_method != "none": @@ -263,17 +259,14 @@ class ExportManager: elif fni.fog_method == "exp2": stream.writeLine("Graphics.Renderer.Fog.SetDefExp2 {} {}".format(fni.fog_end, fni.fog_density)) stream.writeLine("Graphics.Renderer.SetYon {}".format(fni.yon)) - self._exporter().sumfile.append(fname) - def _write_pages(self, path): + def _write_pages(self): + age_name = self._age_info.name + output = self._exporter().output for loc in self._pages.values(): page = self.mgr.FindPage(loc) # not cached because it's C++ owned - # I know that plAgeInfo has its own way of doing this, but we'd have - # to do some looping and stuff. This is easier. - if self.mgr.getVer() <= pvMoul: - chapter = "_District_" - else: - chapter = "_" - f = path.with_name("{}{}{}".format(path.stem, chapter, page.page)).with_suffix(".prp") - self.mgr.WritePage(str(f), page) - self._exporter().sumfile.append(f) + chapter = "_District_" if self.mgr.getVer() <= pvMoul else "_" + f = "{}{}{}.prp".format(age_name, chapter, page.page) + + with output.generate_dat_file(f) as stream: + self.mgr.WritePage(stream, page) diff --git a/korman/exporter/outfile.py b/korman/exporter/outfile.py new file mode 100644 index 0000000..b98f1f1 --- /dev/null +++ b/korman/exporter/outfile.py @@ -0,0 +1,267 @@ +# This file is part of Korman. +# +# Korman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Korman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Korman. If not, see . + +from contextlib import contextmanager +import enum +from hashlib import md5 +import os +from pathlib import Path +from PyHSPlasma import * +import shutil +import time +import weakref +import zipfile + +def _hashfile(filename, hasher, block=0xFFFF): + with open(str(filename), "rb") as handle: + h = hasher() + data = handle.read(block) + while data: + h.update(data) + data = handle.read(block) + return h.digest() + +@enum.unique +class _FileType(enum.Enum): + generated_dat = 0 + sfx = 1 + + +class _OutputFile: + def __init__(self, **kwargs): + self.file_type = kwargs.get("file_type") + self.dirname = kwargs.get("dirname") + self.filename = kwargs.get("filename") + self.skip_hash = kwargs.get("skip_hash", False) + + if self.file_type == _FileType.generated_dat: + self.file_data = kwargs.get("file_data", None) + self.file_path = kwargs.get("file_path", None) + self.mod_time = Path(self.file_path).stat().st_mtime if self.file_path else None + + # need either a data buffer OR a file path + assert bool(self.file_data) ^ bool(self.file_path) + + if self.file_type == _FileType.sfx: + self.id_data = kwargs.get("id_data") + path = Path(self.id_data.filepath).resolve() + if path.exists(): + self.file_path = str(path) + self.mod_time = path.stat().st_mtime + else: + self.file_path = None + self.mod_time = None + if self.id_data.packed_file is not None: + self.file_data = self.id_data.packed_file.data + + def __eq__(self, rhs): + return str(self) == str(rhs) + + def __hash__(self): + return hash(str(self)) + + def hash_md5(self): + if self.file_path is not None: + with open(self.file_path, "rb") as handle: + h = md5() + data = handle.read(0xFFFF) + while data: + h.update(data) + data = handle.read(0xFFFF) + return h.digest() + elif self.file_data is not None: + return md5(self.file_data).digest() + else: + raise RuntimeError() + + def __str__(self): + return "{}/{}".format(self.dirname, self.filename) + + +class OutputFiles: + def __init__(self, exporter, path): + self._exporter = weakref.ref(exporter) + self._export_file = Path(path).resolve() + if exporter.dat_only: + self._export_path = self._export_file.parent + else: + self._export_path = self._export_file.parent.parent + self._files = set() + self._is_zip = self._export_file.suffix.lower() == ".zip" + self._time = time.time() + + def add_python(self, filename, text_id=None, str_data=None): + of = _OutputFile(file_type=_FileType.python_code, + dirname="Python", filename=filename, + id_data=text_id, file_data=str_data, + skip_hash=True) + self._files.add(of) + + def add_sdl(self, filename, text_id=None, str_data=None): + version = self._version + if version == pvEoa: + enc = plEncryptedStream.kEncAes + elif version == pvMoul: + enc = None + else: + enc = plEncryptedStream.kEncXtea + + of = _OutputFile(file_type=_FileType.sdl, + dirname="SDL", filename=filename, + id_data=text_id, file_data=str_data, + enc=enc) + self._files.add(of) + + + def add_sfx(self, sound_id): + of = _OutputFile(file_type=_FileType.sfx, + dirname="sfx", filename=sound_id.name, + id_data=sound_id) + self._files.add(of) + + @contextmanager + def generate_dat_file(self, filename, **kwargs): + if self._is_zip: + stream = hsRAMStream(self._version) + else: + file_path = str(self._export_file.parent / filename) + stream = hsFileStream(self._version) + stream.open(file_path, fmCreate) + backing_stream = stream + + enc = kwargs.get("enc", None) + if enc is not None: + stream = plEncryptedStream(self._version) + stream.open(backing_stream, fmCreate, enc) + + # The actual export code is run at the "yield" statement. If an error occurs, we + # do not want to track this file. Note that the except block is required for the + # else block to be legal. ^_^ + try: + yield stream + except: + raise + else: + # Must call the EncryptedStream close to actually encrypt the data + stream.close() + if not stream is backing_stream: + backing_stream.close() + + kwargs = { + "file_type": _FileType.generated_dat, + "dirname": "dat", + "filename": filename, + "skip_hash": skip_hash, + } + if isinstance(backing_stream, hsRAMStream): + kwargs["file_data"] = backing_stream.buffer + else: + kwargs["file_path"] = file_path + self._files.add(_OutputFile(**kwargs)) + + def _generate_files(self, func=None): + dat_only = self._exporter().dat_only + for i in self._files: + if dat_only and i.dirname != "dat": + continue + if func is not None: + if func(i): + yield i + else: + yield i + + def save(self): + # At this stage, all Plasma data has been generated from whatever crap is in + # Blender. The only remaining part is to make sure any external dependencies are + # copied or packed into the appropriate format OR the asset hashes are generated. + + # Step 1: Handle Python + # ... todo ... + + # Step 2: Generate sumfile + if self._version != pvMoul: + self._write_sumfile() + + # Step 3: Ensure errbody is gut + if self._is_zip: + self._write_zipfile() + else: + self._write_deps() + + def _write_deps(self): + func = lambda x: x.file_type == _FileType.sfx + times = (self._time, self._time) + + for i in self._generate_files(func): + # Will only ever run for non-"dat" directories. + dst_path = str(self._export_path / i.dirname / i.filename) + if i.file_path: + shutil.copy2(i.file_path, dst_path) + elif i.file_data: + with open(dst_path, "wb") as handle: + handle.write(i.file_data) + os.utime(dst_path, times) + else: + raise RuntimeError() + + def _write_sumfile(self): + version = self._version + dat_only = self._exporter().dat_only + enc = plEncryptedStream.kEncAes if version >= pvEoa else plEncryptedStream.kEncXtea + filename = "{}.sum".format(self._exporter().age_name) + if dat_only: + func = lambda x: not x.skip_hash and x.dirname == "dat" + else: + func = lambda x: not x.skip_hash + + with self.generate_dat_file(filename, enc=enc, skip_hash=True) as stream: + files = list(self._generate_files(func)) + stream.writeInt(len(files)) + stream.writeInt(0) + for i in files: + # ABM and UU don't want the directory for PRPs... Bug? + extension = Path(i.filename).suffix.lower() + if extension == ".prp" and version < pvPots: + filename = i.filename + else: + filename = "{}\\{}".format(i.dirname, i.filename) + mod_time = i.mod_time if i.mod_time else self._time + hash_md5 = i.hash_md5() + + stream.writeSafeStr(filename) + stream.write(hash_md5) + stream.writeInt(int(mod_time)) + stream.writeInt(0) + + def _write_zipfile(self): + dat_only = self._exporter().dat_only + export_time = time.localtime(self._time)[:6] + if dat_only: + func = lambda x: x.dirname == "dat" + else: + func = None + + with zipfile.ZipFile(str(self._export_file), 'w', zipfile.ZIP_DEFLATED) as zf: + for i in self._generate_files(func): + arcpath = i.filename if dat_only else "{}/{}".format(i.dirname, i.filename) + if i.file_path: + zf.write(i.file_path, arcpath) + elif i.file_data: + zi = zipfile.ZipInfo(arcpath, export_time) + zf.writestr(zi, i.file_data) + + @property + def _version(self): + return self._exporter().mgr.getVer() diff --git a/korman/exporter/sumfile.py b/korman/exporter/sumfile.py deleted file mode 100644 index 0bd84c8..0000000 --- a/korman/exporter/sumfile.py +++ /dev/null @@ -1,73 +0,0 @@ -# This file is part of Korman. -# -# Korman is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Korman is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Korman. If not, see . - -import hashlib -from pathlib import Path -from PyHSPlasma import * - -def _hashfile(filename, hasher, block=0xFFFF): - with open(str(filename), "rb") as handle: - h = hasher() - data = handle.read(block) - while data: - h.update(data) - data = handle.read(block) - return h.digest() - -class SumFile: - def __init__(self): - self._files = set() - - def append(self, filename): - self._files.add(filename) - - def _collect_files(self, version): - files = [] - for file in self._files: - filename, extension = file.name, file.suffix.lower() - if extension in {".age", ".csv", ".fni", ".loc", ".node", ".p2f", ".pfp", ".sub"}: - filename = Path("dat") / filename - elif extension == ".prp" and version > pvPrime: - # ABM and UU don't want the directory for PRPs... Bug? - filename = Path("dat") / filename - elif extension in {".pak", ".py"}: - filename = Path("Python") / filename - elif extension in {".avi", ".bik", ".oggv", ".webm"}: - filename = Path("avi") / filename - elif extension in {".ogg", ".opus", ".wav"}: - filename = Path("sfx") / filename - elif extension == ".sdl": - filename = Path("SDL") / filename - # else the filename has no directory prefix... oh well - - md5 = _hashfile(file, hashlib.md5) - timestamp = file.stat().st_mtime - files.append((str(filename), md5, int(timestamp))) - return files - - - def write(self, sumpath, version): - """Writes a .sum file for Uru ABM, PotS, Myst 5, etc.""" - files = self._collect_files(version) - enc = plEncryptedStream.kEncAes if version >= pvEoa else plEncryptedStream.kEncXtea - - with plEncryptedStream(version).open(str(sumpath), fmWrite, enc) as stream: - stream.writeInt(len(files)) - stream.writeInt(0) - for file in files: - stream.writeSafeStr(str(file[0])) - stream.write(file[1]) - stream.writeInt(file[2]) - stream.writeInt(0) diff --git a/korman/operators/op_export.py b/korman/operators/op_export.py index adec5bd..d246542 100644 --- a/korman/operators/op_export.py +++ b/korman/operators/op_export.py @@ -79,7 +79,7 @@ class ExportOperator(bpy.types.Operator): # This wigs out and very bad things happen if it's not directly on the operator... filepath = StringProperty(subtype="FILE_PATH") - filter_glob = StringProperty(default="*.age", options={'HIDDEN'}) + filter_glob = StringProperty(default="*.age;*.zip", options={'HIDDEN'}) version = EnumProperty(name="Version", description="Plasma version to export this age for", @@ -87,6 +87,11 @@ class ExportOperator(bpy.types.Operator): default="pvPots", options=set()) + dat_only = BoolProperty(name="Export Only PRPs", + description="Only the Age PRPs should be exported", + default=True, + options={"HIDDEN"}) + def draw(self, context): layout = self.layout age = context.scene.world.plasma_age @@ -133,6 +138,7 @@ class ExportOperator(bpy.types.Operator): except: self.report({"ERROR"}, "Failed to create export directory") return {"CANCELLED"} + path.touch() # We need to back out of edit mode--this ensures that all changes are committed if context.mode != "OBJECT": diff --git a/korman/properties/modifiers/sound.py b/korman/properties/modifiers/sound.py index db3a297..49d5893 100644 --- a/korman/properties/modifiers/sound.py +++ b/korman/properties/modifiers/sound.py @@ -166,6 +166,9 @@ class PlasmaSound(idprops.IDPropMixin, bpy.types.PropertyGroup): header, dataSize = self._get_sound_info() length = dataSize / header.avgBytesPerSec + # HAX: Ensure that the sound file is copied to game, if applicable. + exporter.output.add_sfx(self._sound) + # There is some bug in the MOUL code that causes a crash if this does not match the expected # result. There's no sense in debugging that though--the user should never specify # streaming vs static. That's an implementation detail. diff --git a/korman/ui/ui_world.py b/korman/ui/ui_world.py index 543cc26..a02d388 100644 --- a/korman/ui/ui_world.py +++ b/korman/ui/ui_world.py @@ -58,8 +58,16 @@ class PlasmaGamePanel(AgeButtonsPanel, bpy.types.Panel): row.enabled = bool(age.age_name.strip()) and active_game is not None op = row.operator("export.plasma_age", icon="EXPORT") if active_game is not None: + op.dat_only = False op.filepath = str((Path(active_game.path) / "dat" / age.age_name).with_suffix(".age")) op.version = active_game.version + row = row.row(align=True) + row.operator_context = "INVOKE_DEFAULT" + op = row.operator("export.plasma_age", icon="PACKAGE", text="Package Age") + if active_game is not None: + op.dat_only = False + op.filepath = "{}.zip".format(age.age_name) + op.version = active_game.version class PlasmaGameListRO(bpy.types.UIList):