diff --git a/korman/exporter/convert.py b/korman/exporter/convert.py index 7dc9ae8..81ffc08 100644 --- a/korman/exporter/convert.py +++ b/korman/exporter/convert.py @@ -27,9 +27,9 @@ from . import image from . import logger from . import manager from . import mesh +from . import outfile from . import physics from . import rtlight -from . import sumfile from . import utils class Exporter: @@ -40,10 +40,6 @@ class Exporter: self.node_trees_exported = set() self.want_node_trees = {} - @property - def age_name(self): - return Path(self._op.filepath).stem - def run(self): log = logger.ExportVerboseLogger if self._op.verbose else logger.ExportProgressLogger with ConsoleToggler(self._op.show_console), log(self._op.filepath) as self.report: @@ -53,7 +49,7 @@ class Exporter: self.physics = physics.PhysicsConverter(self) self.light = rtlight.LightConverter(self) self.animation = animation.AnimationConverter(self) - self.sumfile = sumfile.SumFile() + self.output = outfile.OutputFiles(self, self._op.filepath) self.camera = camera.CameraConverter(self) self.image = image.ImageCache(self) @@ -347,8 +343,22 @@ class Exporter: def _save_age(self): self.report.progress_advance() - self.mgr.save_age(Path(self._op.filepath)) - self.image.save() + + # If something bad happens in the final flush, it would be a shame to + # simply toss away the potentially freshly regenerated texture cache. + try: + self.mgr.save_age() + self.output.save() + finally: + self.image.save() + + @property + def age_name(self): + return Path(self._op.filepath).stem + + @property + def dat_only(self): + return self._op.dat_only @property def envmap_method(self): diff --git a/korman/exporter/manager.py b/korman/exporter/manager.py index 6ff74b3..dc7775e 100644 --- a/korman/exporter/manager.py +++ b/korman/exporter/manager.py @@ -233,27 +233,23 @@ class ExportManager: else: return key.location - def save_age(self, path): - ageName = path.stem - sumfile = self._exporter().sumfile - - sumfile.append(path) - self.mgr.WriteAge(str(path), self._age_info) - self._write_fni(path) - self._write_pages(path) - - if self.getVer() != pvMoul: - sumpath = path.with_suffix(".sum") - sumfile.write(sumpath, self.getVer()) - - def _write_fni(self, path): - if self.mgr.getVer() <= pvMoul: - enc = plEncryptedStream.kEncXtea - else: - enc = plEncryptedStream.kEncAES - fname = path.with_suffix(".fni") + def save_age(self): + self._write_age() + self._write_fni() + self._write_pages() + + def _write_age(self): + f = "{}.age".format(self._age_info.name) + output = self._exporter().output + + with output.generate_dat_file(f, enc=plEncryptedStream.kEncAuto) as stream: + self._age_info.writeToStream(stream) - with plEncryptedStream(self.mgr.getVer()).open(str(fname), fmWrite, enc) as stream: + def _write_fni(self): + f = "{}.fni".format(self._age_info.name) + output = self._exporter().output + + with output.generate_dat_file(f, enc=plEncryptedStream.kEncAuto) as stream: fni = bpy.context.scene.world.plasma_fni stream.writeLine("Graphics.Renderer.SetClearColor {} {} {}".format(*fni.clear_color)) if fni.fog_method != "none": @@ -263,17 +259,14 @@ class ExportManager: elif fni.fog_method == "exp2": stream.writeLine("Graphics.Renderer.Fog.SetDefExp2 {} {}".format(fni.fog_end, fni.fog_density)) stream.writeLine("Graphics.Renderer.SetYon {}".format(fni.yon)) - self._exporter().sumfile.append(fname) - def _write_pages(self, path): + def _write_pages(self): + age_name = self._age_info.name + output = self._exporter().output for loc in self._pages.values(): page = self.mgr.FindPage(loc) # not cached because it's C++ owned - # I know that plAgeInfo has its own way of doing this, but we'd have - # to do some looping and stuff. This is easier. - if self.mgr.getVer() <= pvMoul: - chapter = "_District_" - else: - chapter = "_" - f = path.with_name("{}{}{}".format(path.stem, chapter, page.page)).with_suffix(".prp") - self.mgr.WritePage(str(f), page) - self._exporter().sumfile.append(f) + chapter = "_District_" if self.mgr.getVer() <= pvMoul else "_" + f = "{}{}{}.prp".format(age_name, chapter, page.page) + + with output.generate_dat_file(f) as stream: + self.mgr.WritePage(stream, page) diff --git a/korman/exporter/outfile.py b/korman/exporter/outfile.py new file mode 100644 index 0000000..b98f1f1 --- /dev/null +++ b/korman/exporter/outfile.py @@ -0,0 +1,267 @@ +# This file is part of Korman. +# +# Korman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Korman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Korman. If not, see . + +from contextlib import contextmanager +import enum +from hashlib import md5 +import os +from pathlib import Path +from PyHSPlasma import * +import shutil +import time +import weakref +import zipfile + +def _hashfile(filename, hasher, block=0xFFFF): + with open(str(filename), "rb") as handle: + h = hasher() + data = handle.read(block) + while data: + h.update(data) + data = handle.read(block) + return h.digest() + +@enum.unique +class _FileType(enum.Enum): + generated_dat = 0 + sfx = 1 + + +class _OutputFile: + def __init__(self, **kwargs): + self.file_type = kwargs.get("file_type") + self.dirname = kwargs.get("dirname") + self.filename = kwargs.get("filename") + self.skip_hash = kwargs.get("skip_hash", False) + + if self.file_type == _FileType.generated_dat: + self.file_data = kwargs.get("file_data", None) + self.file_path = kwargs.get("file_path", None) + self.mod_time = Path(self.file_path).stat().st_mtime if self.file_path else None + + # need either a data buffer OR a file path + assert bool(self.file_data) ^ bool(self.file_path) + + if self.file_type == _FileType.sfx: + self.id_data = kwargs.get("id_data") + path = Path(self.id_data.filepath).resolve() + if path.exists(): + self.file_path = str(path) + self.mod_time = path.stat().st_mtime + else: + self.file_path = None + self.mod_time = None + if self.id_data.packed_file is not None: + self.file_data = self.id_data.packed_file.data + + def __eq__(self, rhs): + return str(self) == str(rhs) + + def __hash__(self): + return hash(str(self)) + + def hash_md5(self): + if self.file_path is not None: + with open(self.file_path, "rb") as handle: + h = md5() + data = handle.read(0xFFFF) + while data: + h.update(data) + data = handle.read(0xFFFF) + return h.digest() + elif self.file_data is not None: + return md5(self.file_data).digest() + else: + raise RuntimeError() + + def __str__(self): + return "{}/{}".format(self.dirname, self.filename) + + +class OutputFiles: + def __init__(self, exporter, path): + self._exporter = weakref.ref(exporter) + self._export_file = Path(path).resolve() + if exporter.dat_only: + self._export_path = self._export_file.parent + else: + self._export_path = self._export_file.parent.parent + self._files = set() + self._is_zip = self._export_file.suffix.lower() == ".zip" + self._time = time.time() + + def add_python(self, filename, text_id=None, str_data=None): + of = _OutputFile(file_type=_FileType.python_code, + dirname="Python", filename=filename, + id_data=text_id, file_data=str_data, + skip_hash=True) + self._files.add(of) + + def add_sdl(self, filename, text_id=None, str_data=None): + version = self._version + if version == pvEoa: + enc = plEncryptedStream.kEncAes + elif version == pvMoul: + enc = None + else: + enc = plEncryptedStream.kEncXtea + + of = _OutputFile(file_type=_FileType.sdl, + dirname="SDL", filename=filename, + id_data=text_id, file_data=str_data, + enc=enc) + self._files.add(of) + + + def add_sfx(self, sound_id): + of = _OutputFile(file_type=_FileType.sfx, + dirname="sfx", filename=sound_id.name, + id_data=sound_id) + self._files.add(of) + + @contextmanager + def generate_dat_file(self, filename, **kwargs): + if self._is_zip: + stream = hsRAMStream(self._version) + else: + file_path = str(self._export_file.parent / filename) + stream = hsFileStream(self._version) + stream.open(file_path, fmCreate) + backing_stream = stream + + enc = kwargs.get("enc", None) + if enc is not None: + stream = plEncryptedStream(self._version) + stream.open(backing_stream, fmCreate, enc) + + # The actual export code is run at the "yield" statement. If an error occurs, we + # do not want to track this file. Note that the except block is required for the + # else block to be legal. ^_^ + try: + yield stream + except: + raise + else: + # Must call the EncryptedStream close to actually encrypt the data + stream.close() + if not stream is backing_stream: + backing_stream.close() + + kwargs = { + "file_type": _FileType.generated_dat, + "dirname": "dat", + "filename": filename, + "skip_hash": skip_hash, + } + if isinstance(backing_stream, hsRAMStream): + kwargs["file_data"] = backing_stream.buffer + else: + kwargs["file_path"] = file_path + self._files.add(_OutputFile(**kwargs)) + + def _generate_files(self, func=None): + dat_only = self._exporter().dat_only + for i in self._files: + if dat_only and i.dirname != "dat": + continue + if func is not None: + if func(i): + yield i + else: + yield i + + def save(self): + # At this stage, all Plasma data has been generated from whatever crap is in + # Blender. The only remaining part is to make sure any external dependencies are + # copied or packed into the appropriate format OR the asset hashes are generated. + + # Step 1: Handle Python + # ... todo ... + + # Step 2: Generate sumfile + if self._version != pvMoul: + self._write_sumfile() + + # Step 3: Ensure errbody is gut + if self._is_zip: + self._write_zipfile() + else: + self._write_deps() + + def _write_deps(self): + func = lambda x: x.file_type == _FileType.sfx + times = (self._time, self._time) + + for i in self._generate_files(func): + # Will only ever run for non-"dat" directories. + dst_path = str(self._export_path / i.dirname / i.filename) + if i.file_path: + shutil.copy2(i.file_path, dst_path) + elif i.file_data: + with open(dst_path, "wb") as handle: + handle.write(i.file_data) + os.utime(dst_path, times) + else: + raise RuntimeError() + + def _write_sumfile(self): + version = self._version + dat_only = self._exporter().dat_only + enc = plEncryptedStream.kEncAes if version >= pvEoa else plEncryptedStream.kEncXtea + filename = "{}.sum".format(self._exporter().age_name) + if dat_only: + func = lambda x: not x.skip_hash and x.dirname == "dat" + else: + func = lambda x: not x.skip_hash + + with self.generate_dat_file(filename, enc=enc, skip_hash=True) as stream: + files = list(self._generate_files(func)) + stream.writeInt(len(files)) + stream.writeInt(0) + for i in files: + # ABM and UU don't want the directory for PRPs... Bug? + extension = Path(i.filename).suffix.lower() + if extension == ".prp" and version < pvPots: + filename = i.filename + else: + filename = "{}\\{}".format(i.dirname, i.filename) + mod_time = i.mod_time if i.mod_time else self._time + hash_md5 = i.hash_md5() + + stream.writeSafeStr(filename) + stream.write(hash_md5) + stream.writeInt(int(mod_time)) + stream.writeInt(0) + + def _write_zipfile(self): + dat_only = self._exporter().dat_only + export_time = time.localtime(self._time)[:6] + if dat_only: + func = lambda x: x.dirname == "dat" + else: + func = None + + with zipfile.ZipFile(str(self._export_file), 'w', zipfile.ZIP_DEFLATED) as zf: + for i in self._generate_files(func): + arcpath = i.filename if dat_only else "{}/{}".format(i.dirname, i.filename) + if i.file_path: + zf.write(i.file_path, arcpath) + elif i.file_data: + zi = zipfile.ZipInfo(arcpath, export_time) + zf.writestr(zi, i.file_data) + + @property + def _version(self): + return self._exporter().mgr.getVer() diff --git a/korman/exporter/sumfile.py b/korman/exporter/sumfile.py deleted file mode 100644 index 0bd84c8..0000000 --- a/korman/exporter/sumfile.py +++ /dev/null @@ -1,73 +0,0 @@ -# This file is part of Korman. -# -# Korman is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Korman is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Korman. If not, see . - -import hashlib -from pathlib import Path -from PyHSPlasma import * - -def _hashfile(filename, hasher, block=0xFFFF): - with open(str(filename), "rb") as handle: - h = hasher() - data = handle.read(block) - while data: - h.update(data) - data = handle.read(block) - return h.digest() - -class SumFile: - def __init__(self): - self._files = set() - - def append(self, filename): - self._files.add(filename) - - def _collect_files(self, version): - files = [] - for file in self._files: - filename, extension = file.name, file.suffix.lower() - if extension in {".age", ".csv", ".fni", ".loc", ".node", ".p2f", ".pfp", ".sub"}: - filename = Path("dat") / filename - elif extension == ".prp" and version > pvPrime: - # ABM and UU don't want the directory for PRPs... Bug? - filename = Path("dat") / filename - elif extension in {".pak", ".py"}: - filename = Path("Python") / filename - elif extension in {".avi", ".bik", ".oggv", ".webm"}: - filename = Path("avi") / filename - elif extension in {".ogg", ".opus", ".wav"}: - filename = Path("sfx") / filename - elif extension == ".sdl": - filename = Path("SDL") / filename - # else the filename has no directory prefix... oh well - - md5 = _hashfile(file, hashlib.md5) - timestamp = file.stat().st_mtime - files.append((str(filename), md5, int(timestamp))) - return files - - - def write(self, sumpath, version): - """Writes a .sum file for Uru ABM, PotS, Myst 5, etc.""" - files = self._collect_files(version) - enc = plEncryptedStream.kEncAes if version >= pvEoa else plEncryptedStream.kEncXtea - - with plEncryptedStream(version).open(str(sumpath), fmWrite, enc) as stream: - stream.writeInt(len(files)) - stream.writeInt(0) - for file in files: - stream.writeSafeStr(str(file[0])) - stream.write(file[1]) - stream.writeInt(file[2]) - stream.writeInt(0) diff --git a/korman/operators/op_export.py b/korman/operators/op_export.py index adec5bd..d246542 100644 --- a/korman/operators/op_export.py +++ b/korman/operators/op_export.py @@ -79,7 +79,7 @@ class ExportOperator(bpy.types.Operator): # This wigs out and very bad things happen if it's not directly on the operator... filepath = StringProperty(subtype="FILE_PATH") - filter_glob = StringProperty(default="*.age", options={'HIDDEN'}) + filter_glob = StringProperty(default="*.age;*.zip", options={'HIDDEN'}) version = EnumProperty(name="Version", description="Plasma version to export this age for", @@ -87,6 +87,11 @@ class ExportOperator(bpy.types.Operator): default="pvPots", options=set()) + dat_only = BoolProperty(name="Export Only PRPs", + description="Only the Age PRPs should be exported", + default=True, + options={"HIDDEN"}) + def draw(self, context): layout = self.layout age = context.scene.world.plasma_age @@ -133,6 +138,7 @@ class ExportOperator(bpy.types.Operator): except: self.report({"ERROR"}, "Failed to create export directory") return {"CANCELLED"} + path.touch() # We need to back out of edit mode--this ensures that all changes are committed if context.mode != "OBJECT": diff --git a/korman/properties/modifiers/sound.py b/korman/properties/modifiers/sound.py index db3a297..49d5893 100644 --- a/korman/properties/modifiers/sound.py +++ b/korman/properties/modifiers/sound.py @@ -166,6 +166,9 @@ class PlasmaSound(idprops.IDPropMixin, bpy.types.PropertyGroup): header, dataSize = self._get_sound_info() length = dataSize / header.avgBytesPerSec + # HAX: Ensure that the sound file is copied to game, if applicable. + exporter.output.add_sfx(self._sound) + # There is some bug in the MOUL code that causes a crash if this does not match the expected # result. There's no sense in debugging that though--the user should never specify # streaming vs static. That's an implementation detail. diff --git a/korman/ui/ui_world.py b/korman/ui/ui_world.py index 543cc26..a02d388 100644 --- a/korman/ui/ui_world.py +++ b/korman/ui/ui_world.py @@ -58,8 +58,16 @@ class PlasmaGamePanel(AgeButtonsPanel, bpy.types.Panel): row.enabled = bool(age.age_name.strip()) and active_game is not None op = row.operator("export.plasma_age", icon="EXPORT") if active_game is not None: + op.dat_only = False op.filepath = str((Path(active_game.path) / "dat" / age.age_name).with_suffix(".age")) op.version = active_game.version + row = row.row(align=True) + row.operator_context = "INVOKE_DEFAULT" + op = row.operator("export.plasma_age", icon="PACKAGE", text="Package Age") + if active_game is not None: + op.dat_only = False + op.filepath = "{}.zip".format(age.age_name) + op.version = active_game.version class PlasmaGameListRO(bpy.types.UIList):