Don't touch my Col layer!

In user testing, the "Bake All" operator overwriting the "Col" layer was
blowing away too much manual shading, reducing the usefulness of the
feature severely. This changes us to using the "autocolor" layer and
making it a somewhat ephemeral coloring layer. Any "autocolor" layer
generated by the exporter should be removed when the export finishes.
Otherwise, it should persist.

korman/exporter/convert.py

@@ -14,10 +14,13 @@
 #    along with Korman.  If not, see <http://www.gnu.org/licenses/>.
 
 import bpy
-from ..korlib import ConsoleToggler
+
 from pathlib import Path
+from contextlib import ExitStack
+
+from ..korlib import ConsoleToggler
+
 from PyHSPlasma import *
-import time
 
 from . import animation
 from . import camera
@@ -44,7 +47,7 @@ class Exporter:
 
     def run(self):
         log = logger.ExportVerboseLogger if self._op.verbose else logger.ExportProgressLogger
-        with ConsoleToggler(self._op.show_console), log(self._op.filepath) as self.report:
+        with ConsoleToggler(self._op.show_console), log(self._op.filepath) as self.report, ExitStack() as self.context_stack:
             # Step 0: Init export resmgr and stuff
             self.mgr = manager.ExportManager(self)
             self.mesh = mesh.MeshConverter(self)
@@ -56,7 +59,7 @@ class Exporter:
             self.image = image.ImageCache(self)
             self.locman = locman.LocalizationConverter(self)
             self.decal = decal.DecalConverter(self)
-            self.oven = etlight.LightBaker(self.report)
+            self.oven = etlight.LightBaker(self.report, stack=self.context_stack)
 
             # Step 0.8: Init the progress mgr
             self.mesh.add_progress_presteps(self.report)

korman/exporter/etlight.py

@@ -15,6 +15,7 @@
 
 import bpy
 
+from contextlib import ExitStack
 import itertools
 
 from .explosions import *
@@ -27,7 +28,7 @@ _NUM_RENDER_LAYERS = 20
 class LightBaker(_MeshManager):
     """ExportTime Lighting"""
 
-    def __init__(self, report=None, verbose=False):
+    def __init__(self, report=None, stack=None, *, verbose=False):
         self._lightgroups = {}
         if report is None:
             self._report = ExportVerboseLogger() if verbose else ExportProgressLogger()
@@ -38,6 +39,8 @@ class LightBaker(_MeshManager):
             self._report = report
             self._own_report = False
         super().__init__(self._report)
+
+        self._context_stack = stack
         self.vcol_layer_name = "autocolor"
         self.lightmap_name = "{}_LIGHTMAPGEN.png"
         self.lightmap_uvtex_name = "LIGHTMAPGEN"
@@ -240,6 +243,36 @@ class LightBaker(_MeshManager):
         bake, bake_passes = {}, bpy.context.scene.plasma_scene.bake_passes
         bake_vcol = bake.setdefault(("vcol",) + default_layers, [])
 
+        def lightmap_bake_required(obj) -> bool:
+            mod = obj.plasma_modifiers.lightmap
+            if mod.bake_lightmap:
+                if self.force:
+                    return True
+                if mod.image is not None:
+                    uv_texture_names = frozenset((i.name for i in obj.data.uv_textures))
+                    if self.lightmap_uvtex_name in uv_texture_names:
+                        self._report.msg("'{}': Skipping due to valid lightmap override", obj.name, indent=1)
+                    else:
+                        self._report.msg("'{}': Have lightmap but UVs are missing???", obj.name, indent=1)
+                    return False
+                return True
+            return False
+
+        def vcol_bake_required(obj) -> bool:
+            if obj.plasma_modifiers.lightmap.bake_lightmap:
+                return False
+            vcol_layer_names = frozenset((vcol_layer.name.lower() for vcol_layer in obj.data.vertex_colors))
+            manual_layer_names = _VERTEX_COLOR_LAYERS & vcol_layer_names
+            if manual_layer_names:
+                self._report.msg("'{}': Skipping due to valid manual vertex color layer(s): '{}'", obj.name, manual_layer_names.pop(), indent=1)
+                return False
+            if self.force:
+                return True
+            if self.vcol_layer_name.lower() in vcol_layer_names:
+                self._report.msg("'{}': Skipping due to valid matching vertex color layer(s): '{}'", obj.name, self.vcol_layer_name, indent=1)
+                return False
+            return True
+
         for i in filter(lambda x: x.type == "MESH" and bool(x.data.materials), objs):
             mods = i.plasma_modifiers
             lightmap_mod = mods.lightmap
@@ -260,15 +293,7 @@ class LightBaker(_MeshManager):
                 if not lm_active_layers & obj_active_layers:
                     raise ExportError("Bake Lighting '{}': At least one layer the object is on must be selected".format(i.name))
 
-                # OK, now that the sanity checking is done, we could opt-out if an image is already
+                if lightmap_bake_required(i) is False and vcol_bake_required(i) is False:
-                # set and we're not being forced...
-                if not self.force:
-                    want_image = lightmap_mod.bake_lightmap
-                    if want_image and lightmap_mod.image is not None and self.lightmap_uvtex_name in i.data.uv_textures:
-                        self._report.msg("'{}': Skipping due to valid lightmap override", i.name, indent=1)
-                        continue
-                    elif not want_image and any((vcol_layer.name.lower() in _VERTEX_COLOR_LAYERS for vcol_layer in i.data.vertex_colors)):
-                        self._report.msg("'{}': Skipping due to valid vertex color layer", i.name, indent=1)
                     continue
 
                 method = "lightmap" if lightmap_mod.bake_lightmap else "vcol"
@@ -276,8 +301,7 @@ class LightBaker(_MeshManager):
                 bake_pass = bake.setdefault(key, [])
                 bake_pass.append(i)
                 self._report.msg("'{}': Bake to {}", i.name, method, indent=1)
-            elif mods.lighting.preshade:
+            elif mods.lighting.preshade and vcol_bake_required(i):
-                if self.force or not any((vcol_layer.name.lower() in _VERTEX_COLOR_LAYERS for vcol_layer in i.data.vertex_colors)):
                 self._report.msg("'{}': Bake to vcol (crappy)", i.name, indent=1)
                 bake_vcol.append(i)
         return bake
@@ -404,7 +428,8 @@ class LightBaker(_MeshManager):
 
         vcol_layer_name = self.vcol_layer_name
         autocolor = vcols.get(vcol_layer_name)
-        if autocolor is None:
+        needs_vcol_layer = autocolor is None
+        if needs_vcol_layer:
             autocolor = vcols.new(vcol_layer_name)
         toggle.track(vcols, "active", autocolor)
 
@@ -415,6 +440,14 @@ class LightBaker(_MeshManager):
             toggle.track(vcol_layer, "active", autocol)
         mesh.update()
 
+        # Vertex colors are sort of ephemeral, so if we have an exit stack, we want to
+        # terminate this layer when the exporter is done. But, this is not an unconditional
+        # nukage. If we're in the lightmap operators, we clearly want this to persist for
+        # future exports as an optimization. We won't reach this point if there is already an
+        # autocolor layer (gulp).
+        if self._context_stack is not None and needs_vcol_layer:
+            self._context_stack.enter_context(TemporaryObject(vcol_layer, vcols.remove))
+
         # Indicate we should bake
         return True
 

korman/operators/op_lightmap.py

@@ -24,8 +24,6 @@ from ..helpers import UiHelper
 from ..korlib import ConsoleToggler
 
 class _LightingOperator:
-    _FINAL_VERTEX_COLOR_LAYER = "Col"
-
     @contextmanager
     def _oven(self, context):
         if context.scene.world is not None:
@@ -56,17 +54,9 @@ class LightmapAutobakePreviewOperator(_LightingOperator, bpy.types.Operator):
     def __init__(self):
         super().__init__()
 
-    def draw(self, context):
-        layout = self.layout
-
-        layout.label("This will overwrite the following vertex color layer:")
-        layout.label(self._FINAL_VERTEX_COLOR_LAYER, icon="GROUP_VCOL")
-
     def execute(self, context):
         with self._oven(context) as bake:
-            if self.final:
+            if not self.final:
-                bake.vcol_layer_name = self._FINAL_VERTEX_COLOR_LAYER
-            else:
                 bake.lightmap_name = "{}_LIGHTMAPGEN_PREVIEW.png"
                 bake.lightmap_uvtex_name = "LIGHTMAPGEN_PREVIEW"
             bake.force = self.final
@@ -91,14 +81,6 @@ class LightmapAutobakePreviewOperator(_LightingOperator, bpy.types.Operator):
 
         return {"FINISHED"}
 
-    def invoke(self, context, event):
-        # If this is a vertex color bake, we need to be sure that the user really
-        # wants to blow away any color layer they have.
-        if self.final and context.object.plasma_modifiers.lightmap.bake_type == "vcol":
-            if any((i.name == self._FINAL_VERTEX_COLOR_LAYER for i in context.object.data.vertex_colors)):
-                return context.window_manager.invoke_props_dialog(self)
-        return self.execute(context)
-
 
 class LightmapBakeMultiOperator(_LightingOperator, bpy.types.Operator):
     bl_idname = "object.plasma_lightmap_bake"
@@ -118,7 +100,6 @@ class LightmapBakeMultiOperator(_LightingOperator, bpy.types.Operator):
 
         with self._oven(context) as bake:
             bake.force = True
-            bake.vcol_layer_name = self._FINAL_VERTEX_COLOR_LAYER
             if not bake.bake_static_lighting(filtered_objects):
                 self.report({"WARNING"}, "Nothing was baked.")
                 return {"FINISHED"}
@@ -149,45 +130,19 @@ class LightmapClearMultiOperator(_LightingOperator, bpy.types.Operator):
     def _iter_vcols(self, objects):
         yield from filter(lambda x: x.type == "MESH" and not x.plasma_modifiers.lightmap.bake_lightmap, objects)
 
-    def _iter_final_vcols(self, objects):
-        yield from filter(lambda x: x.data.vertex_colors.get(self._FINAL_VERTEX_COLOR_LAYER), self._iter_vcols(objects))
-
-    def draw(self, context):
-        layout = self.layout
-
-        layout.label("This will remove the vertex color layer '{}' on:".format(self._FINAL_VERTEX_COLOR_LAYER))
-        col = layout.column_flow()
-
-        _MAX_OBJECTS = 50
-        vcol_iter = enumerate(self._iter_final_vcols(self._get_objects(context)))
-        for _, bo in itertools.takewhile(lambda x: x[0] < _MAX_OBJECTS, vcol_iter):
-            col.label(bo.name, icon="OBJECT_DATA")
-        remainder = sum((1 for _, _ in vcol_iter))
-        if remainder:
-            layout.label("... and {} other objects.".format(remainder))
-
-    def _get_objects(self, context):
-        return context.selected_objects if self.clear_selection else context.scene.objects
-
     def execute(self, context):
-        all_objects = self._get_objects(context)
+        all_objects = context.selected_objects if self.clear_selection else context.scene.objects
 
         for i in self._iter_lightmaps(all_objects):
             i.plasma_modifiers.lightmap.image = None
 
         for i in self._iter_vcols(all_objects):
             vcols = i.data.vertex_colors
-            col_layer = vcols.get(self._FINAL_VERTEX_COLOR_LAYER)
+            col_layer = vcols.get("autocolor")
             if col_layer is not None:
                 vcols.remove(col_layer)
         return {"FINISHED"}
 
-    def invoke(self, context, event):
-        all_objects = self._get_objects(context)
-        if any(self._iter_final_vcols(all_objects)):
-            return context.window_manager.invoke_props_dialog(self)
-        return self.execute(context)
-
 
 @bpy.app.handlers.persistent
 def _toss_garbage(scene):
@@ -204,9 +159,6 @@ def _toss_garbage(scene):
         uvtex = i.uv_textures.get("LIGHTMAPGEN_PREVIEW")
         if uvtex is not None:
             i.uv_textures.remove(uvtex)
-        vcol_layer = i.vertex_colors.get("autocolor")
-        if vcol_layer is not None:
-            i.vertex_colors.remove(vcol_layer)
 
 # collects light baking garbage
 bpy.app.handlers.save_pre.append(_toss_garbage)

korman/ui/modifiers/render.py

@@ -205,8 +205,11 @@ def lightmap(modifier, layout, context):
         layout.label("Transparent objects cannot be lightmapped.", icon="ERROR")
     else:
         row = layout.row(align=True)
+        if modifier.bake_lightmap:
             row.operator("object.plasma_lightmap_preview", "Preview", icon="RENDER_STILL").final = False
             row.operator("object.plasma_lightmap_preview", "Bake for Export", icon="RENDER_STILL").final = True
+        else:
+            row.operator("object.plasma_lightmap_preview", "Bake", icon="RENDER_STILL").final = True
 
         # Kind of clever stuff to show the user a preview...
         # We can't show images, so we make a hidden ImageTexture called LIGHTMAPGEN_PREVIEW. We check