Cycles: load 8 bit image textures as half float for some color spaces

For non-raw, non-sRGB color spaces, always use half float even if that uses
more memory. Otherwise the precision loss from conversion to scene linear or
sRGB (as natively understood by the texture sampling) can be too much.

This also required a change to do alpha association ourselves instead of OIIO,
because in OIIO alpha multiplication happens before conversion to half float
and that gives too much precision loss.

Ref T68926

intern/cycles/blender/image.cpp

@@ -7,6 +7,8 @@
 #include "blender/session.h"
 #include "blender/util.h"
 
+#include "util/half.h"
+
 CCL_NAMESPACE_BEGIN
 
 /* Packed Images */
@@ -62,79 +64,133 @@ bool BlenderImageLoader::load_metadata(const ImageDeviceFeatures &, ImageMetaDat
 }
 
 bool BlenderImageLoader::load_pixels(const ImageMetaData &metadata,
-                                     void *pixels,
-                                     const size_t pixels_size,
+                                     void *out_pixels,
+                                     const size_t out_pixels_size,
                                      const bool associate_alpha)
 {
   const size_t num_pixels = ((size_t)metadata.width) * metadata.height;
   const int channels = metadata.channels;
 
-  if (b_image.is_float()) {
-    /* image data */
-    float *image_pixels;
-    image_pixels = image_get_float_pixels_for_frame(b_image, frame, tile_number);
+  if (metadata.type == IMAGE_DATA_TYPE_FLOAT || metadata.type == IMAGE_DATA_TYPE_FLOAT4) {
+    /* Float. */
+    float *in_pixels = image_get_float_pixels_for_frame(b_image, frame, tile_number);
 
-    if (image_pixels && num_pixels * channels == pixels_size) {
-      memcpy(pixels, image_pixels, pixels_size * sizeof(float));
+    if (in_pixels && num_pixels * channels == out_pixels_size) {
+      /* Straight copy pixel data. */
+      memcpy(out_pixels, in_pixels, out_pixels_size * sizeof(float));
     }
     else {
+      /* Missing or invalid pixel data. */
       if (channels == 1) {
-        memset(pixels, 0, num_pixels * sizeof(float));
+        memset(out_pixels, 0, num_pixels * sizeof(float));
       }
       else {
-        const size_t num_pixels_safe = pixels_size / channels;
-        float *fp = (float *)pixels;
-        for (int i = 0; i < num_pixels_safe; i++, fp += channels) {
-          fp[0] = 1.0f;
-          fp[1] = 0.0f;
-          fp[2] = 1.0f;
+        const size_t num_pixels_safe = out_pixels_size / channels;
+        float *out_pixel = (float *)out_pixels;
+        for (int i = 0; i < num_pixels_safe; i++, out_pixel += channels) {
+          out_pixel[0] = 1.0f;
+          out_pixel[1] = 0.0f;
+          out_pixel[2] = 1.0f;
           if (channels == 4) {
-            fp[3] = 1.0f;
+            out_pixel[3] = 1.0f;
           }
         }
       }
     }
 
-    if (image_pixels) {
-      MEM_freeN(image_pixels);
+    if (in_pixels) {
+      MEM_freeN(in_pixels);
+    }
+  }
+  else if (metadata.type == IMAGE_DATA_TYPE_HALF || metadata.type == IMAGE_DATA_TYPE_HALF4) {
+    /* Half float. Blender does not have a half type, but in some cases
+     * we upsample byte to half to avoid precision loss for colorspace
+     * conversion. */
+    unsigned char *in_pixels = image_get_pixels_for_frame(b_image, frame, tile_number);
+
+    if (in_pixels && num_pixels * channels == out_pixels_size) {
+      /* Convert uchar to half. */
+      const uchar *in_pixel = in_pixels;
+      half *out_pixel = (half *)out_pixels;
+      if (associate_alpha && channels == 4) {
+        for (size_t i = 0; i < num_pixels; i++, in_pixel += 4, out_pixel += 4) {
+          const float alpha = util_image_cast_to_float(in_pixel[3]);
+          out_pixel[0] = float_to_half_image(util_image_cast_to_float(in_pixel[0]) * alpha);
+          out_pixel[1] = float_to_half_image(util_image_cast_to_float(in_pixel[1]) * alpha);
+          out_pixel[2] = float_to_half_image(util_image_cast_to_float(in_pixel[2]) * alpha);
+          out_pixel[3] = float_to_half_image(alpha);
+        }
+      }
+      else {
+        for (size_t i = 0; i < num_pixels; i++) {
+          for (int c = 0; c < channels; c++, in_pixel++, out_pixel++) {
+            *out_pixel = float_to_half_image(util_image_cast_to_float(*in_pixel));
+          }
+        }
+      }
+    }
+    else {
+      /* Missing or invalid pixel data. */
+      if (channels == 1) {
+        memset(out_pixels, 0, num_pixels * sizeof(half));
+      }
+      else {
+        const size_t num_pixels_safe = out_pixels_size / channels;
+        half *out_pixel = (half *)out_pixels;
+        for (int i = 0; i < num_pixels_safe; i++, out_pixel += channels) {
+          out_pixel[0] = float_to_half_image(1.0f);
+          out_pixel[1] = float_to_half_image(0.0f);
+          out_pixel[2] = float_to_half_image(1.0f);
+          if (channels == 4) {
+            out_pixel[3] = float_to_half_image(1.0f);
+          }
+        }
+      }
+    }
+
+    if (in_pixels) {
+      MEM_freeN(in_pixels);
     }
   }
   else {
-    unsigned char *image_pixels = image_get_pixels_for_frame(b_image, frame, tile_number);
+    /* Byte. */
+    unsigned char *in_pixels = image_get_pixels_for_frame(b_image, frame, tile_number);
 
-    if (image_pixels && num_pixels * channels == pixels_size) {
-      memcpy(pixels, image_pixels, pixels_size * sizeof(unsigned char));
+    if (in_pixels && num_pixels * channels == out_pixels_size) {
+      /* Straight copy pixel data. */
+      memcpy(out_pixels, in_pixels, out_pixels_size * sizeof(unsigned char));
+
+      if (associate_alpha && channels == 4) {
+        /* Premultiply, byte images are always straight for Blender. */
+        unsigned char *out_pixel = (unsigned char *)out_pixels;
+        for (size_t i = 0; i < num_pixels; i++, out_pixel += 4) {
+          out_pixel[0] = (out_pixel[0] * out_pixel[3]) / 255;
+          out_pixel[1] = (out_pixel[1] * out_pixel[3]) / 255;
+          out_pixel[2] = (out_pixel[2] * out_pixel[3]) / 255;
+        }
+      }
     }
     else {
+      /* Missing or invalid pixel data. */
       if (channels == 1) {
-        memset(pixels, 0, pixels_size * sizeof(unsigned char));
+        memset(out_pixels, 0, out_pixels_size * sizeof(unsigned char));
       }
       else {
-        const size_t num_pixels_safe = pixels_size / channels;
-        unsigned char *cp = (unsigned char *)pixels;
-        for (size_t i = 0; i < num_pixels_safe; i++, cp += channels) {
-          cp[0] = 255;
-          cp[1] = 0;
-          cp[2] = 255;
+        const size_t num_pixels_safe = out_pixels_size / channels;
+        unsigned char *out_pixel = (unsigned char *)out_pixels;
+        for (size_t i = 0; i < num_pixels_safe; i++, out_pixel += channels) {
+          out_pixel[0] = 255;
+          out_pixel[1] = 0;
+          out_pixel[2] = 255;
           if (channels == 4) {
-            cp[3] = 255;
+            out_pixel[3] = 255;
           }
         }
       }
     }
 
-    if (image_pixels) {
-      MEM_freeN(image_pixels);
-    }
-
-    if (associate_alpha) {
-      /* Premultiply, byte images are always straight for Blender. */
-      unsigned char *cp = (unsigned char *)pixels;
-      for (size_t i = 0; i < num_pixels; i++, cp += channels) {
-        cp[0] = (cp[0] * cp[3]) / 255;
-        cp[1] = (cp[1] * cp[3]) / 255;
-        cp[2] = (cp[2] * cp[3]) / 255;
-      }
+    if (in_pixels) {
+      MEM_freeN(in_pixels);
     }
   }
 

intern/cycles/scene/image.cpp

@@ -272,17 +272,12 @@ void ImageMetaData::detect_colorspace()
     compress_as_srgb = true;
   }
   else {
-    /* Always compress non-raw 8bit images as scene linear + sRGB, as a
-     * heuristic to keep memory usage the same without too much data loss
-     * due to quantization in common cases. */
-    compress_as_srgb = (type == IMAGE_DATA_TYPE_BYTE || type == IMAGE_DATA_TYPE_BYTE4);
-
     /* If colorspace conversion needed, use half instead of short so we can
      * represent HDR values that might result from conversion. */
-    if (type == IMAGE_DATA_TYPE_USHORT) {
+    if (type == IMAGE_DATA_TYPE_BYTE || type == IMAGE_DATA_TYPE_USHORT) {
       type = IMAGE_DATA_TYPE_HALF;
     }
-    else if (type == IMAGE_DATA_TYPE_USHORT4) {
+    else if (type == IMAGE_DATA_TYPE_BYTE4 || type == IMAGE_DATA_TYPE_USHORT4) {
       type = IMAGE_DATA_TYPE_HALF4;
     }
   }

intern/cycles/scene/image_oiio.cpp

@@ -94,10 +94,11 @@ bool OIIOImageLoader::load_metadata(const ImageDeviceFeatures & /*features*/,
 template<TypeDesc::BASETYPE FileFormat, typename StorageType>
 static void oiio_load_pixels(const ImageMetaData &metadata,
                              const unique_ptr<ImageInput> &in,
+                             const bool associate_alpha,
                              StorageType *pixels)
 {
-  const int width = metadata.width;
-  const int height = metadata.height;
+  const size_t width = metadata.width;
+  const size_t height = metadata.height;
   const int depth = metadata.depth;
   const int components = metadata.channels;
 
@@ -105,12 +106,12 @@ static void oiio_load_pixels(const ImageMetaData &metadata,
   StorageType *readpixels = pixels;
   vector<StorageType> tmppixels;
   if (components > 4) {
-    tmppixels.resize(((size_t)width) * height * components);
+    tmppixels.resize(width * height * components);
     readpixels = &tmppixels[0];
   }
 
   if (depth <= 1) {
-    size_t scanlinesize = ((size_t)width) * components * sizeof(StorageType);
+    size_t scanlinesize = width * components * sizeof(StorageType);
     in->read_image(FileFormat,
                    (uchar *)readpixels + (height - 1) * scanlinesize,
                    AutoStride,
@@ -122,7 +123,7 @@ static void oiio_load_pixels(const ImageMetaData &metadata,
   }
 
   if (components > 4) {
-    size_t dimensions = ((size_t)width) * height;
+    size_t dimensions = width * height;
     for (size_t i = dimensions - 1, pixel = 0; pixel < dimensions; pixel++, i--) {
       pixels[i * 4 + 3] = tmppixels[i * components + 3];
       pixels[i * 4 + 2] = tmppixels[i * components + 2];
@@ -137,7 +138,7 @@ static void oiio_load_pixels(const ImageMetaData &metadata,
   if (cmyk) {
     const StorageType one = util_image_cast_from_float<StorageType>(1.0f);
 
-    const size_t num_pixels = ((size_t)width) * height * depth;
+    const size_t num_pixels = width * height * depth;
     for (size_t i = num_pixels - 1, pixel = 0; pixel < num_pixels; pixel++, i--) {
       float c = util_image_cast_to_float(pixels[i * 4 + 0]);
       float m = util_image_cast_to_float(pixels[i * 4 + 1]);
@@ -149,6 +150,16 @@ static void oiio_load_pixels(const ImageMetaData &metadata,
       pixels[i * 4 + 3] = one;
     }
   }
+
+  if (components == 4 && associate_alpha) {
+    size_t dimensions = width * height;
+    for (size_t i = dimensions - 1, pixel = 0; pixel < dimensions; pixel++, i--) {
+      const StorageType alpha = pixels[i * 4 + 3];
+      pixels[i * 4 + 0] = util_image_multiply_native(pixels[i * 4 + 0], alpha);
+      pixels[i * 4 + 1] = util_image_multiply_native(pixels[i * 4 + 1], alpha);
+      pixels[i * 4 + 2] = util_image_multiply_native(pixels[i * 4 + 2], alpha);
+    }
+  }
 }
 
 bool OIIOImageLoader::load_pixels(const ImageMetaData &metadata,
@@ -172,30 +183,36 @@ bool OIIOImageLoader::load_pixels(const ImageMetaData &metadata,
   ImageSpec spec = ImageSpec();
   ImageSpec config = ImageSpec();
 
-  if (!associate_alpha) {
-    config.attribute("oiio:UnassociatedAlpha", 1);
-  }
+  /* Load without automatic OIIO alpha conversion, we do it ourselves. OIIO
+   * will associate alpha in the the 8bit buffer for PNGs, which leads to too
+   * much precision loss when we load it as half float to do a colorspace
+   * transform. */
+  config.attribute("oiio:UnassociatedAlpha", 1);
 
   if (!in->open(filepath.string(), spec, config)) {
     return false;
   }
 
+  const bool do_associate_alpha = associate_alpha &&
+                                  spec.get_int_attribute("oiio:UnassociatedAlpha", 0);
+
   switch (metadata.type) {
     case IMAGE_DATA_TYPE_BYTE:
     case IMAGE_DATA_TYPE_BYTE4:
-      oiio_load_pixels<TypeDesc::UINT8, uchar>(metadata, in, (uchar *)pixels);
+      oiio_load_pixels<TypeDesc::UINT8, uchar>(metadata, in, do_associate_alpha, (uchar *)pixels);
       break;
     case IMAGE_DATA_TYPE_USHORT:
     case IMAGE_DATA_TYPE_USHORT4:
-      oiio_load_pixels<TypeDesc::USHORT, uint16_t>(metadata, in, (uint16_t *)pixels);
+      oiio_load_pixels<TypeDesc::USHORT, uint16_t>(
+          metadata, in, do_associate_alpha, (uint16_t *)pixels);
       break;
     case IMAGE_DATA_TYPE_HALF:
     case IMAGE_DATA_TYPE_HALF4:
-      oiio_load_pixels<TypeDesc::HALF, half>(metadata, in, (half *)pixels);
+      oiio_load_pixels<TypeDesc::HALF, half>(metadata, in, do_associate_alpha, (half *)pixels);
       break;
     case IMAGE_DATA_TYPE_FLOAT:
     case IMAGE_DATA_TYPE_FLOAT4:
-      oiio_load_pixels<TypeDesc::FLOAT, float>(metadata, in, (float *)pixels);
+      oiio_load_pixels<TypeDesc::FLOAT, float>(metadata, in, do_associate_alpha, (float *)pixels);
       break;
     case IMAGE_DATA_TYPE_NANOVDB_FLOAT:
     case IMAGE_DATA_TYPE_NANOVDB_FLOAT3:

intern/cycles/util/image.h

@@ -78,6 +78,26 @@ template<> inline half util_image_cast_from_float(float value)
   return float_to_half_image(value);
 }
 
+/* Multiply image pixels in native data format. */
+template<typename T> inline T util_image_multiply_native(T a, T b);
+
+template<> inline float util_image_multiply_native(float a, float b)
+{
+  return a * b;
+}
+template<> inline uchar util_image_multiply_native(uchar a, uchar b)
+{
+  return ((uint32_t)a * (uint32_t)b) / 255;
+}
+template<> inline uint16_t util_image_multiply_native(uint16_t a, uint16_t b)
+{
+  return ((uint32_t)a * (uint32_t)b) / 65535;
+}
+template<> inline half util_image_multiply_native(half a, half b)
+{
+  return float_to_half_image(half_to_float_image(a) * half_to_float_image(b));
+}
+
 CCL_NAMESPACE_END
 
 #endif /* __UTIL_IMAGE_H__ */