Fix integer overflow in Vulkan/Metal buffer size and pitch calculations

src/libANGLE/renderer/metal/FrameBufferMtl.mm

@@ -92,7 +92,7 @@ void OverrideMTLClearColor(const mtl::TextureRef &texture,
         sizeInBytes.ValueOrDie(), &tempBuffer));
 
     gl::Rectangle region(0, 0, width, height);
-    uint32_t bytesPerRow = angleFormat.pixelBytes * width;
+    size_t bytesPerRow = static_cast<size_t>(angleFormat.pixelBytes) * width;
     uint32_t destOffset  = 0;
     ANGLE_TRY(mtl::ReadTexturePerSliceBytesToBuffer(context, srcTexture, bytesPerRow, region,
                                                     mipNativeLevel, layerIndex, destOffset,

src/libANGLE/renderer/vulkan/SurfaceVk.cpp

@@ -283,9 +283,8 @@ angle::Result LockSurfaceImpl(DisplayVk *displayVk,
 {
     const gl::InternalFormat &internalFormat =
         gl::GetSizedInternalFormatInfo(image->getActualFormat().glInternalFormat);
-    GLuint rowStride = image->getActualFormat().pixelBytes * width;
-    VkDeviceSize bufferSize =
-        (static_cast<VkDeviceSize>(rowStride) * static_cast<VkDeviceSize>(height));
+    VkDeviceSize rowStride = static_cast<VkDeviceSize>(image->getActualFormat().pixelBytes) * width;
+    VkDeviceSize bufferSize = rowStride * static_cast<VkDeviceSize>(height);
 
     if (!lockBufferHelper.valid() || (lockBufferHelper.getSize() != bufferSize))
     {

src/libANGLE/renderer/vulkan/TextureVk.cpp

@@ -3173,14 +3173,14 @@ angle::Result TextureVk::reinitImageAsRenderable(ContextVk *contextVk, const vk:
             &dstData, dstFormat.id));
 
         // Source and destination data is tightly packed
-        GLuint srcDataRowPitch = sourceBox.width * srcFormat.pixelBytes;
-        GLuint dstDataRowPitch = sourceBox.width * dstFormat.pixelBytes;
+        size_t srcDataRowPitch = static_cast<size_t>(sourceBox.width) * srcFormat.pixelBytes;
+        size_t dstDataRowPitch = static_cast<size_t>(sourceBox.width) * dstFormat.pixelBytes;
 
-        GLuint srcDataDepthPitch = srcDataRowPitch * sourceBox.height;
-        GLuint dstDataDepthPitch = dstDataRowPitch * sourceBox.height;
+        size_t srcDataDepthPitch = srcDataRowPitch * sourceBox.height;
+        size_t dstDataDepthPitch = dstDataRowPitch * sourceBox.height;
 
-        GLuint srcDataLayerPitch = srcDataDepthPitch * sourceBox.depth;
-        GLuint dstDataLayerPitch = dstDataDepthPitch * sourceBox.depth;
+        size_t srcDataLayerPitch = srcDataDepthPitch * sourceBox.depth;
+        size_t dstDataLayerPitch = dstDataDepthPitch * sourceBox.depth;
 
         rx::PixelReadFunction pixelReadFunction   = srcFormat.pixelReadFunction;
         rx::PixelWriteFunction pixelWriteFunction = dstFormat.pixelWriteFunction;

src/libANGLE/renderer/vulkan/UtilsVk.cpp

@@ -4284,10 +4284,10 @@ angle::Result UtilsVk::transCodeEtcToBc(ContextVk *contextVk,
 
     ASSERT(dstImage->getType() != VK_IMAGE_TYPE_1D && dstImage->getType() != VK_IMAGE_TYPE_3D);
 
-    GLuint sliceTexels = (copyRegion->bufferRowLength / info.compressedBlockWidth) *
+    size_t sliceTexels = static_cast<size_t>(copyRegion->bufferRowLength / info.compressedBlockWidth) *
                          (copyRegion->bufferImageHeight / info.compressedBlockHeight);
-    GLuint sliceSize     = sliceTexels * intendedFormat.pixelBytes;
-    GLuint texBufferSize = sliceSize * copyRegion->imageSubresource.layerCount;
+    size_t sliceSize     = sliceTexels * intendedFormat.pixelBytes;
+    size_t texBufferSize = sliceSize * copyRegion->imageSubresource.layerCount;
 
     // Make sure the texture buffer size not out of limit.
     // Usually the limit is more than 128M.

src/libANGLE/renderer/vulkan/vk_helpers.cpp

@@ -11535,7 +11535,9 @@ angle::Result ImageHelper::readPixelsImpl(ContextVk *contextVk,
 
     uint8_t *readPixelBuffer   = nullptr;
     VkDeviceSize stagingOffset = 0;
-    size_t allocationSize      = readFormat->pixelBytes * area.width * area.height;
+    size_t allocationSize      = static_cast<size_t>(readFormat->pixelBytes) *
+                                  static_cast<size_t>(area.width) *
+                                  static_cast<size_t>(area.height);
 
     ANGLE_TRY(contextVk->initBufferForImageCopy(stagingBuffer, allocationSize,
                                                 MemoryCoherency::CachedPreferCoherent,