summaryrefslogtreecommitdiffstats
path: root/drivers/media
diff options
context:
space:
mode:
authorMoudy Ho <moudy.ho@mediatek.com>2023-03-27 11:13:24 +0800
committerGreg Kroah-Hartman <gregkh@linuxfoundation.org>2023-05-11 23:17:08 +0900
commit2457058d9ac942df63a4705ba6c696ccbf27bd28 (patch)
treef0a2f9f4a8f841385a303cc5a1ebf1573f52ac1d /drivers/media
parent3a60e51489a3ec61565f5bc53f726ac9ccc6083c (diff)
downloadlinux-stable-2457058d9ac942df63a4705ba6c696ccbf27bd28.tar.gz
linux-stable-2457058d9ac942df63a4705ba6c696ccbf27bd28.tar.bz2
linux-stable-2457058d9ac942df63a4705ba6c696ccbf27bd28.zip
media: platform: mtk-mdp3: fix potential frame size overflow in mdp_try_fmt_mplane()
[ Upstream commit 4168720753ce6c14c5d3a35302fc2e1841383443 ] Fix overflow risk when setting certain formats whose frame size exceeds a RGB24 with 7723x7723 resolution. For example, a 7723x7724 RGB24 frame: 1. bpl (byte per line) = 7723 * 3. 2. Overflow occurs when bpl * 7724 * depth. Fixes: 61890ccaefaf ("media: platform: mtk-mdp3: add MediaTek MDP3 driver") Signed-off-by: Moudy Ho <moudy.ho@mediatek.com> Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl> Signed-off-by: Sasha Levin <sashal@kernel.org>
Diffstat (limited to 'drivers/media')
-rw-r--r--drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.c10
1 files changed, 6 insertions, 4 deletions
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.c
index 4e84a37ecdfc..36336d169bd9 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.c
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.c
@@ -4,6 +4,7 @@
* Author: Ping-Hsun Wu <ping-hsun.wu@mediatek.com>
*/
+#include <linux/math64.h>
#include <media/v4l2-common.h>
#include <media/videobuf2-v4l2.h>
#include <media/videobuf2-dma-contig.h>
@@ -428,14 +429,15 @@ const struct mdp_format *mdp_try_fmt_mplane(struct v4l2_format *f,
u32 bpl = pix_mp->plane_fmt[i].bytesperline;
u32 min_si, max_si;
u32 si = pix_mp->plane_fmt[i].sizeimage;
+ u64 di;
bpl = clamp(bpl, min_bpl, max_bpl);
pix_mp->plane_fmt[i].bytesperline = bpl;
- min_si = (bpl * pix_mp->height * fmt->depth[i]) /
- fmt->row_depth[i];
- max_si = (bpl * s.max_height * fmt->depth[i]) /
- fmt->row_depth[i];
+ di = (u64)bpl * pix_mp->height * fmt->depth[i];
+ min_si = (u32)div_u64(di, fmt->row_depth[i]);
+ di = (u64)bpl * s.max_height * fmt->depth[i];
+ max_si = (u32)div_u64(di, fmt->row_depth[i]);
si = clamp(si, min_si, max_si);
pix_mp->plane_fmt[i].sizeimage = si;