Skip to content

Commit af6d52e

Browse files
haasnhaasn
committed
swscale: use 16-bit intermediate precision for RGB/XYZ conversion
The current logic uses 12-bit linear light math, which is woefully insufficient and leads to nasty postarization artifacts. This patch simply switches the internal logic to 16-bit precision. This raises the memory requirement of these tables from 32 kB to 272 kB. All relevant FATE tests updated for improved accuracy. Fixes: #4829 Signed-off-by: Niklas Haas <[email protected]> Sponsored-by: Sovereign Tech Fund
1 parent 3e6d89c commit af6d52e

24 files changed

+55
-50
lines changed

‎libswscale/swscale.c

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -773,10 +773,10 @@ void ff_xyz12Torgb48(const SwsInternal *c, uint8_t *dst, int dst_stride,
773773
c->xyz2rgb_matrix[2][1] * y +
774774
c->xyz2rgb_matrix[2][2] * z >> 12;
775775

776-
// limit values to 12-bit depth
777-
r = av_clip_uintp2(r, 12);
778-
g = av_clip_uintp2(g, 12);
779-
b = av_clip_uintp2(b, 12);
776+
// limit values to 16-bit depth
777+
r = av_clip_uint16(r);
778+
g = av_clip_uint16(g);
779+
b = av_clip_uint16(b);
780780

781781
// convert from sRGBlinear to RGB and scale from 12bit to 16bit
782782
if (desc->flags & AV_PIX_FMT_FLAG_BE) {
@@ -832,10 +832,10 @@ void ff_rgb48Toxyz12(const SwsInternal *c, uint8_t *dst, int dst_stride,
832832
c->rgb2xyz_matrix[2][1] * g +
833833
c->rgb2xyz_matrix[2][2] * b >> 12;
834834

835-
// limit values to 12-bit depth
836-
x = av_clip_uintp2(x, 12);
837-
y = av_clip_uintp2(y, 12);
838-
z = av_clip_uintp2(z, 12);
835+
// limit values to 16-bit depth
836+
x = av_clip_uint16(x);
837+
y = av_clip_uint16(y);
838+
z = av_clip_uint16(z);
839839

840840
// convert from XYZlinear to X'Y'Z' and scale from 12bit to 16bit
841841
if (desc->flags & AV_PIX_FMT_FLAG_BE) {

‎libswscale/swscale_internal.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -547,10 +547,10 @@ struct SwsInternal {
547547
/* pre defined color-spaces gamma */
548548
#define XYZ_GAMMA (2.6f)
549549
#define RGB_GAMMA (2.2f)
550-
int16_t *xyzgamma;
551-
int16_t *rgbgamma;
552-
int16_t *xyzgammainv;
553-
int16_t *rgbgammainv;
550+
uint16_t *xyzgamma;
551+
uint16_t *rgbgamma;
552+
uint16_t *xyzgammainv;
553+
uint16_t *rgbgammainv;
554554
int16_t xyz2rgb_matrix[3][4];
555555
int16_t rgb2xyz_matrix[3][4];
556556

‎libswscale/utils.c

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -951,7 +951,8 @@ static void fill_xyztables(SwsInternal *c)
951951
{1689, 1464, 739},
952952
{ 871, 2929, 296},
953953
{ 79, 488, 3891} };
954-
static int16_t xyzgamma_tab[4096], rgbgamma_tab[4096], xyzgammainv_tab[4096], rgbgammainv_tab[4096];
954+
static uint16_t xyzgamma_tab[4096], rgbgammainv_tab[4096];
955+
static uint16_t rgbgamma_tab[65536], xyzgammainv_tab[65536];
955956

956957
memcpy(c->xyz2rgb_matrix, xyz2rgb_matrix, sizeof(c->xyz2rgb_matrix));
957958
memcpy(c->rgb2xyz_matrix, rgb2xyz_matrix, sizeof(c->rgb2xyz_matrix));
@@ -960,15 +961,19 @@ static void fill_xyztables(SwsInternal *c)
960961
c->xyzgammainv = xyzgammainv_tab;
961962
c->rgbgammainv = rgbgammainv_tab;
962963

963-
if (rgbgamma_tab[4095])
964+
if (xyzgamma_tab[4095])
964965
return;
965966

966-
/* set gamma vectors */
967+
/* set input gamma vectors */
967968
for (i = 0; i < 4096; i++) {
968-
xyzgamma_tab[i] = lrint(pow(i / 4095.0, xyzgamma) * 4095.0);
969-
rgbgamma_tab[i] = lrint(pow(i / 4095.0, rgbgamma) * 4095.0);
970-
xyzgammainv_tab[i] = lrint(pow(i / 4095.0, xyzgammainv) * 4095.0);
971-
rgbgammainv_tab[i] = lrint(pow(i / 4095.0, rgbgammainv) * 4095.0);
969+
xyzgamma_tab[i] = lrint(pow(i / 4095.0, xyzgamma) * 65535.0);
970+
rgbgammainv_tab[i] = lrint(pow(i / 4095.0, rgbgammainv) * 65535.0);
971+
}
972+
973+
/* set output gamma vectors */
974+
for (i = 0; i < 65536; i++) {
975+
rgbgamma_tab[i] = lrint(pow(i / 65535.0, rgbgamma) * 4095.0);
976+
xyzgammainv_tab[i] = lrint(pow(i / 65535.0, xyzgammainv) * 4095.0);
972977
}
973978
}
974979

‎tests/ref/fate/filter-pixdesc-xyz12be

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
pixdesc-xyz12be 4ec824668b9753e26c1bccffca866e27
1+
pixdesc-xyz12be 1508a33dea936c45d9ee13f7743af00d

‎tests/ref/fate/filter-pixdesc-xyz12le

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
pixdesc-xyz12le 88d2563589044a3e28f6cde9a43599f9
1+
pixdesc-xyz12le da2d1326fa5747a7f6ce5ac1e1494aea

‎tests/ref/fate/filter-pixfmts-copy

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,8 @@ xv36be 9f556ee59a672fd8725f0bb36ce3e4b0
111111
xv36le e08dcbde02f1c28a3554f372ad1278e2
112112
xv48be ce34993b4b4411bba1d852b9b86aa39e
113113
xv48le df913a7e61b162aa98303e5393e60c63
114-
xyz12be a1ef56bf746d71f59669c28e48fc8450
115-
xyz12le 831ff03c1ba4ef19374686f16a064d8c
114+
xyz12be f257f86373207af8aed0a1a05171df3b
115+
xyz12le 7922f99edc44a2c26a25becbea9914cc
116116
y210le 04e9487b6cce38e7531437e946cdd586
117117
y212le 825768be8fe92708ae80be84855066ed
118118
y216le 0e99aeddfee304e72d525d72998d9e9b

‎tests/ref/fate/filter-pixfmts-crop

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,8 @@ xv36be 23b6f253fcb375e4145cfcb562268c5f
108108
xv36le 778286003497f92b84d0bd8258d6b85d
109109
xv48be c90889b2cf54cc78bd58e8c47d4eb791
110110
xv48le 2c15c1254449ec5f9135ae61bdf4e1d5
111-
xyz12be cb4571f9aaa7b59f999ef327276104b7
112-
xyz12le cd6aae8d26b18bdb4b9d068586276d91
111+
xyz12be e2f9f6a1ec205ab675a5a1c9521dfa6c
112+
xyz12le fea1da11c07736303b139bc52b7d4759
113113
ya16be 071add03126a11dc6a06209e9b409f8d
114114
ya16le b723211dc0647c944768c6e45e066b36
115115
ya8 51a8dd297e35d40b06d3ebe8f4717895

‎tests/ref/fate/filter-pixfmts-field

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,8 @@ xv36be bcc7bda2d0a5d43db4464af6a4cb5d65
111111
xv36le ba99f258370f2a56993e8760e6b30194
112112
xv48be 2abcd986a34789ba4310be3969020d0d
113113
xv48le f6f2e33f260f48334197538f3331f7bc
114-
xyz12be d2fa69ec91d3ed862f2dac3f8e7a3437
115-
xyz12le 02bccd5e0b6824779a1f848b0ea3e3b5
114+
xyz12be 3b6eb75517263b9e54b9bfa869de394f
115+
xyz12le 27d1d6a488cbc5d53e8d12fa0e162ddb
116116
y210le 4c2fba1dc40322584977d15dd07c9146
117117
y212le ac2a47c45187dd54d0f55293cbffd954
118118
y216le e65b5bfae1b40edbbed2012e9cd45e31

‎tests/ref/fate/filter-pixfmts-fieldorder

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,8 +100,8 @@ xv36be 962386c88268f4382004c3a7a82c5eb8
100100
xv36le bcceffc985aaa8414c4b8072aa0889bd
101101
xv48be 4d6e4004b03767f12df8bb4e76c98ddf
102102
xv48le 9e94d82461a2131063157ac0dbe9467b
103-
xyz12be 15f5cda71de5fef9cec5e75e3833b6bc
104-
xyz12le 7be6c8781f38c21a6b8f602f62ca31e6
103+
xyz12be ba6928f85c202cd77e216934f6bf0698
104+
xyz12le 964680cd3f3db8a7ef5510f90196961a
105105
y210le 22b1a02a39c4b325726bf8793bf1e8f2
106106
y212le 2f08fb195b948056c844acb1eee8d649
107107
y216le 360cb98ac80b13d3a8ec61c9f1ff3bac

‎tests/ref/fate/filter-pixfmts-hflip

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,8 @@ xv36be 98f578df965eed369f46cb135e2d1345
108108
xv36le e478b4b54698beb3ce1b9a2dd691d544
109109
xv48be e030a2c7b1b600cfacb691b6e90c2e3d
110110
xv48le fbd7f8c65cd6fc9f9108dc9a1f977dc3
111-
xyz12be 25f90259ff8a226befdaec3dfe82996e
112-
xyz12le 926c0791d59aaff61b2778e8ada3316d
111+
xyz12be 3c50a51a3c486a0c6853e4bbbcf3f244
112+
xyz12le e020897d826ea20ded16f30ea1eb018d
113113
ya16be 70fa41c32ecaf3370edc38add6096db2
114114
ya16le 3b2c20f9e80717628ced6c6468507f63
115115
ya8 4ad5920716de3d2fbbc49f95adb60345

‎tests/ref/fate/filter-pixfmts-il

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,8 @@ xv36be 3bbb949278ea55cc947ee03bd9c27c2d
110110
xv36le 102c0e817d375ddd6b2cfbb4262dec95
111111
xv48be 4d7376651fb7b3e84d00abad6c785aad
112112
xv48le a1a8ff16d9a864568e5e557734bf3d6d
113-
xyz12be 7c7d54c55f136cbbc50b18029f3be0b3
114-
xyz12le 090ba6b1170baf2b1358b43b971d33b0
113+
xyz12be b7d50e283360bf69fd661369110b26ef
114+
xyz12le d5b1d45c3a136bb3d04f70a619c86c8d
115115
y210le d4cf9b53cd7ff22f087743d483e88480
116116
y212le d5a2b4677ddb4a3bc3e5cd5cbb20f426
117117
y216le 9e44c6d76b09bcbe71738423b4b3d67a

‎tests/ref/fate/filter-pixfmts-null

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,8 @@ xv36be 9f556ee59a672fd8725f0bb36ce3e4b0
111111
xv36le e08dcbde02f1c28a3554f372ad1278e2
112112
xv48be ce34993b4b4411bba1d852b9b86aa39e
113113
xv48le df913a7e61b162aa98303e5393e60c63
114-
xyz12be a1ef56bf746d71f59669c28e48fc8450
115-
xyz12le 831ff03c1ba4ef19374686f16a064d8c
114+
xyz12be f257f86373207af8aed0a1a05171df3b
115+
xyz12le 7922f99edc44a2c26a25becbea9914cc
116116
y210le 04e9487b6cce38e7531437e946cdd586
117117
y212le 825768be8fe92708ae80be84855066ed
118118
y216le 0e99aeddfee304e72d525d72998d9e9b

‎tests/ref/fate/filter-pixfmts-scale

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,8 @@ xv36be 4d084adca0228d7750d1e2e877e0d79b
111111
xv36le de9c74e94dc19c828e1572aa283d8aca
112112
xv48be 9e58d1a045df100b0dec116e13be5b4e
113113
xv48le fd873d53609b2fbdfe99470f515a234c
114-
xyz12be c7ba8345998c0141ddc079cdd29b1a40
115-
xyz12le 95f5d3a0de834cc495c9032a14987cde
114+
xyz12be f1905012d9b845306d9bef68d0fc81d5
115+
xyz12le cfe1a3bbe391d83d381f590a00e1a16d
116116
y210le 7c2aef142d88ab343ec01acd45f38466
117117
y212le 39a3c0c843041ad4501b3107dd91ef17
118118
y216le 17be2999e97d36b8ed903f07ef428c09

‎tests/ref/fate/filter-pixfmts-transpose

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,8 +100,8 @@ xv36be 2261a0e3db5ee607d37f68d19704ae15
100100
xv36le 9202133de91bf64c76ca27d5cd0c816a
101101
xv48be 14373b7fe123225689e76fe2ce43fb93
102102
xv48le 319df9724a067c7b5efa215f9f54d127
103-
xyz12be 68e5cba640f6e4ef72dff950e88b5342
104-
xyz12le 8b6b6a6db4d7561e80db88ccaecce7a9
103+
xyz12be 69737aceb508a73365664d04c340dd3b
104+
xyz12le 70dd5fab9d8383b0d2e772b3b6569df4
105105
ya16be 6098f7d2ede0aab6b2d93d2b4f4d915a
106106
ya16le 1fae63e3e320ba9e6c12c29a48c44eff
107107
ya8 d4b7a62f80681fa44c977ff3a64f4ce4

‎tests/ref/fate/filter-pixfmts-vflip

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,8 @@ xv36be c0272372d3e1a59adb3931ee433a5d5b
111111
xv36le ffe6ab75ebc09134c3451f8f6ef0d501
112112
xv48be bdfc3217ae456b370dbdcf4d52606a3f
113113
xv48le 53dbebab73a66539c04644ef56dc6bbb
114-
xyz12be 23fa9fb36d49dce61e284d41b83e0e6b
115-
xyz12le ef73e6d1f932a9a355df1eedd628394f
114+
xyz12be 1bffa153a4a3ae61fd18e370f95161d9
115+
xyz12le a0e93443826621a9d6c48354d949898a
116116
y210le f8847bedd3ae6e1c0cf84a823f275e31
117117
y212le c801725ae31e3b8f5be269359d49f191
118118
y216le 985db498aedf3fb1c547ad07442b7258

‎tests/ref/pixfmt/gbrp-xyz12le

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
86b542ae625ef5a61cef6b0b703f553f *tests/data/pixfmt/gbrp-xyz12le.yuv
1+
543fe8482e02d14790b2dc48ec0812e8 *tests/data/pixfmt/gbrp-xyz12le.yuv
22
7603200 tests/data/pixfmt/gbrp-xyz12le.yuv

‎tests/ref/pixfmt/gbrp10-xyz12le

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
64eb126dca4cf2bae0b5c7677ed7f266 *tests/data/pixfmt/gbrp10-xyz12le.yuv
1+
68d9a907c0b823c6af7b8cc253b2d148 *tests/data/pixfmt/gbrp10-xyz12le.yuv
22
15206400 tests/data/pixfmt/gbrp10-xyz12le.yuv

‎tests/ref/pixfmt/gbrp12-xyz12le

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
6780af8d13a4d8dcbf479d86f1099e23 *tests/data/pixfmt/gbrp12-xyz12le.yuv
1+
6aeb56c9f41206c8f4a2439b9b755e81 *tests/data/pixfmt/gbrp12-xyz12le.yuv
22
15206400 tests/data/pixfmt/gbrp12-xyz12le.yuv

‎tests/ref/pixfmt/rgb24-xyz12le

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
460b2b2fc5611d1125c87d5f7716511c *tests/data/pixfmt/rgb24-xyz12le.yuv
1+
be4b641ec6f5b3c534d6b69e3144c8ca *tests/data/pixfmt/rgb24-xyz12le.yuv
22
7603200 tests/data/pixfmt/rgb24-xyz12le.yuv

‎tests/ref/pixfmt/rgb48-xyz12le

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
16a15582e2fe7ff56951ca9d06d3a7a3 *tests/data/pixfmt/rgb48-xyz12le.yuv
1+
380feb3972a3fd9f4a839585adda3f9b *tests/data/pixfmt/rgb48-xyz12le.yuv
22
15206400 tests/data/pixfmt/rgb48-xyz12le.yuv

‎tests/ref/pixfmt/xyz12le

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
a5a4f73b21d5e2b07789f41d84265d9a *tests/data/pixfmt/xyz12le.yuv
1+
47b890678bb9195107c2f7afb2344bbf *tests/data/pixfmt/xyz12le.yuv
22
304128 tests/data/pixfmt/xyz12le.yuv

‎tests/ref/pixfmt/yuv444p-xyz12le

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
a0acdd9c25a84c8698d0dc1e52950c8d *tests/data/pixfmt/yuv444p-xyz12le.yuv
1+
54a3bdec56e545c9ae2782e78b010f76 *tests/data/pixfmt/yuv444p-xyz12le.yuv
22
7603200 tests/data/pixfmt/yuv444p-xyz12le.yuv

‎tests/ref/pixfmt/yuv444p10-xyz12le

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
52b4ba6b111bd781c99ed8350e3b25e1 *tests/data/pixfmt/yuv444p10-xyz12le.yuv
1+
749b383b2fb0920de0a35010fca42acb *tests/data/pixfmt/yuv444p10-xyz12le.yuv
22
15206400 tests/data/pixfmt/yuv444p10-xyz12le.yuv

‎tests/ref/pixfmt/yuv444p12-xyz12le

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
7dd2de39e7bc112ebd94f39284b16f06 *tests/data/pixfmt/yuv444p12-xyz12le.yuv
1+
3e19343dbcf62d1aa817c30643408439 *tests/data/pixfmt/yuv444p12-xyz12le.yuv
22
15206400 tests/data/pixfmt/yuv444p12-xyz12le.yuv

0 commit comments

Comments
 (0)