Skip to content

Fix HTJ2K bytes-per-line integer overflow in internal_ht.cpp #2342

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
cary-ilm wants to merge 1 commit into from
+35 −15
Closed

Fix HTJ2K bytes-per-line integer overflow in internal_ht.cpp #2342

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 35 additions & 15 deletions src/lib/OpenEXRCore/internal_ht.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
** Copyright Contributors to the OpenEXR Project.
*/

#include <cstdint>
#include <limits>
#include <string>
#include <fstream>
Expand Down Expand Up @@ -183,11 +184,20 @@ ht_undo_impl (
if (file_i >= decode->channel_count)
return EXR_ERR_CORRUPT_CHUNK;

size_t computedoffset = 0;
uint64_t computedoffset = 0;
for (int i = 0; i < file_i; ++i)
computedoffset += decode->channels[i].width *
decode->channels[i].bytes_per_element;
cs_to_file_ch[cs_i].raster_line_offset = computedoffset;
{
int32_t w = decode->channels[i].width;
int8_t bpe = decode->channels[i].bytes_per_element;
if (w < 0 || bpe < 0) return EXR_ERR_CORRUPT_CHUNK;
Copy link
Copy Markdown
Contributor

@palemieux palemieux Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

w and bpe are the responsibility of the caller, shouldn't the check happen there?

Copy link
Copy Markdown
Member Author

@cary-ilm cary-ilm Apr 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's true that width and bytes_per_element should have valid values before entry, but the test here still seems worthwhile before a cast that would fail if they're invalid. It's a simply test, I'd vote for leaving it in place.

Do you understand the code path well enough to understand exactly where that validation should happen if not here?

Copy link
Copy Markdown
Contributor

@palemieux palemieux Apr 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Validation already happens at https://github.com/AcademySoftwareFoundation/openexr/blob/main/src/lib/OpenEXRCore/validation.c

uint64_t term = (uint64_t) w * (uint64_t) bpe;
if (computedoffset > std::numeric_limits<uint64_t>::max () - term)
return EXR_ERR_CORRUPT_CHUNK;
computedoffset += term;
}
if (computedoffset > std::numeric_limits<size_t>::max ())
return EXR_ERR_CORRUPT_CHUNK;
cs_to_file_ch[cs_i].raster_line_offset = (size_t) computedoffset;
}

ojph::mem_infile infile;
Expand All @@ -208,16 +218,20 @@ ht_undo_impl (
|| decode->channel_count != siz.get_num_components())
return EXR_ERR_CORRUPT_CHUNK;

int bpl = 0;
bool is_planar = false;
int64_t bpl = 0;
bool is_planar = false;
for (int16_t c = 0; c < decode->channel_count; c++)
{
bpl +=
decode->channels[c].bytes_per_element * decode->channels[c].width;
int64_t term = (int64_t) decode->channels[c].bytes_per_element *
(int64_t) decode->channels[c].width;
if (term < 0) return EXR_ERR_CORRUPT_CHUNK;
bpl += term;
if (decode->channels[c].x_samples > 1 ||
decode->channels[c].y_samples > 1)
{ is_planar = true; }
}
if (bpl > (int64_t) std::numeric_limits<int>::max ())
return EXR_ERR_CORRUPT_CHUNK;
cs.set_planar (is_planar);

cs.create ();
Expand Down Expand Up @@ -286,7 +300,7 @@ ht_undo_impl (
{
uint8_t* line_pixels = static_cast<uint8_t*> (uncompressed_data);

assert (bpl * image_height == uncompressed_size);
assert (static_cast<uint64_t> (bpl) * image_height == uncompressed_size);

for (uint32_t y = 0; y < image_height; ++y)
{
Expand Down Expand Up @@ -316,7 +330,7 @@ ht_undo_impl (
}
}
}
line_pixels += bpl;
line_pixels += static_cast<ptrdiff_t> (bpl);
}
}

Expand Down Expand Up @@ -368,7 +382,7 @@ ht_apply_impl (exr_encode_pipeline_t* encode)

bool isPlanar = false;
siz.set_num_components (encode->channel_count);
int bpl = 0;
int64_t bpl = 0;
for (int16_t c = 0; c < encode->channel_count; c++)
{
int file_c = cs_to_file_ch[c].file_index;
Expand All @@ -388,10 +402,15 @@ ht_apply_impl (exr_encode_pipeline_t* encode)
encode->channels[file_c].y_samples > 1)
{ isPlanar = true; }

bpl += encode->channels[file_c].bytes_per_element *
encode->channels[file_c].width;
int64_t term = (int64_t) encode->channels[file_c].bytes_per_element *
(int64_t) encode->channels[file_c].width;
if (term < 0) return EXR_ERR_CORRUPT_CHUNK;
bpl += term;
}

if (bpl > (int64_t) std::numeric_limits<int>::max ())
return EXR_ERR_CORRUPT_CHUNK;

cs.set_planar (isPlanar);

siz.set_image_offset (ojph::point (0, 0));
Expand Down Expand Up @@ -480,7 +499,8 @@ ht_apply_impl (exr_encode_pipeline_t* encode)
const uint8_t* line_pixels =
static_cast<const uint8_t*> (encode->packed_buffer);

assert (bpl * image_height == encode->packed_bytes);
assert (static_cast<uint64_t> (bpl) * (uint64_t) image_height ==
encode->packed_bytes);

for (int y = 0; y < image_height; y++)
{
Expand Down Expand Up @@ -511,7 +531,7 @@ ht_apply_impl (exr_encode_pipeline_t* encode)
assert (next_comp == c);
cur_line = cs.exchange (cur_line, next_comp);
}
line_pixels += bpl;
line_pixels += static_cast<ptrdiff_t> (bpl);
}
}

Expand Down
Loading