Ticket #13820: 13820.patch

File 13820.patch, 1.6 KB (added by ohnx56, 6 years ago)

  • src/system/libroot/posix/malloc/wrapper.cpp 

    From e01c6767e0da5bc72d8593c1ae518d3429641196 Mon Sep 17 00:00:00 2001
From: Mason X <me@masonx.ca>
Date: Fri, 1 Dec 2017 13:45:59 +0000
Subject: [PATCH] Catch integer overflow in calloc (Fixes #13820)

calloc, when multiplying to find the true size of memory desired
by a user, will occasionally overflow if the numbers are too
large. This commit makes calloc ensure that the multiplication
does not overflow by dividing the result by one of the numbers,
and checking to ensure that the result is equal to the other
number. If there is overflow, it is treated as an OOM error.
---
 src/system/libroot/posix/malloc/wrapper.cpp | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/system/libroot/posix/malloc/wrapper.cpp b/src/system/libroot/posix/malloc/wrapper.cpp
index 38c4125dff..4465f512ea 100644
    a b calloc(size_t nelem, size_t elsize)  
    328328{
    329329    static processHeap *pHeap = getAllocator();
    330330    size_t size = nelem * elsize;
     331    void *ptr = NULL;
     332
     333    if ((nelem > 0) && ((size/nelem) != elsize))
     334        goto nomem;
    331335
    332336#if HEAP_WALL
    333337    size += 2 * HEAP_WALL_SIZE;
     338
     339    if (nelem == 0 || elsize == 0)
     340        goto ok;
     341    if (size < (nelem * size)&& size < (elsize * size))
     342        goto nomem;
    334343#endif
    335344
     345ok:
    336346    defer_signals();
    337347
    338     void *ptr = pHeap->getHeap(pHeap->getHeapIndex()).malloc(size);
     348    ptr = pHeap->getHeap(pHeap->getHeapIndex()).malloc(size);
    339349    if (ptr == NULL) {
    340350        undefer_signals();
     351    nomem:
    341352        __set_errno(B_NO_MEMORY);
    342353        KTRACE("calloc(%lu, %lu) -> NULL", nelem, elsize);
    343354        return NULL;