Attachment 311570 Details for Bug 455008 – Proposed patch from David against the release25-maint branch of python

[patch] Proposed patch from David against the release25-maint branch of python

CVE-2008-2315-release25-maint.diff (text/plain), 17.81 KB, created by Jan Lieskovsky on 2008-07-11 13:10:34 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Jan Lieskovsky

Created: 2008-07-11 13:10:34 UTC

Size: 17.81 KB

patch

obsolete

>Index: Objects/unicodeobject.c
>===================================================================
>--- Objects/unicodeobject.c	(revision 64642)
>+++ Objects/unicodeobject.c	(working copy)
>@@ -255,6 +255,8 @@
> 	    }
> 	}
>         else {
>+            if (length > (PY_SSIZE_T_MAX - 1) / sizeof(Py_UNICODE))
>+                return NULL;
> 	    size_t new_size = sizeof(Py_UNICODE) * ((size_t)length + 1);
> 	    unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size);
>         }
>@@ -265,7 +267,9 @@
>         unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type);
>         if (unicode == NULL)
>             return NULL;
>-	new_size = sizeof(Py_UNICODE) * ((size_t)length + 1);
>+        if (length > (PY_SSIZE_T_MAX - 1) / sizeof(Py_UNICODE))
>+            return NULL;
>+        new_size = sizeof(Py_UNICODE) * ((size_t)length + 1);	
> 	unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size);
>     }
> 
>@@ -1095,6 +1099,9 @@
>     char * out;
>     char * start;
> 
>+    if (cbAllocated / 5 != size)
>+        return PyErr_NoMemory();
>+
>     if (size == 0)
> 		return PyString_FromStringAndSize(NULL, 0);
> 
>@@ -1693,8 +1700,9 @@
> {
>     PyObject *v;
>     unsigned char *p;
>+    Py_ssize_t nsize, bytesize;
> #ifdef Py_UNICODE_WIDE
>-    int i, pairs;
>+    Py_ssize_t i, pairs;
> #else
>     const int pairs = 0;
> #endif
>@@ -1717,8 +1725,14 @@
> 	if (s[i] >= 0x10000)
> 	    pairs++;
> #endif
>-    v = PyString_FromStringAndSize(NULL,
>-		  2 * (size + pairs + (byteorder == 0)));
>+    /* 2 * (size + pairs + (byteorder == 0)) */
>+    if (size > PY_SSIZE_T_MAX || size > PY_SSIZE_T_MAX - pairs - (byteorder == 0))
>+	return PyErr_NoMemory();
>+    nsize = (size + pairs + (byteorder == 0));
>+    bytesize = nsize * 2;
>+    if (bytesize / 2 != nsize)
>+	return PyErr_NoMemory();
>+    v = PyString_FromStringAndSize(NULL, bytesize);
>     if (v == NULL)
>         return NULL;
> 
>@@ -2045,7 +2059,12 @@
>     PyObject *repr;
>     char *p;
> 
>-    static const char *hexdigit = "0123456789abcdef";
>+    static const char *hexdigit = "0123456789abcdef";	
>+#ifdef Py_UNICODE_WIDE
>+    const Py_ssize_t expandsize = 10;
>+#else
>+    const Py_ssize_t expandsize = 6;
>+#endif
> 
>     /* Initial allocation is based on the longest-possible unichr
>        escape.
>@@ -2061,13 +2080,12 @@
>        escape.
>     */
> 
>+    if (size > (PY_SSIZE_T_MAX - 2 - 1) / expandsize )
>+	return PyErr_NoMemory();
>+
>     repr = PyString_FromStringAndSize(NULL,
>         2
>-#ifdef Py_UNICODE_WIDE
>-        + 10*size
>-#else
>-        + 6*size
>-#endif
>+        + expandsize*size
>         + 1);
>     if (repr == NULL)
>         return NULL;
>@@ -2319,13 +2337,17 @@
>     char *p;
>     char *q;
> 
>-    static const char *hexdigit = "0123456789abcdef";
>-
>+    static const char *hexdigit = "0123456789abcdef";	
> #ifdef Py_UNICODE_WIDE
>-    repr = PyString_FromStringAndSize(NULL, 10 * size);
>+    const Py_ssize_t expandsize = 10;
> #else
>-    repr = PyString_FromStringAndSize(NULL, 6 * size);
>+    const Py_ssize_t expandsize = 6;
> #endif
>+    
>+    if (size > PY_SSIZE_T_MAX / expandsize )
>+	return PyErr_NoMemory();
>+    
>+    repr = PyString_FromStringAndSize(NULL, expandsize * size);
>     if (repr == NULL)
>         return NULL;
>     if (size == 0)
>@@ -4761,7 +4783,13 @@
>         return self;
>     }
> 
>-    u = _PyUnicode_New(left + self->length + right);
>+    if (left > PY_SSIZE_T_MAX - self->length || right > PY_SSIZE_T_MAX - (left + self->length)
>+            ||
>+        !(u = _PyUnicode_New(left + self->length + right))
>+    ) {
>+        PyErr_SetString(PyExc_OverflowError, "padded string is too long");
>+        return NULL;
>+    }
>     if (u) {
>         if (left)
>             Py_UNICODE_FILL(u->str, fill, left);
>Index: Objects/tupleobject.c
>===================================================================
>--- Objects/tupleobject.c	(revision 64642)
>+++ Objects/tupleobject.c	(working copy)
>@@ -60,11 +60,12 @@
> 		Py_ssize_t nbytes = size * sizeof(PyObject *);
> 		/* Check for overflow */
> 		if (nbytes / sizeof(PyObject *) != (size_t)size ||
>-		    (nbytes += sizeof(PyTupleObject) - sizeof(PyObject *))
>-		    <= 0)
>+		    (nbytes > PY_SSIZE_T_MAX - sizeof(PyTupleObject) - sizeof(PyObject *)))
> 		{
> 			return PyErr_NoMemory();
> 		}
>+		nbytes += sizeof(PyTupleObject) - sizeof(PyObject *);
>+
> 		op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
> 		if (op == NULL)
> 			return NULL;
>Index: Objects/bufferobject.c
>===================================================================
>--- Objects/bufferobject.c	(revision 64642)
>+++ Objects/bufferobject.c	(working copy)
>@@ -427,6 +427,11 @@
> 		count = 0;
> 	if (!get_buf(self, &ptr, &size, ANY_BUFFER))
> 		return NULL;
>+	if (count > PY_SSIZE_T_MAX / size) {
>+		PyErr_SetString(PyExc_MemoryError,
>+				"result too large");
>+		return NULL;
>+	}
> 	ob = PyString_FromStringAndSize(NULL, size * count);
> 	if ( ob == NULL )
> 		return NULL;
>Index: Objects/longobject.c
>===================================================================
>--- Objects/longobject.c	(revision 64642)
>+++ Objects/longobject.c	(working copy)
>@@ -70,6 +70,8 @@
> 		PyErr_NoMemory();
> 		return NULL;
> 	}
>+	/* This can overflow -- PyObject_NEW_VAR /  _PyObject_VAR_SIZE
>+	   need to detect overflow */
> 	return PyObject_NEW_VAR(PyLongObject, &PyLong_Type, size);
> }
> 
>Index: Objects/stringobject.c
>===================================================================
>--- Objects/stringobject.c	(revision 64642)
>+++ Objects/stringobject.c	(working copy)
>@@ -972,13 +972,18 @@
> 		return (PyObject *)a;
> 	}
> 	size = a->ob_size + b->ob_size;
>-	if (size < 0) {
>+	if (a->ob_size < 0 || b->ob_size < 0 || a->ob_size > PY_SSIZE_T_MAX - b->ob_size) {
> 		PyErr_SetString(PyExc_OverflowError,
> 				"strings are too large to concat");
> 		return NULL;
> 	}
> 	  
> 	/* Inline PyObject_NewVar */
>+	if (size > PY_SSIZE_T_MAX - sizeof(PyStringObject)) {
>+		PyErr_SetString(PyExc_OverflowError,
>+				"strings are too large to concat");
>+		return NULL;
>+	}
> 	op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size);
> 	if (op == NULL)
> 		return PyErr_NoMemory();
>Index: Modules/mmapmodule.c
>===================================================================
>--- Modules/mmapmodule.c	(revision 64642)
>+++ Modules/mmapmodule.c	(working copy)
>@@ -223,7 +223,7 @@
> 		return(NULL);
> 
> 	/* silently 'adjust' out-of-range requests */
>-	if ((self->pos + num_bytes) > self->size) {
>+	if (num_bytes > self->size - self->pos) {
> 		num_bytes -= (self->pos+num_bytes) - self->size;
> 	}
> 	result = Py_BuildValue("s#", self->data+self->pos, num_bytes);
>Index: Modules/stropmodule.c
>===================================================================
>--- Modules/stropmodule.c	(revision 64642)
>+++ Modules/stropmodule.c	(working copy)
>@@ -216,6 +216,11 @@
> 				return NULL;
> 			}
> 			slen = PyString_GET_SIZE(item);
>+			if (slen > PY_SSIZE_T_MAX - reslen || seplen > PY_SSIZE_T_MAX - reslen - seplen) {
>+				PyErr_SetString(PyExc_OverflowError, "input too long");
>+				Py_DECREF(res);
>+				return NULL;
>+			}
> 			while (reslen + slen + seplen >= sz) {
> 				if (_PyString_Resize(&res, sz * 2) < 0)
> 					return NULL;
>@@ -253,6 +258,12 @@
> 			return NULL;
> 		}
> 		slen = PyString_GET_SIZE(item);
>+		if (slen > PY_SSIZE_T_MAX - reslen || seplen > PY_SSIZE_T_MAX - reslen - seplen) {
>+			PyErr_SetString(PyExc_OverflowError, "input too long");
>+			Py_DECREF(res);
>+			Py_XDECREF(item);
>+			return NULL;
>+		}
> 		while (reslen + slen + seplen >= sz) {
> 			if (_PyString_Resize(&res, sz * 2) < 0) {
> 				Py_DECREF(item);
>Index: Modules/gcmodule.c
>===================================================================
>--- Modules/gcmodule.c	(revision 64642)
>+++ Modules/gcmodule.c	(working copy)
>@@ -1318,7 +1318,10 @@
> _PyObject_GC_Malloc(size_t basicsize)
> {
> 	PyObject *op;
>-	PyGC_Head *g = (PyGC_Head *)PyObject_MALLOC(
>+	PyGC_Head *g;
>+	if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head))
>+		return PyErr_NoMemory();
>+	g = (PyGC_Head *)PyObject_MALLOC(
>                 sizeof(PyGC_Head) + basicsize);
> 	if (g == NULL)
> 		return PyErr_NoMemory();
>@@ -1361,6 +1364,8 @@
> {
> 	const size_t basicsize = _PyObject_VAR_SIZE(op->ob_type, nitems);
> 	PyGC_Head *g = AS_GC(op);
>+	if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head))
>+		return (PyVarObject *)PyErr_NoMemory();
> 	g = (PyGC_Head *)PyObject_REALLOC(g,  sizeof(PyGC_Head) + basicsize);
> 	if (g == NULL)
> 		return (PyVarObject *)PyErr_NoMemory();
>Index: Lib/test/test_strop.py
>===================================================================
>--- Lib/test/test_strop.py	(revision 64642)
>+++ Lib/test/test_strop.py	(working copy)
>@@ -115,7 +115,26 @@
>         strop.uppercase
>         strop.whitespace
> 
>+    @test_support.precisionbigmemtest(size=test_support._2G - 1, memuse=5)
>+    def test_stropjoin_huge_list(self, size):
>+        a = "A" * size
>+        try:
>+            r = strop.join([a, a], a)
>+        except OverflowError:
>+            pass
>+        else:
>+            self.assertEquals(len(r), len(a) * 3)
> 
>+    @test_support.precisionbigmemtest(size=test_support._2G - 1, memuse=1)
>+    def test_stropjoin_huge_tup(self, size):
>+        a = "A" * size
>+        try:
>+            r = strop.join((a, a), a)
>+        except OverflowError:
>+            pass # acceptable on 32-bit
>+        else:
>+            self.assertEquals(len(r), len(a) * 3)
>+
> transtable = '\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037 !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`xyzdefghijklmnopqrstuvwxyz{|}~\177\200\201\202\203\204\205\206\207\210\211\212\213\214\215\216\217\220\221\222\223\224\225\226\227\230\231\232\233\234\235\236\237\240\241\242\243\244\245\246\247\250\251\252\253\254\255\256\257\260\261\262\263\264\265\266\267\270\271\272\273\274\275\276\277\300\301\302\303\304\305\306\307\310\311\312\313\314\315\316\317\320\321\322\323\324\325\326\327\330\331\332\333\334\335\336\337\340\341\342\343\344\345\346\347\350\351\352\353\354\355\356\357\360\361\362\363\364\365\366\367\370\371\372\373\374\375\376\377'
> 
> 
>Index: Lib/test/test_bigmem.py
>===================================================================
>--- Lib/test/test_bigmem.py	(revision 64642)
>+++ Lib/test/test_bigmem.py	(working copy)
>@@ -1,5 +1,5 @@
> from test import test_support
>-from test.test_support import bigmemtest, _1G, _2G
>+from test.test_support import bigmemtest, _1G, _2G, _4G, precisionbigmemtest
> 
> import unittest
> import operator
>@@ -53,7 +53,23 @@
>             lpadsize += 1
>         self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)
>         self.assertEquals(s.strip(), SUBSTR.strip())
>-
>+    
>+    @precisionbigmemtest(size=_2G - 1, memuse=1)
>+    def test_center_unicode(self, size):
>+        SUBSTR = u' abc def ghi'
>+        try:
>+            s = SUBSTR.center(size)
>+        except OverflowError:
>+            pass # acceptable on 32-bit
>+        else:
>+            self.assertEquals(len(s), size)
>+            lpadsize = rpadsize = (len(s) - len(SUBSTR)) // 2
>+            if len(s) % 2:
>+                lpadsize += 1
>+            self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)
>+            self.assertEquals(s.strip(), SUBSTR.strip())
>+            del s
>+    
>     @bigmemtest(minsize=_2G, memuse=2)
>     def test_count(self, size):
>         SUBSTR = ' abc def ghi'
>@@ -69,11 +85,45 @@
>     def test_decode(self, size):
>         s = '.' * size
>         self.assertEquals(len(s.decode('utf-8')), size)
>+    
>+    def basic_encode_test(self, size, enc, c=u'.', expectedsize=None):
>+        if expectedsize is None:
>+            expectedsize = size
>+        
>+        s = c * size
>+        self.assertEquals(len(s.encode(enc)), expectedsize)
> 
>     @bigmemtest(minsize=_2G + 2, memuse=3)
>     def test_encode(self, size):
>-        s = u'.' * size
>-        self.assertEquals(len(s.encode('utf-8')), size)
>+        return self.basic_encode_test(size, 'utf-8')
>+    
>+    @precisionbigmemtest(size=_4G / 6 + 2, memuse=2)
>+    def test_encode_raw_unicode_escape(self, size):
>+        try:
>+            return self.basic_encode_test(size, 'raw_unicode_escape')
>+        except MemoryError:
>+            pass # acceptable on 32-bit
>+    
>+    @precisionbigmemtest(size=_4G / 5 + 70, memuse=3)
>+    def test_encode_utf7(self, size):
>+        try:
>+            return self.basic_encode_test(size, 'utf7')
>+        except MemoryError:
>+            pass # acceptable on 32-bit
>+    
>+    @precisionbigmemtest(size=_2G-1, memuse=2)
>+    def test_decodeascii(self, size):
>+        return self.basic_encode_test(size, 'ascii', c='A')
>+    
>+    @precisionbigmemtest(size=_4G / 5, memuse=6+2)
>+    def test_unicode_repr_oflw(self, size):
>+        try:
>+            s = u"\uAAAA"*size
>+            r = repr(s)
>+        except MemoryError:
>+            pass # acceptable on 32-bit
>+        else:
>+            self.failUnless(s == eval(r))
> 
>     @bigmemtest(minsize=_2G, memuse=2)
>     def test_endswith(self, size):
>@@ -458,7 +508,12 @@
>         self.assertEquals(s[-1], "'")
>         self.assertEquals(s.count('\\'), size)
>         self.assertEquals(s.count('0'), size * 2)
>-
>+    
>+    @bigmemtest(minsize=2**32 / 5, memuse=6+2)
>+    def test_unicode_repr(self, size):
>+        s = u"\uAAAA" * size
>+        self.failUnless(len(repr(s)) > size)
>+    
>     # This test is meaningful even with size < 2G, as long as the
>     # doubled string is > 2G (but it tests more if both are > 2G :)
>     @bigmemtest(minsize=_1G + 2, memuse=3)
>@@ -642,6 +697,35 @@
>     def test_repeat_large(self, size):
>         return self.basic_test_repeat(size)
> 
>+    @bigmemtest(minsize=_1G - 1, memuse=12)
>+    def test_repeat_large_2(self, size):
>+        return self.basic_test_repeat(size)
>+    
>+    @precisionbigmemtest(size=_1G - 1, memuse=9)
>+    def test_from_2G_generator(self, size):
>+        try:
>+            t = tuple(xrange(size))
>+        except MemoryError:
>+            pass # acceptable on 32-bit
>+        else:
>+            count = 0
>+            for item in t:
>+                self.assertEquals(item, count)
>+                count += 1
>+            self.assertEquals(count, size)
>+    
>+    @precisionbigmemtest(size=_1G - 25, memuse=9)
>+    def test_from_almost_2G_generator(self, size):
>+        try:
>+            t = tuple(xrange(size))
>+            count = 0
>+            for item in t:
>+                self.assertEquals(item, count)
>+                count += 1
>+            self.assertEquals(count, size)
>+        except MemoryError:
>+            pass # acceptable, expected on 32-bit
>+
>     # Like test_concat, split in two.
>     def basic_test_repr(self, size):
>         t = (0,) * size
>@@ -957,9 +1041,35 @@
>         self.assertEquals(l[:10], [1] * 10)
>         self.assertEquals(l[-10:], [5] * 10)
> 
>+class BufferTest(unittest.TestCase):
>+    
>+    @precisionbigmemtest(size=_1G, memuse=4)
>+    def test_repeat(self, size):
>+        try:
>+            b = buffer("AAAA")*size
>+        except MemoryError:
>+            pass # acceptable on 32-bit
>+        else:
>+            count = 0
>+            for c in b:
>+                self.assertEquals(c, 'A')
>+                count += 1
>+            self.assertEquals(count, size*4)
>+    
> def test_main():
>-    test_support.run_unittest(StrTest, TupleTest, ListTest)
>+    test_support.run_unittest(StrTest, TupleTest, ListTest, BufferTest)
> 
>+# Expected failures (crashers)
>+# del StrTest.test_center_unicode
>+del StrTest.test_decodeascii
>+# del StrTest.test_encode_utf32
>+# del StrTest.test_encode_utf7
>+# del StrTest.test_encode_raw_unicode_escape
>+# 
>+# del TupleTest.test_from_2G_generator
>+#
>+# del BufferTest.test_repeat
>+
> if __name__ == '__main__':
>     if len(sys.argv) > 1:
>         test_support.set_memlimit(sys.argv[1])
>Index: Lib/test/test_support.py
>===================================================================
>--- Lib/test/test_support.py	(revision 64642)
>+++ Lib/test/test_support.py	(working copy)
>@@ -33,6 +33,7 @@
> use_resources = None     # Flag set to [] by regrtest.py
> max_memuse = 0           # Disable bigmem tests (they will still be run with
>                          # small sizes, to make sure they work.)
>+real_max_memuse = 0
> 
> # _original_stdout is meant to hold stdout at the time regrtest began.
> # This may be "the real" stdout, or IDLE's emulation of stdout, or whatever.
>@@ -323,6 +324,7 @@
> _1M = 1024*1024
> _1G = 1024 * _1M
> _2G = 2 * _1G
>+_4G = 4 * _1G
> 
> # Hack to get at the maximum value an internal index can take.
> class _Dummy:
>@@ -333,6 +335,7 @@
> def set_memlimit(limit):
>     import re
>     global max_memuse
>+    global real_max_memuse
>     sizes = {
>         'k': 1024,
>         'm': _1M,
>@@ -344,6 +347,7 @@
>     if m is None:
>         raise ValueError('Invalid memory limit %r' % (limit,))
>     memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()])
>+    real_max_memuse = memlimit
>     if memlimit > MAX_Py_ssize_t:
>         memlimit = MAX_Py_ssize_t
>     if memlimit < _2G - 1:
>@@ -389,6 +393,27 @@
>         return wrapper
>     return decorator
> 
>+def precisionbigmemtest(size, memuse, overhead=5*_1M):
>+    def decorator(f):
>+        def wrapper(self):
>+            if not real_max_memuse:
>+                maxsize = 5147
>+            else:
>+                maxsize = size
>+            
>+                if real_max_memuse and real_max_memuse < maxsize * memuse:
>+                    if verbose:
>+                        sys.stderr.write("Skipping %s because of memory "
>+                                         "constraint\n" % (f.__name__,))
>+                    return
>+            
>+            return f(self, maxsize)
>+        wrapper.size = size
>+        wrapper.memuse = memuse
>+        wrapper.overhead = overhead
>+        return wrapper
>+    return decorator
>+
> def bigaddrspacetest(f):
>     """Decorator for tests that fill the address space."""
>     def wrapper(self):
>Index: Lib/test/seq_tests.py
>===================================================================
>--- Lib/test/seq_tests.py	(revision 64642)
>+++ Lib/test/seq_tests.py	(working copy)
>@@ -307,6 +307,8 @@
>             self.assertEqual(id(s), id(s*1))
> 
>     def test_bigrepeat(self):
>+        import sys
>+        if sys.maxint <= 2147483647:
>         x = self.type2test([0])
>         x *= 2**16
>         self.assertRaises(MemoryError, x.__mul__, 2**16)

Actions: View | Diff

Attachments on bug 455008: 311570