1 year ago

#324765

test-img

Mike T

Error: unpack_from requires a buffer of at least 3235956936 bytes

I am writing a program to open a gzip file. The script should read an entire compressed binary TAQ quotes file into memory, uncompress it, and gives its clients access to the contents of the file via a set of get methods. Here is my script:

import gzip
import struct

class TAQQuotesReader(object):
      def __init__(self, filePathName ):
        '''
        Do all of the heavy lifting here and give users getters for the
        results.
        '''
        self._filePathName = filePathName
        with gzip.open( self._filePathName, 'rb') as f:
            file_content = f.read()
            self._header = struct.unpack_from(">2i",file_content[0:8])
            
            # millis from midnight
            endI = 8 + ( 4 * self._header[1] )
            self._ts = struct.unpack_from( ( ">%di" % self._header[ 1 ] ), file_content[ 8:endI ] )
            startI = endI
            
            # bid size
            endI = endI + ( 4 * self._header[1] )
            self._bs = struct.unpack_from( ( ">%di" % self._header[ 1 ] ), file_content[ startI:endI ] )
            startI = endI

            # bid price
            endI = endI + ( 4 * self._header[1] )
            self._bp = struct.unpack_from( ( ">%df" % self._header[ 1 ] ), file_content[ startI:endI ] )
            startI = endI
            
            # ask size
            endI = endI + ( 4 * self._header[1] )
            self._as = struct.unpack_from( ( ">%di" % self._header[ 1 ] ), file_content[ startI:endI ] )
            startI = endI

            # ask price
            endI = endI + ( 4 * self._header[1] )
            self._ap = struct.unpack_from( ( ">%df" % self._header[ 1 ] ), file_content[ startI:endI ] )

    def getN(self):
        return self._header[1]
    
    def getSecsFromEpocToMidn(self):
        return self._header[0]
    
    def getMillisFromMidn( self, index ):
        return self._ts[ index ]

    def getAskSize( self, index ):
        return self._as[ index ]
    
    def getAskPrice( self, index ):
        return self._ap[ index ]

    def getBidSize( self, index ):
        return self._bs[ index ]
    
    def getBidPrice( self, index ):
        return self._bp[ index ]

How I used the script:

filePathName="quote.gz"
TAQQuotesReader(filePathName)

However, I get the following error:

error                                     Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_9204/690688085.py in <module>
      1 filePathName="quote.gz"
----> 2 TAQQuotesReader(filePathName)

~\AppData\Local\Temp/ipykernel_9204/3810628070.py in __init__(self, filePathName)
     18             # millis from midnight
     19             endI = 8 + ( 4 * self._header[1] )
---> 20             self._ts = struct.unpack_from( ( ">%di" % self._header[ 1 ] ), file_content[ 8:endI ] )
     21             startI = endI
     22 

error: unpack_from requires a buffer of at least 3235956936 bytes for unpacking 3235956936 bytes at offset 0 (actual buffer size is 191352312)

How do I debug the error above?

python

python-3.x

struct

gzip

unpack

0 Answers

Your Answer

Accepted video resources