1 year ago
#324765
Mike T
Error: unpack_from requires a buffer of at least 3235956936 bytes
I am writing a program to open a gzip file. The script should read an entire compressed binary TAQ quotes file into memory, uncompress it, and gives its clients access to the contents of the file via a set of get methods. Here is my script:
import gzip
import struct
class TAQQuotesReader(object):
def __init__(self, filePathName ):
'''
Do all of the heavy lifting here and give users getters for the
results.
'''
self._filePathName = filePathName
with gzip.open( self._filePathName, 'rb') as f:
file_content = f.read()
self._header = struct.unpack_from(">2i",file_content[0:8])
# millis from midnight
endI = 8 + ( 4 * self._header[1] )
self._ts = struct.unpack_from( ( ">%di" % self._header[ 1 ] ), file_content[ 8:endI ] )
startI = endI
# bid size
endI = endI + ( 4 * self._header[1] )
self._bs = struct.unpack_from( ( ">%di" % self._header[ 1 ] ), file_content[ startI:endI ] )
startI = endI
# bid price
endI = endI + ( 4 * self._header[1] )
self._bp = struct.unpack_from( ( ">%df" % self._header[ 1 ] ), file_content[ startI:endI ] )
startI = endI
# ask size
endI = endI + ( 4 * self._header[1] )
self._as = struct.unpack_from( ( ">%di" % self._header[ 1 ] ), file_content[ startI:endI ] )
startI = endI
# ask price
endI = endI + ( 4 * self._header[1] )
self._ap = struct.unpack_from( ( ">%df" % self._header[ 1 ] ), file_content[ startI:endI ] )
def getN(self):
return self._header[1]
def getSecsFromEpocToMidn(self):
return self._header[0]
def getMillisFromMidn( self, index ):
return self._ts[ index ]
def getAskSize( self, index ):
return self._as[ index ]
def getAskPrice( self, index ):
return self._ap[ index ]
def getBidSize( self, index ):
return self._bs[ index ]
def getBidPrice( self, index ):
return self._bp[ index ]
How I used the script:
filePathName="quote.gz"
TAQQuotesReader(filePathName)
However, I get the following error:
error Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_9204/690688085.py in <module>
1 filePathName="quote.gz"
----> 2 TAQQuotesReader(filePathName)
~\AppData\Local\Temp/ipykernel_9204/3810628070.py in __init__(self, filePathName)
18 # millis from midnight
19 endI = 8 + ( 4 * self._header[1] )
---> 20 self._ts = struct.unpack_from( ( ">%di" % self._header[ 1 ] ), file_content[ 8:endI ] )
21 startI = endI
22
error: unpack_from requires a buffer of at least 3235956936 bytes for unpacking 3235956936 bytes at offset 0 (actual buffer size is 191352312)
How do I debug the error above?
python
python-3.x
struct
gzip
unpack
0 Answers
Your Answer