Operation Manual

Experiments in Python
106
Notes:
Lesson 3.6: Limited resources – memory & storage
import temple, heapq
# this is a generator (notice the yield)
def linesFromFile(sortedFile):
while True:
element = sortedFile.readline().strip()
if not element: # no element, so le is empty
break
yield element
# open the lmlist (this doesn't read anything yet)
lms = open('lmlist', 'r')
generatorList = []
while True:
# read up-to 10 lines into an array
elementsToSort = []
for i in range( 10):
lm = lms.readline() # just reads one line
if not lm:
break
elementsToSort.append(lm)
# if no lines were read, we're done reading the very large le
if not elementsToSort:
break
# create a temporary le and sort the 10 items into that le
sortedFile = temple.TemporaryFile()
for item in sorted( e l e m e n t s T o S o r t ):
sortedFile.write(bytes(it e m .e n c o d e()))
# return this le to the start ready for
# reading back by the heapq
sortedFile.seek(0)
# put the generator into the generatorList array;
# remember this function isn't executed
# until a loop requests the next value.
generatorList.append(linesFromFile(sortedFile))
# use the magical heapq merge, which will merge two sorted lists
# together but it will only pull the data in as it is needed
for element in heapq.merge(*generatorList):
print(element.decode())