Generators in Python 3

Presentation given at Mumbai Technology Meetup by Anand Chitipothu on March 15, 2015.

The talk will be given via Google Hangouts on Air from 4PM to 5PM IST. Any one can join it by visiting the hangout page.

This webpage will be available at:

http://anandology.com/nb/2015/python3-generators-mtp/

It'll be updated along with the talk. You'll have to reload the web page to see latest contents.

Iterators

In [2]:
for a in [1, 2, 3, 4]:
    print(a)

1
2
3
4
In [3]:
for a in (1, 2, 3, 4):
    print(a)

1
2
3
4
In [4]:
for c in "hello":
    print(c)

h
e
l
l
o
In [5]:
for k in {"x": 1, "y": 2}:
    print(k)

x
y
In [6]:
max([1, 2, 3, 4])
Out[6]:
4
In [8]:
max("helloworld")
Out[8]:
'w'
In [9]:
",".join("hello")
Out[9]:
'h,e,l,l,o'
In [10]:
"-".join(["hello", "world"])
Out[10]:
'hello-world'
In [11]:
"-".join({"x": 1, "y": 2})
Out[11]:
'x-y'

Iteration Protocol

In [12]:
x = [1, 2, 3, 4]
it = iter(x)
In [13]:
it
Out[13]:
<list_iterator at 0x104da2fd0>
In [14]:
next(it)
Out[14]:
1
In [15]:
next(it)
Out[15]:
2
In [16]:
next(it)
Out[16]:
3
In [17]:
next(it)
Out[17]:
4
In [18]:
next(it)

---------------------------------------------------------------------------
StopIteration                             Traceback (most recent call last)
<ipython-input-18-2cdb14c0d4d6> in <module>()
----> 1 next(it)

StopIteration:

Generators

In [19]:
def numbers():
    yield 1
    yield 2
    yield 3
    

n = numbers()
In [20]:
print(n)

<generator object numbers at 0x104d9d630>
In [21]:
next(n)
Out[21]:
1
In [22]:
next(n)
Out[22]:
2
In [23]:
next(n)
Out[23]:
3
In [24]:
next(n)

---------------------------------------------------------------------------
StopIteration                             Traceback (most recent call last)
<ipython-input-24-1c47c7af397e> in <module>()
----> 1 next(n)

StopIteration:
In [25]:
def numbers():
    print("begin numbers")
    yield 1
    print("after 1")
    yield 2
    print("after 2")    
    yield 3
    print("end numbers")    
    
n = numbers()
In [26]:
next(n)

begin numbers
Out[26]:
1
In [27]:
next(n)

after 1
Out[27]:
2
In [28]:
next(n)

after 2
Out[28]:
3
In [29]:
next(n)

end numbers

---------------------------------------------------------------------------
StopIteration                             Traceback (most recent call last)
<ipython-input-29-1c47c7af397e> in <module>()
----> 1 next(n)

StopIteration:
In [31]:
for i in numbers():
    print(i)

begin numbers
1
after 1
2
after 2
3
end numbers
In [32]:
sum(numbers())

begin numbers
after 1
after 2
end numbers
Out[32]:
6
In [33]:
max(numbers())

begin numbers
after 1
after 2
end numbers
Out[33]:
3
In [34]:
def squares(nums):
    """Takes a sequence of numbers and returns 
    a sequence of their squares.
    """
    for n in nums:
        yield n*n

nums = range(100)
numsq = squares(nums)
print(sum(numsq))    

328350
In [35]:
!seq 1000 > numbers.txt
In [38]:
%%file sumofsq1.py
# usual way of computing sum of squares

def main(filename):
    result = 0
    for line in open(filename):
        n = int(line)
        n_sq = n*n
        result += n_sq
    print(result)
        
if __name__ == "__main__":
    import sys
    main(sys.argv[1])
    

Overwriting sumofsq1.py
In [39]:
!python sumofsq1.py numbers.txt

333833500
In [46]:
%%file sumofsq2.py
# computing sum of squares using generators

def ints(strings):
    for s in strings:
        yield int(s)
        
def squares(nums):
    for n in nums:
        yield n*n

def main(filename):
#     lines = open(filename)
#     numbers = ints(lines)
#     sq = squares(numbers)
#     print(sum(sq))
    print(sum(squares(ints(open(filename)))))
        
if __name__ == "__main__":
    import sys
    main(sys.argv[1])
    

Overwriting sumofsq2.py
In [47]:
!python sumofsq2.py numbers.txt

333833500

Generator Expressions

In [51]:
def squares(nums):
    return (n*n for n in nums)


print(list(squares(range(10))))

[0, 1, 4, 9, 16, 25, 36, 49, 64, 81]
In [52]:
print(sum(squares(range(10))))

285
In [53]:
sum((n*n for n in range(10)))
Out[53]:
285
In [54]:
sum(n*n for n in range(10))
Out[54]:
285
In [55]:
sum(n*n for n in range(1000000))
Out[55]:
333332833333500000
In [56]:
sum(n*n for n in range(1000000) if n%2 == 0)
Out[56]:
166666166667000000

Example: Tree Traversing

In [57]:
class Node:
    def __init__(self, value, left=None, right=None):
        self.value = value
        self.left = left
        self.right = right
       

n1 = Node(5)
n2 = Node(4)
n3 = Node(2)
n12 = Node(3, n1, n2)
root = Node(10, n12, n3)

def inorder(root):
    if root.left:
        values = inorder(root.left)
        for v in values:
            yield v
    yield root.value
    if root.right: 
        values = inorder(root.right)
        for v in values:
            yield v
        
print(list(inorder(root)))

[5, 3, 4, 10, 2]
In [58]:
def inorder(root):
    if root.left:
        yield from inorder(root.left)
    yield root.value
    if root.right: 
        yield from inorder(root.right)
        
print(list(inorder(root)))

[5, 3, 4, 10, 2]

Example: Cooperative multithreading

In [85]:
%%file multitask.py
import sys

def f(n, label="f"):
    for i in range(n):
        print(label, i)
        yield

def g(n, label="g"):
    for i in range(n):
        print(label, i)
        yield

def h(n):
    print("begin h")
    yield from f(n, label="h-f")
    print('after f in h')
    #yield
    yield from g(n, label="h-g")
    print("end h")        
    
tasks = []
def start_task(f, *args):
    tasks.append(f(*args))
    
def run_tasks():
    while tasks:
        for t in tasks[:]:
            try:
                next(t)
            except StopIteration:
                tasks.remove(t)

def main():
    start_task(f, 4)
    #start_task(g, 7)
    start_task(h, 2)    
    run_tasks()

if __name__ == "__main__":
    main()

Overwriting multitask.py
In [84]:
!python multitask.py

f 0
begin h
h-f 0
f 1
h-f 1
f 2
after f in h
h-g 0
f 3
h-g 1
end h
In [86]:
!python multitask.py

f 0
begin h
h-f 0
f 1
h-f 1
f 2
after f in h
f 3
h-g 0
h-g 1
end h
In [ ]: