Support 'yield from' (Python 3.3) and await (Python 3.5) expressions.

Currently unsupported are yield from in newer Python versions, and
yield/yield from into an assignment expression.

ASTNode.h

@@ -14,7 +14,7 @@ public:
         NODE_DELETE, NODE_FUNCTION, NODE_CLASS, NODE_CALL, NODE_IMPORT,
         NODE_TUPLE, NODE_LIST, NODE_MAP, NODE_SUBSCR, NODE_PRINT,
         NODE_CONVERT, NODE_KEYWORD, NODE_RAISE, NODE_EXEC, NODE_BLOCK,
-        NODE_COMPREHENSION, NODE_LOADBUILDCLASS,
+        NODE_COMPREHENSION, NODE_LOADBUILDCLASS, NODE_AWAITABLE,
 
         // Empty node types
         NODE_LOCALS,
@@ -180,7 +180,7 @@ private:
 class ASTReturn : public ASTNode {
 public:
     enum RetType {
-        RETURN, YIELD
+        RETURN, YIELD, YIELD_FROM
     };
 
     ASTReturn(PycRef<ASTNode> value, RetType rettype = RETURN)
@@ -599,4 +599,15 @@ private:
     PycRef<PycObject> m_obj;
 };
 
+class ASTAwaitable : public ASTNode {
+public:
+    ASTAwaitable(PycRef<ASTNode> expr)
+        : ASTNode(NODE_AWAITABLE), m_expr(std::move(expr)) { }
+
+    PycRef<ASTNode> expression() const { return m_expr; }
+
+private:
+    PycRef<ASTNode> m_expr;
+};
+
 #endif

ASTree.cpp

@@ -782,6 +782,13 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
                 stack.push(NULL); // We can totally hack this >_>
             }
             break;
+        case Pyc::GET_AWAITABLE:
+            {
+                PycRef<ASTNode> object = stack.top();
+                stack.pop();
+                stack.push(new ASTAwaitable(object));
+            }
+            break;
         case Pyc::GET_ITER:
             /* We just entirely ignore this */
             break;
@@ -2097,6 +2104,16 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
                 stack.push(new ASTTuple(vals));
             }
             break;
+        case Pyc::YIELD_FROM:
+            {
+                PycRef<ASTNode> dest = stack.top();
+                stack.pop();
+                // TODO: Support yielding into a non-null destination
+                PycRef<ASTNode> value = stack.top();
+                value->setProcessed();
+                curblock->append(new ASTReturn(value, ASTReturn::YIELD_FROM));
+            }
+            break;
         case Pyc::YIELD_VALUE:
             {
                 PycRef<ASTNode> value = stack.top();
@@ -2515,17 +2532,26 @@ void print_src(PycRef<ASTNode> node, PycModule* mod)
     case ASTNode::NODE_RETURN:
         {
             PycRef<ASTReturn> ret = node.cast<ASTReturn>();
+            PycRef<ASTNode> value = ret->value();
             if (!inLambda) {
                 switch (ret->rettype()) {
-                    case ASTReturn::RETURN:
-                        fputs("return ", pyc_output);
-                        break;
-                    case ASTReturn::YIELD:
-                        fputs("yield ", pyc_output);
-                        break;
+                case ASTReturn::RETURN:
+                    fputs("return ", pyc_output);
+                    break;
+                case ASTReturn::YIELD:
+                    fputs("yield ", pyc_output);
+                    break;
+                case ASTReturn::YIELD_FROM:
+                    if (value.type() == ASTNode::NODE_AWAITABLE) {
+                        fputs("await ", pyc_output);
+                        value = value.cast<ASTAwaitable>()->expression();
+                    } else {
+                        fputs("yield from ", pyc_output);
+                    }
+                    break;
                 }
             }
-            print_src(ret->value(), mod);
+            print_src(value, mod);
         }
         break;
     case ASTNode::NODE_SLICE:

tests/input/async_def.py

@@ -1,2 +1,5 @@
 async def foobar():
     pass
+
+async def barfoo():
+    await foobar()

tests/input/yield_from.py

@@ -0,0 +1,11 @@
+def gen10():
+    for i in range(10):
+        yield i
+
+def gen20():
+    for i in range(10, 20):
+        yield i
+
+def generator():
+    yield from gen10()
+    yield from gen20()

tests/tokenized/async_def.txt

@@ -1,3 +1,7 @@
 async def foobar ( ) : <EOL>
 <INDENT>
 pass <EOL>
+<OUTDENT>
+async def barfoo ( ) : <EOL>
+<INDENT>
+await foobar ( ) <EOL>

tests/tokenized/yield_from.txt

@@ -0,0 +1,18 @@
+def gen10 ( ) : <EOL>
+<INDENT>
+for i in range ( 10 ) : <EOL>
+<INDENT>
+yield i <EOL>
+<OUTDENT>
+<OUTDENT>
+def gen20 ( ) : <EOL>
+<INDENT>
+for i in range ( 10 , 20 ) : <EOL>
+<INDENT>
+yield i <EOL>
+<OUTDENT>
+<OUTDENT>
+def generator ( ) : <EOL>
+<INDENT>
+yield from gen10 ( ) <EOL>
+yield from gen20 ( ) <EOL>