Merge pull request #1849 from markshannon/python-add-syntax-comments

Python: Add syntax example comments for automatic document generation.

python/ql/src/semmle/python/AstExtended.qll

@@ -3,6 +3,11 @@ import python
 /** Syntactic node (Class, Function, Module, Expr, Stmt or Comprehension) corresponding to a flow node */
 abstract class AstNode extends AstNode_ {
 
+    /* Special comment for documentation generation.
+     * All subclasses of `AstNode` that represent concrete syntax should have
+     * a comment of the form: */
+    /* syntax: ... */
+
     /** Gets the scope that this node occurs in */
     abstract Scope getScope();
 
@@ -206,6 +211,8 @@ class ComprehensionList extends ComprehensionList_ {
 /** A list of expressions */
 class ExprList extends ExprList_ {
 
+    /* syntax: Expr, ... */
+
 }
 
 

python/ql/src/semmle/python/Class.qll

@@ -63,6 +63,8 @@ class ClassExpr extends ClassExpr_ {
 /** A class statement. Note that ClassDef extends Assign as a class definition binds the newly created class */
 class ClassDef extends Assign {
 
+    /* syntax: class name(...): ... */
+
     ClassDef() {
         /* This is an artificial assignment the rhs of which is a (possibly decorated) ClassExpr */
         exists(ClassExpr c | this.getValue() = c or this.getValue() = c.getADecoratorCall())

python/ql/src/semmle/python/Exprs.qll

@@ -131,6 +131,8 @@ class Expr extends Expr_, AstNode {
 /** An attribute expression, such as `value.attr` */
 class Attribute extends Attribute_ {
 
+    /* syntax: Expr.name */
+
     override Expr getASubExpression() {
         result = this.getObject()
     }
@@ -160,6 +162,8 @@ class Attribute extends Attribute_ {
 /** A subscript expression, such as `value[slice]` */
 class Subscript extends Subscript_ {
 
+    /* syntax: Expr[Expr] */
+
     override Expr getASubExpression() {
         result = this.getIndex()
         or
@@ -176,6 +180,8 @@ class Subscript extends Subscript_ {
 /** A call expression, such as `func(...)` */
 class Call extends Call_ {
 
+    /* syntax: Expr(...) */
+
     override Expr getASubExpression() {
         result = this.getAPositionalArg() or
         result = this.getAKeyword().getValue() or
@@ -268,6 +274,8 @@ class Call extends Call_ {
 /** A conditional expression such as, `body if test else orelse` */
 class IfExp extends IfExp_ {
 
+    /* syntax: Expr if Expr else Expr */
+
     override Expr getASubExpression() {
         result = this.getTest() or result = this.getBody() or result = this.getOrelse()
     }
@@ -278,6 +286,8 @@ class IfExp extends IfExp_ {
 /** A starred expression, such as the `*rest` in the assignment `first, *rest = seq` */
 class Starred extends Starred_ {
 
+    /* syntax: *Expr */
+
     override Expr getASubExpression() {
         result = this.getValue()
     }
@@ -288,6 +298,8 @@ class Starred extends Starred_ {
 /** A yield expression, such as `yield value` */
 class Yield extends Yield_ {
 
+    /* syntax: yield Expr */
+
     override Expr getASubExpression() {
         result = this.getValue()
     }
@@ -301,6 +313,8 @@ class Yield extends Yield_ {
 /** A yield expression, such as `yield from value` */
 class YieldFrom extends YieldFrom_ {
 
+    /* syntax: yield from Expr */
+
     override Expr getASubExpression() {
         result = this.getValue()
     }
@@ -314,6 +328,8 @@ class YieldFrom extends YieldFrom_ {
 /** A repr (backticks) expression, such as `` `value` `` */
 class Repr extends Repr_ {
 
+    /* syntax: `Expr` */
+
     override Expr getASubExpression() {
         result = this.getValue()
     }
@@ -330,6 +346,8 @@ class Repr extends Repr_ {
    `"hello"` are treated as Bytes for Python2, but Unicode for Python3. */
 class Bytes extends StrConst {
 
+    /* syntax: b"hello" */
+
     Bytes() {
         not this.isUnicode()
     }
@@ -355,6 +373,8 @@ class Bytes extends StrConst {
 /** An ellipsis expression, such as `...` */
 class Ellipsis extends Ellipsis_ {
 
+    /* syntax: ... */
+
     override Expr getASubExpression() {
         none()
     }
@@ -394,6 +414,8 @@ abstract class Num extends Num_, ImmutableLiteral {
 /** An integer numeric constant, such as `7` or `0x9` */
 class IntegerLiteral extends Num {
 
+    /* syntax: 4 */
+
     IntegerLiteral() {
         not this instanceof FloatLiteral and not this instanceof ImaginaryLiteral
     }
@@ -425,6 +447,8 @@ class IntegerLiteral extends Num {
 /** A floating point numeric constant, such as `0.4` or `4e3` */
 class FloatLiteral extends Num {
 
+    /* syntax: 4.2 */
+
     FloatLiteral() {
         not this instanceof ImaginaryLiteral and
         this.getN().regexpMatch(".*[.eE].*")
@@ -456,6 +480,8 @@ class FloatLiteral extends Num {
 class ImaginaryLiteral extends Num {
     private float value;
 
+    /* syntax: 1.0j */
+
     ImaginaryLiteral() {
         value = this.getN().regexpCapture("(.+)j.*", 1).toFloat()
     }
@@ -513,6 +539,8 @@ class NegativeIntegerLiteral extends ImmutableLiteral, UnaryExpr {
    "hello" are treated as Bytes for Python2, but Unicode for Python3. */
 class Unicode extends StrConst {
 
+    /* syntax: "hello" */
+
     Unicode() {
         this.isUnicode()
     }
@@ -541,6 +569,8 @@ class Unicode extends StrConst {
 /** A dictionary expression, such as `{'key':'value'}` */
 class Dict extends Dict_ {
 
+    /* syntax: {Expr: Expr, ...} */
+
     /** Gets the value of an item of this dict display */
     Expr getAValue() {
         result = this.getAnItem().(DictDisplayItem).getValue()
@@ -566,6 +596,8 @@ class Dict extends Dict_ {
 /** A list expression, such as `[ 1, 3, 5, 7, 9 ]` */
 class List extends List_ {
 
+    /* syntax: [Expr, ...] */
+
     override Expr getASubExpression() {
         result = this.getAnElt()
     }
@@ -575,6 +607,8 @@ class List extends List_ {
 /** A set expression such as `{ 1, 3, 5, 7, 9 }` */
 class Set extends Set_ {
 
+    /* syntax: {Expr, ...} */
+
     override Expr getASubExpression() {
         result = this.getAnElt()
     }
@@ -601,6 +635,8 @@ class PlaceHolder extends PlaceHolder_ {
 /** A tuple expression such as `( 1, 3, 5, 7, 9 )` */
 class Tuple extends Tuple_ {
 
+    /* syntax: (Expr, ...) */
+
     override Expr getASubExpression() {
         result = this.getAnElt()
     }
@@ -612,6 +648,8 @@ class Tuple extends Tuple_ {
  */
 class Name extends Name_ {
 
+    /* syntax: name */
+
     string getId() {
         result = this.getVariable().getId()
     }
@@ -705,6 +743,8 @@ class Slice extends Slice_ {
 /** A string constant. */
 class StrConst extends Str_, ImmutableLiteral {
 
+    /* syntax: "hello" */
+
     predicate isUnicode() {
         this.getPrefix().charAt(_) = "u"
         or
@@ -790,6 +830,8 @@ abstract class BooleanLiteral extends NameConstant {
 /** The boolean named constant `True` */
 class True extends BooleanLiteral {
 
+    /* syntax: True */
+
     True() {
         name_consts(this, "True")
     }
@@ -807,6 +849,8 @@ class True extends BooleanLiteral {
 /** The boolean named constant `False` */
 class False extends BooleanLiteral {
 
+    /* syntax: False */
+
     False() {
         name_consts(this, "False")
     }
@@ -824,6 +868,8 @@ class False extends BooleanLiteral {
 /** `None` */
 class None extends NameConstant {
 
+    /* syntax: None */
+
     None() {
         name_consts(this, "None")
     }
@@ -840,6 +886,8 @@ class None extends NameConstant {
 /** An await expression such as `await coro`. */
 class Await extends Await_ {
 
+    /* syntax: await Expr */
+
     override Expr getASubExpression() {
         result = this.getValue()
     }
@@ -849,6 +897,8 @@ class Await extends Await_ {
 /** A formatted string literal expression, such as `f'hello {world!s}'` */
 class Fstring extends Fstring_ {
 
+    /* syntax: f"Yes!" */
+
     override Expr getASubExpression() {
         result = this.getAValue()
     }

python/ql/src/semmle/python/Function.qll

@@ -187,6 +187,8 @@ class Function extends Function_, Scope, AstNode {
 /** A def statement. Note that FunctionDef extends Assign as a function definition binds the newly created function */
 class FunctionDef extends Assign {
 
+    /* syntax: def name(...): ... */
+
     FunctionDef() {
         /* This is an artificial assignment the rhs of which is a (possibly decorated) FunctionExpr */
         exists(FunctionExpr f | this.getValue() = f or this.getValue() = f.getADecoratorCall())

python/ql/src/semmle/python/Import.qll

@@ -170,6 +170,8 @@ class ImportMember extends ImportMember_ {
 /** An import statement */
 class Import extends Import_ {
 
+    /* syntax: import modname */
+
     private ImportExpr getAModuleExpr() {
         result = this.getAName().getValue()
         or 
@@ -222,6 +224,8 @@ class Import extends Import_ {
 /** An import * statement */
 class ImportStar extends ImportStar_ {
 
+    /* syntax: from modname import * */
+
     ImportExpr getModuleExpr() {
         result = this.getModule()
         or

python/ql/src/semmle/python/Keywords.qll

@@ -2,6 +2,8 @@ import python
 
 class KeyValuePair extends KeyValuePair_, DictDisplayItem {
 
+    /* syntax: Expr : Expr */
+
     override Location getLocation() {
         result = KeyValuePair_.super.getLocation()
     }
@@ -76,6 +78,8 @@ abstract class DictDisplayItem extends DictItem {
 /** A keyword argument in a call. For example `arg=expr` in `foo(0, arg=expr)` */
 class Keyword extends Keyword_, DictUnpackingOrKeyword  {
 
+    /* syntax: name = Expr */
+
     override Location getLocation() {
         result = Keyword_.super.getLocation()
     }

python/ql/src/semmle/python/Stmts.qll

@@ -98,6 +98,8 @@ class Assign extends Assign_ {
 /** An assignment statement */
 class AssignStmt extends Assign {
 
+    /* syntax: Expr, ... = Expr */
+
     AssignStmt() {
         not this instanceof FunctionDef and not this instanceof ClassDef
     }
@@ -110,6 +112,8 @@ class AssignStmt extends Assign {
 /** An augmented assignment statement, such as `x += y` */
 class AugAssign extends AugAssign_ {
 
+    /* syntax: Expr += Expr */
+
     override Expr getASubExpression() {
         result = this.getOperation()
     }
@@ -130,6 +134,8 @@ class AugAssign extends AugAssign_ {
 /** An annotated assignment statement, such as `x: int = 0` */
 class AnnAssign extends AnnAssign_ {
 
+    /* syntax: Expr: Expr = Expr */
+
     override Expr getASubExpression() {
         result = this.getAnnotation() or
         result = this.getTarget() or
@@ -156,6 +162,8 @@ class AnnAssign extends AnnAssign_ {
 /** An exec statement */
 class Exec extends Exec_ {
 
+    /* syntax: exec Expr */
+
     override Expr getASubExpression() {
         result = this.getBody() or
         result = this.getGlobals() or
@@ -171,6 +179,8 @@ class Exec extends Exec_ {
 /** An except statement (part of a `try` statement), such as `except IOError as err:` */
 class ExceptStmt extends ExceptStmt_ {
 
+    /* syntax: except Expr [ as Expr ]: */
+
     /** Gets the immediately enclosing try statement */
     Try getTry() {
         result.getAHandler() = this
@@ -195,6 +205,8 @@ class ExceptStmt extends ExceptStmt_ {
 /** An assert statement, such as `assert a == b, "A is not equal to b"` */
 class Assert extends Assert_ {
 
+    /* syntax: assert Expr [, Expr] */
+
     override Expr getASubExpression() {
         result = this.getMsg() or result = this.getTest()
     }
@@ -208,6 +220,8 @@ class Assert extends Assert_ {
 /** A break statement */
 class Break extends Break_ {
 
+    /* syntax: assert Expr [, Expr] */
+
     override Expr getASubExpression() {
         none()
     }
@@ -221,6 +235,8 @@ class Break extends Break_ {
 /** A continue statement */
 class Continue extends Continue_ {
 
+    /* syntax: continue */
+
     override Expr getASubExpression() {
         none()
     }
@@ -234,6 +250,8 @@ class Continue extends Continue_ {
 /** A delete statement, such as `del x[-1]` */
 class Delete extends Delete_ {
 
+    /* syntax: del Expr, ... */
+
     override Expr getASubExpression() {
         result = this.getATarget()
     }
@@ -247,6 +265,8 @@ class Delete extends Delete_ {
 /** An expression statement, such as `len(x)` or `yield y` */
 class ExprStmt extends ExprStmt_ {
 
+    /* syntax: Expr */
+
     override Expr getASubExpression() {
         result = this.getValue()
     }
@@ -260,6 +280,8 @@ class ExprStmt extends ExprStmt_ {
 /** A for statement, such as `for x in y: print(x)` */
 class For extends For_ {
 
+    /* syntax: for varname in Expr: ... */
+
     override Stmt getASubStatement() {
         result = this.getAStmt() or
         result = this.getAnOrelse()
@@ -279,6 +301,8 @@ class For extends For_ {
 /** A global statement, such as `global var` */
 class Global extends Global_ {
 
+    /* syntax: global varname */
+
     override Expr getASubExpression() {
         none()
     }
@@ -291,6 +315,8 @@ class Global extends Global_ {
 /** An if statement, such as `if eggs: print("spam")` */
 class If extends If_ {
 
+    /* syntax: if Expr: ... */
+
     override Stmt getASubStatement() {
         result = this.getAStmt() or
         result = this.getAnOrelse()
@@ -345,6 +371,8 @@ class If extends If_ {
 /** A nonlocal statement, such as `nonlocal var` */
 class Nonlocal extends Nonlocal_ {
 
+    /* syntax: nonlocal varname */
+
     override Stmt getASubStatement() {
         none()
     }
@@ -363,6 +391,8 @@ class Nonlocal extends Nonlocal_ {
 /** A pass statement */
 class Pass extends Pass_ {
 
+    /* syntax: pass */
+
     override Stmt getASubStatement() {
         none()
     }
@@ -376,6 +406,8 @@ class Pass extends Pass_ {
 /** A print statement (Python 2 only), such as `print 0` */
 class Print extends Print_ {
 
+    /* syntax: print Expr, ... */
+
     override Stmt getASubStatement() {
         none()
     }
@@ -390,6 +422,8 @@ class Print extends Print_ {
 /** A raise statement, such as `raise CompletelyDifferentException()` */
 class Raise extends Raise_ {
 
+    /* syntax: raise Expr */
+
     override Stmt getASubStatement() {
         none()
     }
@@ -424,6 +458,8 @@ class Raise extends Raise_ {
 /** A return statement, such as return None */
 class Return extends Return_ {
 
+    /* syntax: return Expr */
+
     override Stmt getASubStatement() {
         none()
     }
@@ -437,6 +473,8 @@ class Return extends Return_ {
 /** A try statement */
 class Try extends Try_ {
 
+    /* syntax: try: ... */
+
     override Expr getASubExpression() {
         none()
      }
@@ -475,6 +513,8 @@ class Try extends Try_ {
 /** A while statement, such as `while parrot_resting():` */
 class While extends While_ {
 
+    /* syntax: while Expr: ... */
+
     override Expr getASubExpression() {
         result = this.getTest()
     }
@@ -496,6 +536,8 @@ class While extends While_ {
 /** A with statement such as `with f as open("file"): text = f.read()` */
 class With extends With_ {
 
+    /* syntax: with Expr as varname: ... */
+
     override Expr getASubExpression() {
         result = this.getContextExpr() or
         result = this.getOptionalVars()
@@ -526,6 +568,8 @@ class TemplateWrite extends TemplateWrite_ {
 
 class AsyncFor extends For {
 
+    /* syntax: async for varname in Expr: ... */
+
     AsyncFor() {
         this.isAsync()
     }
@@ -534,6 +578,8 @@ class AsyncFor extends For {
 
 class AsyncWith extends With {
 
+    /* syntax: async with Expr as varname: ... */
+
     AsyncWith() {
         this.isAsync()
     }