Load the WebIDL files specified in the binding

This loads the WebIDL specified in the bindings into an Abstract
Syntax Tree (AST) and performs the mixin operations for implements.

Additionally the specs now use a slightly extended IDL syntax. Instead
of wholesale implementing the second edition of the IDL spec the
parser has been updated to cope with iterator and Promise keywords as
those are the only changes used in the dom and html specifications.

A bug was also fixed in the lexer where negative int literals were
not recognised.

6 files changed, 279 insertions, 55 deletions

diff --git a/src/nsgenbind-ast.c b/src/nsgenbind-ast.c
index 1b5f53c..c1acee1 100644
--- a/src/nsgenbind-ast.c
+++ b/src/nsgenbind-ast.c
@@ -20,6 +20,11 @@
 #include "nsgenbind-ast.h"
 #include "options.h"
 
+/**
+ * standard IO handle for parse trace logging.
+ */
+static FILE *genbind_parsetracef;
+
 /* parser and lexer interface */
 extern int nsgenbind_debug;
 extern int nsgenbind__flex_debug;
@@ -448,23 +453,23 @@ static int genbind_ast_dump(FILE *dfile, struct genbind_node *node, int indent)
 /* exported interface documented in nsgenbind-ast.h */
 int genbind_dump_ast(struct genbind_node *node)
 {
-    FILE *dumpf;
+        FILE *dumpf;
 
-    /* only dump AST to file if required */
-    if (!options->debug) {
-        return 0;
-    }
+        /* only dump AST to file if required */
+        if (!options->debug) {
+                return 0;
+        }
 
-    dumpf = genb_fopen("binding-ast", "w");
-    if (dumpf == NULL) {
-        return 2;
-    }
+        dumpf = genb_fopen("binding-ast", "w");
+        if (dumpf == NULL) {
+                return 2;
+        }
 
-    genbind_ast_dump(dumpf, node, 0);
+        genbind_ast_dump(dumpf, node, 0);
 
-    fclose(dumpf);
+        fclose(dumpf);
 
-    return 0;
+        return 0;
 }
 
 FILE *genbindopen(const char *filename)
@@ -545,10 +550,6 @@ FILE *genbindopen(const char *filename)
         return genfile;
 }
 
-/**
- * standard IO handle for parse trace logging.
- */
-static FILE *genbind_parsetracef;
 
 int genbind_parsefile(char *infilename, struct genbind_node **ast)
 {
diff --git a/src/nsgenbind.c b/src/nsgenbind.c
index 8d83d13..914f58e 100644
--- a/src/nsgenbind.c
+++ b/src/nsgenbind.c
@@ -15,6 +15,7 @@
 #include <errno.h>
 
 #include "nsgenbind-ast.h"
+#include "webidl-ast.h"
 #include "jsapi-libdom.h"
 #include "options.h"
 
@@ -104,6 +105,51 @@ static int generate_binding(struct genbind_node *binding_node, void *ctx)
         return res;
 }
 
+
+static int webidl_file_cb(struct genbind_node *node, void *ctx)
+{
+	struct webidl_node **webidl_ast = ctx;
+	char *filename;
+
+	filename = genbind_node_gettext(node);
+
+        if (options->verbose) {
+                printf("Opening IDL file \"%s\"\n", filename);
+        }
+
+	return webidl_parsefile(filename, webidl_ast);
+}
+
+static int genbind_load_idl(struct genbind_node *genbind,
+                            struct webidl_node **webidl_out)
+{
+        int res;
+        struct genbind_node *binding_node;
+
+        binding_node = genbind_node_find_type(genbind, NULL,
+                                              GENBIND_NODE_TYPE_BINDING);
+
+	/* walk AST and load any web IDL files required */
+	res = genbind_node_foreach_type(
+                genbind_node_getnode(binding_node),
+                GENBIND_NODE_TYPE_WEBIDL,
+                webidl_file_cb,
+                webidl_out);
+	if (res != 0) {
+		fprintf(stderr, "Error: failed reading Web IDL\n");
+		return -1;
+	}
+
+        /* implements are implemented as mixins so intercalate them */
+        res = webidl_intercalate_implements(*webidl_out);
+	if (res != 0) {
+		fprintf(stderr, "Error: Failed to intercalate implements\n");
+		return -1;
+	}
+
+        return 0;
+}
+
 /**
  * get the type of binding
  */
@@ -147,6 +193,7 @@ int main(int argc, char **argv)
 {
         int res;
         struct genbind_node *genbind_root = NULL;
+        struct webidl_node *webidl_root = NULL;
         enum bindingtype_e bindingtype;
 
         options = process_cmdline(argc, argv);
@@ -170,8 +217,16 @@ int main(int argc, char **argv)
                 return 3;
         }
 
+        /* load the IDL files specified in the binding */
+        res = genbind_load_idl(genbind_root, &webidl_root);
+        if (res != 0) {
+                return 4;
+        }
+
+	/* debug dump of web idl AST */
+        webidl_dump_ast(webidl_root);
+
 #if 0
-        genbind_load_idl(genbind_root);
 
         /* generate output for each binding */
         res = genbind_node_foreach_type(genbind_root,
diff --git a/src/webidl-ast.c b/src/webidl-ast.c
index 945b5fa..6c24c41 100644
--- a/src/webidl-ast.c
+++ b/src/webidl-ast.c
@@ -11,10 +11,17 @@
 #include <stdio.h>
 #include <string.h>
 #include <errno.h>
+#include <stdarg.h>
 
+#include "utils.h"
 #include "webidl-ast.h"
 #include "options.h"
 
+/**
+ * standard IO handle for parse trace logging.
+ */
+static FILE *webidl_parsetracef;
+
 extern int webidl_debug;
 extern int webidl__flex_debug;
 extern void webidl_restart(FILE*);
@@ -386,13 +393,16 @@ static const char *webidl_node_type_to_str(enum webidl_node_type type)
 
 }
 
-
-/* exported interface defined in webidl-ast.h */
-int webidl_ast_dump(struct webidl_node *node, int indent)
+/**
+ * Recursively dump the AST nodes increasing indent as appropriate
+ */
+static int webidl_ast_dump(FILE *dumpf, struct webidl_node *node, int indent)
 {
-	const char *SPACES="                                                                               ";	char *txt;
+	const char *SPACES="                                                                               ";
+	char *txt;
 	while (node != NULL) {
-		printf("%.*s%s", indent, SPACES, webidl_node_type_to_str(node->type));
+                fprintf(dumpf, "%.*s%s", indent, SPACES,
+                        webidl_node_type_to_str(node->type));
 
 		txt = webidl_node_gettext(node);
 		if (txt == NULL) {
@@ -401,20 +411,43 @@ int webidl_ast_dump(struct webidl_node *node, int indent)
 			next = webidl_node_getnode(node);
 
 			if (next != NULL) {
-				printf("\n");
-				webidl_ast_dump(next, indent + 2);
+				fprintf(dumpf, "\n");
+				webidl_ast_dump(dumpf, next, indent + 2);
 			} else {
 				/* not txt or node has to be an int */
-				printf(": %d\n", webidl_node_getint(node));
+				fprintf(dumpf, ": %d\n",
+                                       webidl_node_getint(node));
 			}
 		} else {
-			printf(": \"%s\"\n", txt);
+			fprintf(dumpf, ": \"%s\"\n", txt);
 		}
 		node = node->l;
 	}
 	return 0;
 }
 
+/* exported interface documented in webidl-ast.h */
+int webidl_dump_ast(struct webidl_node *node)
+{
+        FILE *dumpf;
+
+        /* only dump AST to file if required */
+        if (!options->debug) {
+                return 0;
+        }
+
+        dumpf = genb_fopen("webidl-ast", "w");
+        if (dumpf == NULL) {
+                return 2;
+        }
+
+        webidl_ast_dump(dumpf, node, 0);
+
+        fclose(dumpf);
+
+        return 0;
+}
+
 /* exported interface defined in webidl-ast.h */
 static FILE *idlopen(const char *filename)
 {
@@ -444,8 +477,8 @@ static FILE *idlopen(const char *filename)
 /* exported interface defined in webidl-ast.h */
 int webidl_parsefile(char *filename, struct webidl_node **webidl_ast)
 {
-
 	FILE *idlfile;
+        int ret;
 
 	idlfile = idlopen(filename);
 	if (!idlfile) {
@@ -455,16 +488,51 @@ int webidl_parsefile(char *filename, struct webidl_node **webidl_ast)
 		return 2;
 	}
 
-	if (options->debug) {
+        /* if debugging enabled enable parser tracing and send to file */
+        if (options->debug) {
+                char *tracename;
+                int tracenamelen;
 		webidl_debug = 1;
 		webidl__flex_debug = 1;
-	}
+
+                tracenamelen = SLEN("webidl--trace") + strlen(filename) + 1;
+                tracename = malloc(tracenamelen);
+                snprintf(tracename, tracenamelen,"webidl-%s-trace", filename);
+                webidl_parsetracef = genb_fopen(tracename, "w");
+                free(tracename);
+        } else {
+                webidl_parsetracef = NULL;
+        }
 
 	/* set flex to read from file */
 	webidl_restart(idlfile);
 
 	/* parse the file */
-	return webidl_parse(webidl_ast);
+	ret = webidl_parse(webidl_ast);
+
+        /* close tracefile if open */
+        if (webidl_parsetracef != NULL) {
+                fclose(webidl_parsetracef);
+        }
+        return ret;
+}
+
+/* exported interface defined in webidl-ast.h */
+int webidl_fprintf(FILE *stream, const char *format, ...)
+{
+        va_list ap;
+        int ret;
+
+        va_start(ap, format);
+
+        if (webidl_parsetracef == NULL) {
+                ret = vfprintf(stream, format, ap);
+        } else {
+                ret = vfprintf(webidl_parsetracef, format, ap);
+        }
+        va_end(ap);
+
+        return ret;
 }
 
 /* unlink a child node from a parent */
diff --git a/src/webidl-ast.h b/src/webidl-ast.h
index a494f4e..5d0cbc0 100644
--- a/src/webidl-ast.h
+++ b/src/webidl-ast.h
@@ -90,17 +90,25 @@ int webidl_node_getint(struct webidl_node *node);
 enum webidl_node_type webidl_node_gettype(struct webidl_node *node);
 
 /* node searches */
+
+/**
+ * Iterate nodes children matching their type.
+ *
+ * For each child node where the type is matched the callback function is
+ * called with a context value.
+ */
 int webidl_node_for_each_type(struct webidl_node *node,
-			   enum webidl_node_type type,
-			   webidl_callback_t *cb,
+                              enum webidl_node_type type,
+                              webidl_callback_t *cb,
 			      void *ctx);
 
-int webidl_node_enumerate_type(struct webidl_node *node, enum webidl_node_type type);
+int webidl_node_enumerate_type(struct webidl_node *node,
+                               enum webidl_node_type type);
 
 struct webidl_node *
 webidl_node_find(struct webidl_node *node,
-		  struct webidl_node *prev,
-		  webidl_callback_t *cb,
+                 struct webidl_node *prev,
+                 webidl_callback_t *cb,
 		 void *ctx);
 
 struct webidl_node *
@@ -114,13 +122,26 @@ webidl_node_find_type_ident(struct webidl_node *root_node,
 			    const char *ident);
 
 
-/* debug dump */
-int webidl_ast_dump(struct webidl_node *node, int indent);
 
-/** parse web idl file */
+/**
+ * parse web idl file into Abstract Syntax Tree
+ */
 int webidl_parsefile(char *filename, struct webidl_node **webidl_ast);
 
-/** perform replacement of implements elements with copies of ast data */
+/**
+ * dump AST to file
+ */
+int webidl_dump_ast(struct webidl_node *node);
+
+/**
+ * perform replacement of implements elements with copies of ast data
+ */
 int webidl_intercalate_implements(struct webidl_node *node);
 
+/**
+ * formatted printf to allow webidl trace data to be written to file.
+ */
+int webidl_fprintf(FILE *stream, const char *format, ...);
+
+
 #endif
diff --git a/src/webidl-lexer.l b/src/webidl-lexer.l
index 74b9bb8..e49b813 100644
--- a/src/webidl-lexer.l
+++ b/src/webidl-lexer.l
@@ -86,7 +86,7 @@ lineend             ([\n\r]|{LS}|{PS})
 hexdigit            [0-9A-Fa-f]
 hexint              0(x|X){hexdigit}+
 
-decimalint          0|([1-9][0-9]*)
+decimalint          0|([\+\-]?[1-9][0-9]*)
 
 octalint            (0[0-8]+)
 
@@ -207,6 +207,11 @@ void                return TOK_VOID;
 
 readonly            return TOK_READONLY;
 
+Promise             return TOK_PROMISE;
+
+iterable            return TOK_ITERABLE;
+
+legacyiterable      return TOK_LEGACYITERABLE;
 
 {identifier}        {
                         /* A leading "_" is used to escape an identifier from 
diff --git a/src/webidl-parser.y b/src/webidl-parser.y
index 9324212..9717b8c 100644
--- a/src/webidl-parser.y
+++ b/src/webidl-parser.y
@@ -9,6 +9,9 @@
  *
  * Derived from the the grammar in apendix A of W3C WEB IDL
  *   http://www.w3.org/TR/WebIDL/
+ *
+ * WebIDL now has a second edition draft (mid 2015) that the dom and
+ *   html specs are using. https://heycam.github.io/webidl
  */
 
 #include <stdio.h>
@@ -17,11 +20,17 @@
 #include <stdint.h>
 #include <math.h>
 
-#include "webidl-ast.h"
+#define YYFPRINTF webidl_fprintf
+#define YY_LOCATION_PRINT(File, Loc)                            \
+  webidl_fprintf(File, "%d.%d-%d.%d",                           \
+                 (Loc).first_line, (Loc).first_column,          \
+                 (Loc).last_line,  (Loc).last_column)
 
 #include "webidl-parser.h"
 #include "webidl-lexer.h"
 
+#include "webidl-ast.h"
+
 char *errtxt;
 
 static void
@@ -77,6 +86,8 @@ webidl_error(YYLTYPE *locp, struct webidl_node **winbind_ast, const char *str)
 %token TOK_INFINITY
 %token TOK_INHERIT
 %token TOK_INTERFACE
+%token TOK_ITERABLE
+%token TOK_LEGACYITERABLE
 %token TOK_LONG
 %token TOK_MODULE
 %token TOK_NAN
@@ -88,6 +99,7 @@ webidl_error(YYLTYPE *locp, struct webidl_node **winbind_ast, const char *str)
 %token TOK_OPTIONAL
 %token TOK_OR
 %token TOK_PARTIAL
+%token TOK_PROMISE
 %token TOK_RAISES
 %token TOK_READONLY
 %token TOK_SETRAISES
@@ -105,12 +117,12 @@ webidl_error(YYLTYPE *locp, struct webidl_node **winbind_ast, const char *str)
 
 %token TOK_POUND_SIGN
 
-%token <text>       TOK_IDENTIFIER
-%token <value>      TOK_INT_LITERAL
-%token <text>       TOK_FLOAT_LITERAL
-%token <text>       TOK_STRING_LITERAL
-%token <text>       TOK_OTHER_LITERAL
-%token <text>       TOK_JAVADOC
+%token <text>  TOK_IDENTIFIER
+%token <value> TOK_INT_LITERAL
+%token <text>  TOK_FLOAT_LITERAL
+%token <text>  TOK_STRING_LITERAL
+%token <text>  TOK_OTHER_LITERAL
+%token <text>  TOK_JAVADOC
 
 %type <text> Inheritance
 
@@ -153,6 +165,8 @@ webidl_error(YYLTYPE *locp, struct webidl_node **winbind_ast, const char *str)
 %type <text> ArgumentName
 %type <text> ArgumentNameKeyword
 %type <node> Ellipsis
+%type <node> Iterable
+%type <node> OptionalType
 
 %type <node> Type
 %type <node> ReturnType
@@ -165,6 +179,7 @@ webidl_error(YYLTYPE *locp, struct webidl_node **winbind_ast, const char *str)
 %type <node> FloatType
 %type <node> UnsignedIntegerType
 %type <node> IntegerType
+%type <node> PromiseType
 
 %type <node> TypeSuffix
 %type <node> TypeSuffixStartingWithArray
@@ -356,7 +371,7 @@ InterfaceMembers:
 
             if (ident_node == NULL) {
                 /* something with no ident - possibly constructors? */
-                /* @todo understand this abtter */
+                /* @todo understand this better */
 
                 $$ = webidl_node_prepend($1, $3);
 
@@ -389,11 +404,17 @@ InterfaceMembers:
         }
         ;
 
- /* [10] */
+ /* [10]
+  * SE[10]
+  * Second edition actually splits up AttributeOrOperation completely
+  * here we "just" add Iterable as thats what the specs use
+  */
 InterfaceMember:
         Const
         |
         AttributeOrOperation
+        |
+        Iterable
         ;
 
  /* [11] */
@@ -474,18 +495,28 @@ Enum:
         }
         ;
 
-/* [21] */
+ /* Second edition changes enumeration rules to allow trailing comma */
+
+  /* SE[20] */
 EnumValueList:
-        TOK_STRING_LITERAL EnumValues
+        TOK_STRING_LITERAL EnumValueListComma
         ;
 
-/* [22] */
-EnumValues:
+ /* SE[21] */
+EnumValueListComma:
+        ',' EnumValueListString
+        |
         /* empty */
+        ;
+
+ /* SE[22] */
+EnumValueListString:
+        TOK_STRING_LITERAL EnumValueListComma
         |
-        ',' TOK_STRING_LITERAL EnumValues
+        /* empty */
         ;
 
+
  /* [23] - bug in w3c grammar? it doesnt list the equals as a terminal  */
 CallbackRest:
         TOK_IDENTIFIER '=' ReturnType '(' ArgumentList ')' ';'
@@ -835,6 +866,32 @@ Ellipsis:
         }
         ;
 
+ /* SE[59] */
+Iterable:
+        TOK_ITERABLE '<' Type OptionalType '>' ';'
+        {
+            $$ = NULL;
+        }
+        |
+        TOK_LEGACYITERABLE '<' Type '>' ';'
+        {
+            $$ = NULL;
+        }
+        ;
+
+ /* SE[60] */
+OptionalType:
+        /* empty */
+        {
+            $$ = NULL;
+        }
+        |
+        ',' Type
+        {
+            $$ = NULL;
+        }
+        ;
+
  /* [47] */
 ExceptionMember:
         Const
@@ -1301,13 +1358,22 @@ UnionMemberTypes:
         TOK_OR UnionMemberType UnionMemberTypes
         ;
 
- /* [62] */
+ /* [62]
+  * SE[78]
+  * Second edition adds several types
+  */
 NonAnyType:
         PrimitiveType TypeSuffix
         {
             $$ = webidl_node_prepend($1, $2);
         }
         |
+        PromiseType TypeSuffix
+        {
+            /* second edition adds promise types */
+            $$ = webidl_node_prepend($1, $2);
+        }
+        |
         TOK_STRING TypeSuffix
         {
             $$ = webidl_node_new(WEBIDL_NODE_TYPE_TYPE_BASE, $2, (void *)WEBIDL_TYPE_STRING);
@@ -1442,6 +1508,14 @@ OptionalLong:
         }
         ;
 
+/* SE[87] */
+PromiseType:
+        TOK_PROMISE '<' ReturnType '>'
+        {
+            $$ = NULL;
+        }
+        ;
+
  /* [70] */
 TypeSuffix:
         /* empty */