Fixed an off-by-one error in the lexer when encountering EOF

Fixes mozilla#2151

Author: andrea.bergia

rhino/src/main/java/org/mozilla/javascript/TokenStream.java

@@ -2261,8 +2261,18 @@ private void skipLine() throws IOException {
         // skip to end of line
         int c;
         while ((c = getChar()) != EOF_CHAR && c != '\n') {}
-        ungetChar(c);
-        tokenEnd = cursor;
+        if (c == EOF_CHAR) {
+            // If we've hit EOF, the cursor hasn't been incremented, so we need to save tokenEnd
+            // _before_ ungetChar
+            tokenEnd = cursor;
+            ungetChar(c);
+        } else {
+            // If we've hit a newline, the cursor has been incremented past it, and ungetChar will
+            // point at the newline, so saving tokenEnd after ungetChar will correctly exclude the
+            // newline
+            ungetChar(c);
+            tokenEnd = cursor;
+        }
     }
 
     /** Returns the offset into the current line. */

tests/src/test/java/org/mozilla/javascript/tests/ParserTest.java

@@ -1498,6 +1498,18 @@ public void errorOnInvalidDestructuringDeclaration() {
                 new String[] {"Missing = in destructuring declaration", "syntax error"});
     }
 
+    @Test
+    public void commentValueIsNotTruncatedBecauseOfEof() {
+        String source = "// comment";
+
+        AstRoot root = parse(source);
+        assertNotNull(root.getComments());
+        assertEquals(1, root.getComments().size());
+        Comment comment = root.getComments().first();
+        assertEquals(source, comment.getValue());
+        assertEquals(10, comment.getLength());
+    }
+
     private void expectParseErrors(String string, String[] errors) {
         parse(string, errors, null, false);
     }