A proposed new interface for skipping characters automatically - implemented for Scanners but not yet for RegexParsers

Author: peteraldous

shared/src/main/scala/scala/util/parsing/combinator/PackratParsers.scala

@@ -104,7 +104,7 @@ trait PackratParsers extends Parsers {
   override def phrase[T](p: Parser[T]): PackratParser[T] = {
     val q = super.phrase(p)
     new PackratParser[T] {
-      def apply(in: Input) = in match {
+      def parse(in: Input) = in match {
         case in: PackratReader[_] => q(in)
         case in => q(new PackratReader(in))
       }
@@ -231,7 +231,7 @@ to update each parser involved in the recursion.
    */
   def memo[T](p: super.Parser[T]): PackratParser[T] = {
     new PackratParser[T] {
-      def apply(in: Input) = {
+      def parse(in: Input) = {
         /*
          * transformed reader
          */

shared/src/main/scala/scala/util/parsing/combinator/Parsers.scala

@@ -13,6 +13,7 @@
 package scala
 package util.parsing.combinator
 
+import scala.util.DynamicVariable
 import scala.util.parsing.input._
 import scala.collection.mutable.ListBuffer
 import scala.annotation.tailrec
@@ -221,7 +222,7 @@ trait Parsers {
   }
 
   def Parser[T](f: Input => ParseResult[T]): Parser[T]
-    = new Parser[T]{ def apply(in: Input) = f(in) }
+    = new Parser[T]{ def parse(in: Input) = f(in) }
 
   private[combinator] def Success[U](res: U, next: Input, failure: Option[Failure]): ParseResult[U] =
     new Success(res, next) { override val lastFailure: Option[Failure] = failure }
@@ -236,8 +237,27 @@ trait Parsers {
       case _ => None
     }
 
+  val skipParser: DynamicVariable[Option[Parser[Any]]] = new DynamicVariable(None);
+
+  final def skip(in: Input): Input = {
+    skipParser.value match {
+      case None => in
+      case Some(parser) => {
+        skipParser.withValue(None) {
+          parser(in) match {
+            case Success(_,next) => {
+              next
+            }
+            // A parser whose purpose is to skip shouldn't fail; it should just not skip stuff
+            case _ => in
+          }
+        }
+      }
+    }
+  }
+
   def OnceParser[T](f: Input => ParseResult[T]): Parser[T] with OnceParser[T]
-    = new Parser[T] with OnceParser[T] { def apply(in: Input) = f(in) }
+    = new Parser[T] with OnceParser[T] { def parse(in: Input) = f(in) }
 
   /** The root class of parsers.
    *  Parsers are functions from the Input type to ParseResult.
@@ -248,7 +268,10 @@ trait Parsers {
     override def toString = s"Parser ($name)"
 
     /** An unspecified method that defines the behaviour of this parser. */
-    def apply(in: Input): ParseResult[T]
+    def parse(in: Input): ParseResult[T]
+    def apply(in: Input): ParseResult[T] = {
+      parse(skip(in))
+    }
 
     def flatMap[U](f: T => Parser[U]): Parser[U]
       = Parser{ in => this(in) flatMapWithNext(f)}
@@ -268,6 +291,20 @@ trait Parsers {
       Parser{ in => this(in) append p(in)}
     }
 
+    /** A parser combinator that changes skipping behavior
+     */
+    def << (toSkip: => Option[Parser[Any]]): Parser[T] = {
+      val originalParse: Input => ParseResult[T] = parse
+      new Parser[T] {
+        override def apply(in: Input): ParseResult[T] = {
+          skipParser.withValue(toSkip) {
+            parse(skip(in))
+          }
+        }
+        def parse(in: Input): ParseResult[T] = originalParse(in)
+      }.named(name)
+    }
+
     // the operator formerly known as +++, ++, &, but now, behold the venerable ~
     // it's short, light (looks like whitespace), has few overloaded meaning (thanks to the recent change from ~ to unary_~)
     // and we love it! (or do we like `,` better?)
@@ -324,7 +361,7 @@ trait Parsers {
 
      /* not really useful: V cannot be inferred because Parser is covariant in first type parameter (V is always trivially Nothing)
     def ~~ [U, V](q: => Parser[U])(implicit combine: (T, U) => V): Parser[V] = new Parser[V] {
-      def apply(in: Input) = seq(Parser.this, q)((x, y) => combine(x,y))(in)
+      def parse(in: Input) = seq(Parser.this, q)((x, y) => combine(x,y))(in)
     }  */
 
     /** A parser combinator for non-back-tracking sequential composition.
@@ -391,7 +428,7 @@ trait Parsers {
      */
     def ||| [U >: T](q0: => Parser[U]): Parser[U] = new Parser[U] {
       lazy val q = q0 // lazy argument
-      def apply(in: Input) = {
+      def parse(in: Input) = {
         val res1 = Parser.this(in)
         val res2 = q(in)
 
@@ -427,7 +464,7 @@ trait Parsers {
      */
     def ^^^ [U](v: => U): Parser[U] =  new Parser[U] {
       lazy val v0 = v // lazy argument
-      def apply(in: Input) = Parser.this(in) map (x => v0)
+      def parse(in: Input) = Parser.this(in) map (x => v0)
     }.named(toString+"^^^")
 
     /** A parser combinator for partial function application.
@@ -769,18 +806,18 @@ trait Parsers {
 
     def continue(in: Input, failure: Option[Failure]): ParseResult[List[T]] = {
       val p0 = p    // avoid repeatedly re-evaluating by-name parser
-      @tailrec def applyp(in0: Input, failure: Option[Failure]): ParseResult[List[T]] = p0(in0) match {
+      @tailrec def parsep(in0: Input, failure: Option[Failure]): ParseResult[List[T]] = p0(in0) match {
         case s @ Success(x, rest) =>
           val selectedFailure = selectLastFailure(s.lastFailure, failure)
           elems += x
-          applyp(rest, selectedFailure)
+          parsep(rest, selectedFailure)
         case e @ Error(_, _)  => e  // still have to propagate error
         case f: Failure =>
           val selectedFailure = selectLastFailure(failure, Some(f))
           Success(elems.toList, in0, selectedFailure)
       }
 
-      applyp(in, failure)
+      parsep(in, failure)
     }
 
     first(in) match {
@@ -804,14 +841,14 @@ trait Parsers {
       val elems = new ListBuffer[T]
       val p0 = p    // avoid repeatedly re-evaluating by-name parser
 
-      @tailrec def applyp(in0: Input, failure: Option[Failure]): ParseResult[List[T]] =
+      @tailrec def parsep(in0: Input, failure: Option[Failure]): ParseResult[List[T]] =
         if (elems.length == num) Success(elems.toList, in0, failure)
         else p0(in0) match {
-          case s @ Success(x, rest) => elems += x ; applyp(rest, s.lastFailure)
+          case s @ Success(x, rest) => elems += x ; parsep(rest, s.lastFailure)
           case ns: NoSuccess    => ns
         }
 
-      applyp(in, None)
+      parsep(in, None)
     }
 
   /** A parser generator for a specified range of repetitions interleaved by a
@@ -835,13 +872,13 @@ trait Parsers {
 
     def continue(in: Input): ParseResult[List[T]] = {
       val p0 = sep ~> p // avoid repeatedly re-evaluating by-name parser
-      @tailrec def applyp(in0: Input): ParseResult[List[T]] = p0(in0) match {
-        case Success(x, rest) => elems += x; if (elems.length == m) Success(elems.toList, rest, None) else applyp(rest)
+      @tailrec def parsep(in0: Input): ParseResult[List[T]] = p0(in0) match {
+        case Success(x, rest) => elems += x; if (elems.length == m) Success(elems.toList, rest, None) else parsep(rest)
         case e @ Error(_, _) => e // still have to propagate error
         case _ => Success(elems.toList, in0, None)
       }
 
-      applyp(in)
+      parsep(in)
     }
 
     mandatory(in) match {
@@ -973,7 +1010,7 @@ trait Parsers {
    *           if `p` consumed all the input.
    */
   def phrase[T](p: Parser[T]) = new Parser[T] {
-    def apply(in: Input) = p(in) match {
+    def parse(in: Input) = p(in) match {
       case s @ Success(out, in1) =>
         if (in1.atEnd) s
         else s.lastFailure match {

shared/src/main/scala/scala/util/parsing/combinator/RegexParsers.scala

@@ -48,7 +48,7 @@ import scala.language.implicitConversions
  *      }
  *    }
  *
- *    def apply(input: String): Double = parseAll(expr, input) match {
+ *    def parse(input: String): Double = parseAll(expr, input) match {
  *      case Success(result, _) => result
  *      case failure : NoSuccess => scala.sys.error(failure.msg)
  *    }
@@ -83,7 +83,7 @@ trait RegexParsers extends Parsers {
 
   /** A parser that matches a literal string */
   implicit def literal(s: String): Parser[String] = new Parser[String] {
-    def apply(in: Input) = {
+    def parse(in: Input) = {
       val source = in.source
       val offset = in.offset
       val start = handleWhiteSpace(source, offset)
@@ -104,7 +104,7 @@ trait RegexParsers extends Parsers {
 
   /** A parser that matches a regex string */
   implicit def regex(r: Regex): Parser[String] = new Parser[String] {
-    def apply(in: Input) = {
+    def parse(in: Input) = {
       val source = in.source
       val offset = in.offset
       val start = handleWhiteSpace(source, offset)
@@ -131,7 +131,7 @@ trait RegexParsers extends Parsers {
   override def positioned[T <: Positional](p: => Parser[T]): Parser[T] = {
     val pp = super.positioned(p)
     new Parser[T] {
-      def apply(in: Input) = {
+      def parse(in: Input) = {
         val offset = in.offset
         val start = handleWhiteSpace(in.source, offset)
         pp(in.drop (start - offset))
@@ -141,7 +141,7 @@ trait RegexParsers extends Parsers {
 
   // we might want to make it public/protected in a future version
   private def ws[T](p: Parser[T]): Parser[T] = new Parser[T] {
-    def apply(in: Input) = {
+    def parse(in: Input) = {
       val offset = in.offset
       val start = handleWhiteSpace(in.source, offset)
       p(in.drop (start - offset))

shared/src/main/scala/scala/util/parsing/combinator/lexical/Scanners.scala

@@ -32,9 +32,6 @@ trait Scanners extends Parsers {
   /** A parser that produces a token (from a stream of characters). */
   def token: Parser[Token]
 
-  /** A parser for white-space -- its result will be discarded. */
-  def whitespace: Parser[Any]
-
   /** `Scanner` is essentially¹ a parser that produces `Token`s
    *  from a stream of characters. The tokens it produces are typically
    *  passed to parsers in `TokenParsers`.
@@ -44,21 +41,18 @@ trait Scanners extends Parsers {
   class Scanner(in: Reader[Char]) extends Reader[Token] {
     /** Convenience constructor (makes a character reader out of the given string) */
     def this(in: String) = this(new CharArrayReader(in.toCharArray))
-    private val (tok, rest1, rest2) = whitespace(in) match {
-      case Success(_, in1) =>
-        token(in1) match {
-          case Success(tok, in2) => (tok, in1, in2)
-          case ns: NoSuccess => (errorToken(ns.msg), ns.next, skip(ns.next))
-        }
-      case ns: NoSuccess => (errorToken(ns.msg), ns.next, skip(ns.next))
+    private val in1 = skip(in)
+    private val (tok, rest1, rest2) = token(in1) match {
+      case Success(tok, in2) => (tok, in1, in2)
+      case ns: NoSuccess => (errorToken(ns.msg), ns.next, skipChar(ns.next))
     }
-    private def skip(in: Reader[Char]) = if (in.atEnd) in else in.rest
+    private def skipChar(in: Reader[Char]) = if (in.atEnd) in else in.rest
 
     override def source: java.lang.CharSequence = in.source
     override def offset: Int = in.offset
     def first = tok
     def rest = new Scanner(rest2)
     def pos = rest1.pos
-    def atEnd = in.atEnd || (whitespace(in) match { case Success(_, in1) => in1.atEnd case _ => false })
+    def atEnd = in.atEnd || skip(in).atEnd
   }
 }