Use pump to generate source files.

native_mate/constructor.h

@@ -1,3 +1,7 @@
+// This file was GENERATED by command:
+//     pump.py constructor.h.pump
+// DO NOT EDIT BY HAND!!!
+
 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE.chromium file.
@@ -30,7 +34,7 @@ inline Wrappable* InvokeFactory(
     Arguments* args,
     const base::Callback<Wrappable*(P1)>& callback) {
   typename CallbackParamTraits<P1>::LocalType a1;
-  if (!GetNextArgument(args, 0, false, &a1))
+  if (!GetNextArgument(args, 0, true, &a1))
     return NULL;
   return callback.Run(a1);
 };

native_mate/constructor.h.pump

@@ -0,0 +1,124 @@
+$$ This is a pump file for generating file templates.  Pump is a python
+$$ script that is part of the Google Test suite of utilities.  Description
+$$ can be found here:
+$$
+$$ http://code.google.com/p/googletest/wiki/PumpManual
+$$
+$var MAX_ARITY = 6
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE.chromium file.
+
+#ifndef NATIVE_MATE_WRAPPABLE_CLASS_H_
+#define NATIVE_MATE_WRAPPABLE_CLASS_H_
+
+#include "base/bind.h"
+#include "base/compiler_specific.h"
+#include "native_mate/wrappable.h"
+#include "native_mate/function_template.h"
+
+namespace mate {
+
+class Wrappable;
+
+namespace internal {
+
+// This set of templates invokes a base::Callback by converting the Arguments
+// into native types. It relies on the function_template.h to provide helper
+// templates.
+$range ARITY 0..MAX_ARITY
+$for ARITY [[
+$range ARG 1..ARITY
+
+$if ARITY == 0 [[
+
+]] $else [[
+
+template<$for ARG , [[typename P$(ARG)]]>
+]]
+
+inline Wrappable* InvokeFactory(
+    Arguments* args,
+    const base::Callback<Wrappable*($for ARG , [[P$(ARG)]])>& callback) {
+$if ARITY != 0 [[
+
+
+$for ARG [[  typename CallbackParamTraits<P$(ARG)>::LocalType a$(ARG);
+
+]]
+  if ($for ARG  ||
+      [[!GetNextArgument(args, 0, $if ARG == 1 [[true]] $else [[false]], &a$(ARG))]])
+    return NULL;
+]]
+
+  return callback.Run($for ARG , [[a$(ARG)]]);
+};
+
+]]
+
+}  // namespace internal
+
+
+template<typename Sig>
+class Constructor {
+ public:
+  typedef base::Callback<Sig> WrappableFactoryFunction;
+
+  Constructor(const base::StringPiece& name) : name_(name) {}
+  virtual ~Constructor() {
+    MATE_PERSISTENT_RESET(constructor_);
+  }
+
+  v8::Handle<v8::FunctionTemplate> GetFunctionTemplate(
+      v8::Isolate* isolate, const WrappableFactoryFunction& factory) {
+    if (constructor_.IsEmpty()) {
+      v8::Local<v8::FunctionTemplate> constructor = CreateFunctionTemplate(
+          isolate, base::Bind(&Constructor::New, factory));
+      constructor->InstanceTemplate()->SetInternalFieldCount(1);
+      constructor->SetClassName(StringToV8(isolate, name_));
+      MATE_PERSISTENT_ASSIGN(v8::FunctionTemplate, isolate, constructor_,
+                             constructor);
+    }
+
+    return MATE_PERSISTENT_TO_LOCAL(
+        v8::FunctionTemplate, isolate, constructor_);
+  }
+
+ private:
+  static MATE_METHOD_RETURN_TYPE New(const WrappableFactoryFunction& factory,
+                                     v8::Isolate* isolate, Arguments* args) {
+    Wrappable* object = internal::InvokeFactory(args, factory);
+    if (object)
+      object->Wrap(isolate, args->GetThis());
+    else
+      args->ThrowError();
+
+    MATE_METHOD_RETURN_UNDEFINED();
+  }
+
+  base::StringPiece name_;
+  v8::Persistent<v8::FunctionTemplate> constructor_;
+
+  DISALLOW_COPY_AND_ASSIGN(Constructor);
+};
+
+
+template<typename T>
+Wrappable* NewOperatorFactory() {
+  return new T;
+}
+
+template<typename T, typename Sig>
+v8::Local<v8::Function> CreateConstructor(
+    v8::Isolate* isolate,
+    const base::StringPiece& name,
+    const base::Callback<Sig>& callback) {
+  v8::Local<v8::FunctionTemplate> constructor =
+      Constructor<Sig>(name).GetFunctionTemplate(isolate, callback);
+  T::BuildPrototype(isolate, constructor->PrototypeTemplate());
+  return constructor->GetFunction();
+}
+
+}  // namespace mate
+
+#endif  // NATIVE_MATE_WRAPPABLE_CLASS_H_

native_mate/function_template.h

@@ -1,3 +1,7 @@
+// This file was GENERATED by command:
+//     pump.py function_template.h.pump
+// DO NOT EDIT BY HAND!!!
+
 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE.chromium file.
@@ -116,6 +120,21 @@ struct Invoker<void, P1, P2, P3, P4, P5, P6, P7> {
   }
 };
 
+template<typename R, typename P1, typename P2, typename P3, typename P4,
+    typename P5, typename P6>
+struct Invoker<R, P1, P2, P3, P4, P5, P6, void> {
+  inline static MATE_METHOD_RETURN_TYPE Go(
+      Arguments& args,
+      const base::Callback<R(P1, P2, P3, P4, P5, P6)>& callback,
+      const P1& a1,
+      const P2& a2,
+      const P3& a3,
+      const P4& a4,
+      const P5& a5,
+      const P6& a6) {
+    MATE_METHOD_RETURN(callback.Run(a1, a2, a3, a4, a5, a6));
+  }
+};
 template<typename P1, typename P2, typename P3, typename P4, typename P5,
     typename P6>
 struct Invoker<void, P1, P2, P3, P4, P5, P6, void> {
@@ -529,7 +548,6 @@ struct Dispatcher<R(P1, P2, P3, P4, P5, P6, P7)> {
   }
 };
 
-
 }  // namespace internal
 
 

native_mate/function_template.h.pump

@@ -0,0 +1,225 @@
+$$ This is a pump file for generating file templates.  Pump is a python
+$$ script that is part of the Google Test suite of utilities.  Description
+$$ can be found here:
+$$
+$$ http://code.google.com/p/googletest/wiki/PumpManual
+$$
+$var MAX_ARITY = 7
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE.chromium file.
+
+#ifndef NATIVE_MATE_FUNCTION_TEMPLATE_H_
+#define NATIVE_MATE_FUNCTION_TEMPLATE_H_
+
+#include "base/callback.h"
+#include "base/logging.h"
+#include "native_mate/arguments.h"
+#include "native_mate/compat.h"
+#include "native_mate/converter.h"
+#include "v8/include/v8.h"
+
+namespace mate {
+
+class PerIsolateData;
+
+enum CreateFunctionTemplateFlags {
+  HolderIsFirstArgument = 1 << 0,
+};
+
+namespace internal {
+
+template<typename T>
+struct CallbackParamTraits {
+  typedef T LocalType;
+};
+template<typename T>
+struct CallbackParamTraits<const T&> {
+  typedef T LocalType;
+};
+template<typename T>
+struct CallbackParamTraits<const T*> {
+  typedef T* LocalType;
+};
+
+
+// CallbackHolder and CallbackHolderBase are used to pass a base::Callback from
+// CreateFunctionTemplate through v8 (via v8::FunctionTemplate) to
+// DispatchToCallback, where it is invoked.
+
+// This simple base class is used so that we can share a single object template
+// among every CallbackHolder instance.
+class CallbackHolderBase {
+ public:
+  v8::Handle<v8::External> GetHandle(v8::Isolate* isolate);
+
+ protected:
+  explicit CallbackHolderBase(v8::Isolate* isolate);
+  virtual ~CallbackHolderBase();
+
+ private:
+  static MATE_WEAK_CALLBACK(WeakCallback, v8::External, CallbackHolderBase);
+
+  v8::Persistent<v8::External> v8_ref_;
+
+  DISALLOW_COPY_AND_ASSIGN(CallbackHolderBase);
+};
+
+template<typename Sig>
+class CallbackHolder : public CallbackHolderBase {
+ public:
+  CallbackHolder(v8::Isolate* isolate,
+                 const base::Callback<Sig>& callback,
+                 int flags)
+      : CallbackHolderBase(isolate), callback(callback), flags(flags) {}
+  base::Callback<Sig> callback;
+  int flags;
+ private:
+  virtual ~CallbackHolder() {}
+
+  DISALLOW_COPY_AND_ASSIGN(CallbackHolder);
+};
+
+
+// This set of templates invokes a base::Callback, converts the return type to a
+// JavaScript value, and returns that value to script via the provided
+// mate::Arguments object.
+//
+// In C++, you can declare the function foo(void), but you can't pass a void
+// expression to foo. As a result, we must specialize the case of Callbacks that
+// have the void return type.
+
+$range ARITY 0..MAX_ARITY
+$for ARITY [[
+$var INV_ARITY = MAX_ARITY - ARITY
+$range ARG 1..INV_ARITY
+$range VOID INV_ARITY+1..MAX_ARITY
+
+$if ARITY == 0 [[
+template<typename R$for ARG [[, typename P$(ARG) = void]]>
+struct Invoker {
+]] $else [[
+template<typename R$for ARG [[, typename P$(ARG)]]>
+struct Invoker<R$for ARG [[, P$(ARG)]]$for VOID [[, void]]> {
+]]
+
+  inline static MATE_METHOD_RETURN_TYPE Go(
+      Arguments& args,
+      const base::Callback<R($for ARG , [[P$(ARG)]])>& callback$for ARG [[,
+      const P$(ARG)& a$(ARG)]]) {
+    MATE_METHOD_RETURN(callback.Run($for ARG, [[a$(ARG)]]));
+  }
+};
+template<$for ARG , [[typename P$(ARG)]]>
+struct Invoker<void$for ARG [[, P$(ARG)]]$for VOID [[, void]]> {
+  inline static MATE_METHOD_RETURN_TYPE Go(
+      Arguments& args,
+      const base::Callback<void($for ARG , [[P$(ARG)]])>& callback$for ARG [[,
+      const P$(ARG)& a$(ARG)]]) {
+    callback.Run($for ARG, [[a$(ARG)]]);
+    MATE_METHOD_RETURN_UNDEFINED();
+  }
+};
+
+
+]]
+
+template<typename T>
+bool GetNextArgument(Arguments* args, int create_flags, bool is_first,
+                     T* result) {
+  if (is_first && (create_flags & HolderIsFirstArgument) != 0) {
+    return args->GetHolder(result);
+  } else {
+    return args->GetNext(result);
+  }
+}
+
+// For advanced use cases, we allow callers to request the unparsed Arguments
+// object and poke around in it directly.
+inline bool GetNextArgument(Arguments* args, int create_flags, bool is_first,
+                            Arguments* result) {
+  *result = *args;
+  return true;
+}
+inline bool GetNextArgument(Arguments* args, int create_flags, bool is_first,
+                            Arguments** result) {
+  *result = args;
+  return true;
+}
+
+// It's common for clients to just need the isolate, so we make that easy.
+inline bool GetNextArgument(Arguments* args, int create_flags,
+                            bool is_first, v8::Isolate** result) {
+  *result = args->isolate();
+  return true;
+}
+
+
+// DispatchToCallback converts all the JavaScript arguments to C++ types and
+// invokes the base::Callback.
+template<typename Sig>
+struct Dispatcher {
+};
+
+$range ARITY 0..MAX_ARITY
+$for ARITY [[
+$range ARG 1..ARITY
+
+template<typename R$for ARG [[, typename P$(ARG)]]>
+struct Dispatcher<R($for ARG , [[P$(ARG)]])> {
+  static MATE_METHOD(DispatchToCallback) {
+    Arguments args(info);
+    v8::Handle<v8::External> v8_holder;
+    CHECK(args.GetData(&v8_holder));
+    CallbackHolderBase* holder_base = reinterpret_cast<CallbackHolderBase*>(
+        v8_holder->Value());
+
+    typedef CallbackHolder<R($for ARG , [[P$(ARG)]])> HolderT;
+    HolderT* holder = static_cast<HolderT*>(holder_base);
+
+$if ARITY != 0 [[
+
+
+$for ARG [[    typename CallbackParamTraits<P$(ARG)>::LocalType a$(ARG);
+
+]]
+    if ($for ARG  ||
+        [[!GetNextArgument(&args, holder->flags, $if ARG == 1 [[true]] $else [[false]], &a$(ARG))]]) {
+      args.ThrowError();
+      MATE_METHOD_RETURN_UNDEFINED();
+    }
+
+]]
+
+    return Invoker<R$for ARG [[, P$(ARG)]]>::Go(args, holder->callback$for ARG [[, a$(ARG)]]);
+  }
+};
+
+]]
+
+}  // namespace internal
+
+
+// CreateFunctionTemplate creates a v8::FunctionTemplate that will create
+// JavaScript functions that execute a provided C++ function or base::Callback.
+// JavaScript arguments are automatically converted via mate::Converter, as is
+// the return value of the C++ function, if any.
+template<typename Sig>
+v8::Local<v8::FunctionTemplate> CreateFunctionTemplate(
+    v8::Isolate* isolate, const base::Callback<Sig> callback,
+    int callback_flags = 0) {
+  typedef internal::CallbackHolder<Sig> HolderT;
+  HolderT* holder = new HolderT(isolate, callback, callback_flags);
+
+  return v8::FunctionTemplate::New(
+#if NODE_VERSION_AT_LEAST(0, 11, 11)
+      isolate,
+#endif
+      &internal::Dispatcher<Sig>::DispatchToCallback,
+      ConvertToV8<v8::Handle<v8::External> >(isolate,
+                                             holder->GetHandle(isolate)));
+}
+
+}  // namespace mate
+
+#endif  // NATIVE_MATE_FUNCTION_TEMPLATE_H_

script/pump.py

@@ -0,0 +1,855 @@
+#!/usr/bin/env python
+#
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""pump v0.2.0 - Pretty Useful for Meta Programming.
+
+A tool for preprocessor meta programming.  Useful for generating
+repetitive boilerplate code.  Especially useful for writing C++
+classes, functions, macros, and templates that need to work with
+various number of arguments.
+
+USAGE:
+       pump.py SOURCE_FILE
+
+EXAMPLES:
+       pump.py foo.cc.pump
+         Converts foo.cc.pump to foo.cc.
+
+GRAMMAR:
+       CODE ::= ATOMIC_CODE*
+       ATOMIC_CODE ::= $var ID = EXPRESSION
+           | $var ID = [[ CODE ]]
+           | $range ID EXPRESSION..EXPRESSION
+           | $for ID SEPARATOR [[ CODE ]]
+           | $($)
+           | $ID
+           | $(EXPRESSION)
+           | $if EXPRESSION [[ CODE ]] ELSE_BRANCH
+           | [[ CODE ]]
+           | RAW_CODE
+       SEPARATOR ::= RAW_CODE | EMPTY
+       ELSE_BRANCH ::= $else [[ CODE ]]
+           | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH
+           | EMPTY
+       EXPRESSION has Python syntax.
+"""
+
+__author__ = 'wan@google.com (Zhanyong Wan)'
+
+import os
+import re
+import sys
+
+
+TOKEN_TABLE = [
+    (re.compile(r'\$var\s+'), '$var'),
+    (re.compile(r'\$elif\s+'), '$elif'),
+    (re.compile(r'\$else\s+'), '$else'),
+    (re.compile(r'\$for\s+'), '$for'),
+    (re.compile(r'\$if\s+'), '$if'),
+    (re.compile(r'\$range\s+'), '$range'),
+    (re.compile(r'\$[_A-Za-z]\w*'), '$id'),
+    (re.compile(r'\$\(\$\)'), '$($)'),
+    (re.compile(r'\$'), '$'),
+    (re.compile(r'\[\[\n?'), '[['),
+    (re.compile(r'\]\]\n?'), ']]'),
+    ]
+
+
+class Cursor:
+  """Represents a position (line and column) in a text file."""
+
+  def __init__(self, line=-1, column=-1):
+    self.line = line
+    self.column = column
+
+  def __eq__(self, rhs):
+    return self.line == rhs.line and self.column == rhs.column
+
+  def __ne__(self, rhs):
+    return not self == rhs
+
+  def __lt__(self, rhs):
+    return self.line < rhs.line or (
+        self.line == rhs.line and self.column < rhs.column)
+
+  def __le__(self, rhs):
+    return self < rhs or self == rhs
+
+  def __gt__(self, rhs):
+    return rhs < self
+
+  def __ge__(self, rhs):
+    return rhs <= self
+
+  def __str__(self):
+    if self == Eof():
+      return 'EOF'
+    else:
+      return '%s(%s)' % (self.line + 1, self.column)
+
+  def __add__(self, offset):
+    return Cursor(self.line, self.column + offset)
+
+  def __sub__(self, offset):
+    return Cursor(self.line, self.column - offset)
+
+  def Clone(self):
+    """Returns a copy of self."""
+
+    return Cursor(self.line, self.column)
+
+
+# Special cursor to indicate the end-of-file.
+def Eof():
+  """Returns the special cursor to denote the end-of-file."""
+  return Cursor(-1, -1)
+
+
+class Token:
+  """Represents a token in a Pump source file."""
+
+  def __init__(self, start=None, end=None, value=None, token_type=None):
+    if start is None:
+      self.start = Eof()
+    else:
+      self.start = start
+    if end is None:
+      self.end = Eof()
+    else:
+      self.end = end
+    self.value = value
+    self.token_type = token_type
+
+  def __str__(self):
+    return 'Token @%s: \'%s\' type=%s' % (
+        self.start, self.value, self.token_type)
+
+  def Clone(self):
+    """Returns a copy of self."""
+
+    return Token(self.start.Clone(), self.end.Clone(), self.value,
+                 self.token_type)
+
+
+def StartsWith(lines, pos, string):
+  """Returns True iff the given position in lines starts with 'string'."""
+
+  return lines[pos.line][pos.column:].startswith(string)
+
+
+def FindFirstInLine(line, token_table):
+  best_match_start = -1
+  for (regex, token_type) in token_table:
+    m = regex.search(line)
+    if m:
+      # We found regex in lines
+      if best_match_start < 0 or m.start() < best_match_start:
+        best_match_start = m.start()
+        best_match_length = m.end() - m.start()
+        best_match_token_type = token_type
+
+  if best_match_start < 0:
+    return None
+
+  return (best_match_start, best_match_length, best_match_token_type)
+
+
+def FindFirst(lines, token_table, cursor):
+  """Finds the first occurrence of any string in strings in lines."""
+
+  start = cursor.Clone()
+  cur_line_number = cursor.line
+  for line in lines[start.line:]:
+    if cur_line_number == start.line:
+      line = line[start.column:]
+    m = FindFirstInLine(line, token_table)
+    if m:
+      # We found a regex in line.
+      (start_column, length, token_type) = m
+      if cur_line_number == start.line:
+        start_column += start.column
+      found_start = Cursor(cur_line_number, start_column)
+      found_end = found_start + length
+      return MakeToken(lines, found_start, found_end, token_type)
+    cur_line_number += 1
+  # We failed to find str in lines
+  return None
+
+
+def SubString(lines, start, end):
+  """Returns a substring in lines."""
+
+  if end == Eof():
+    end = Cursor(len(lines) - 1, len(lines[-1]))
+
+  if start >= end:
+    return ''
+
+  if start.line == end.line:
+    return lines[start.line][start.column:end.column]
+
+  result_lines = ([lines[start.line][start.column:]] +
+                  lines[start.line + 1:end.line] +
+                  [lines[end.line][:end.column]])
+  return ''.join(result_lines)
+
+
+def StripMetaComments(str):
+  """Strip meta comments from each line in the given string."""
+
+  # First, completely remove lines containing nothing but a meta
+  # comment, including the trailing \n.
+  str = re.sub(r'^\s*\$\$.*\n', '', str)
+
+  # Then, remove meta comments from contentful lines.
+  return re.sub(r'\s*\$\$.*', '', str)
+
+
+def MakeToken(lines, start, end, token_type):
+  """Creates a new instance of Token."""
+
+  return Token(start, end, SubString(lines, start, end), token_type)
+
+
+def ParseToken(lines, pos, regex, token_type):
+  line = lines[pos.line][pos.column:]
+  m = regex.search(line)
+  if m and not m.start():
+    return MakeToken(lines, pos, pos + m.end(), token_type)
+  else:
+    print 'ERROR: %s expected at %s.' % (token_type, pos)
+    sys.exit(1)
+
+
+ID_REGEX = re.compile(r'[_A-Za-z]\w*')
+EQ_REGEX = re.compile(r'=')
+REST_OF_LINE_REGEX = re.compile(r'.*?(?=$|\$\$)')
+OPTIONAL_WHITE_SPACES_REGEX = re.compile(r'\s*')
+WHITE_SPACE_REGEX = re.compile(r'\s')
+DOT_DOT_REGEX = re.compile(r'\.\.')
+
+
+def Skip(lines, pos, regex):
+  line = lines[pos.line][pos.column:]
+  m = re.search(regex, line)
+  if m and not m.start():
+    return pos + m.end()
+  else:
+    return pos
+
+
+def SkipUntil(lines, pos, regex, token_type):
+  line = lines[pos.line][pos.column:]
+  m = re.search(regex, line)
+  if m:
+    return pos + m.start()
+  else:
+    print ('ERROR: %s expected on line %s after column %s.' %
+           (token_type, pos.line + 1, pos.column))
+    sys.exit(1)
+
+
+def ParseExpTokenInParens(lines, pos):
+  def ParseInParens(pos):
+    pos = Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX)
+    pos = Skip(lines, pos, r'\(')
+    pos = Parse(pos)
+    pos = Skip(lines, pos, r'\)')
+    return pos
+
+  def Parse(pos):
+    pos = SkipUntil(lines, pos, r'\(|\)', ')')
+    if SubString(lines, pos, pos + 1) == '(':
+      pos = Parse(pos + 1)
+      pos = Skip(lines, pos, r'\)')
+      return Parse(pos)
+    else:
+      return pos
+
+  start = pos.Clone()
+  pos = ParseInParens(pos)
+  return MakeToken(lines, start, pos, 'exp')
+
+
+def RStripNewLineFromToken(token):
+  if token.value.endswith('\n'):
+    return Token(token.start, token.end, token.value[:-1], token.token_type)
+  else:
+    return token
+
+
+def TokenizeLines(lines, pos):
+  while True:
+    found = FindFirst(lines, TOKEN_TABLE, pos)
+    if not found:
+      yield MakeToken(lines, pos, Eof(), 'code')
+      return
+
+    if found.start == pos:
+      prev_token = None
+      prev_token_rstripped = None
+    else:
+      prev_token = MakeToken(lines, pos, found.start, 'code')
+      prev_token_rstripped = RStripNewLineFromToken(prev_token)
+
+    if found.token_type == '$var':
+      if prev_token_rstripped:
+        yield prev_token_rstripped
+      yield found
+      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
+      yield id_token
+      pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
+
+      eq_token = ParseToken(lines, pos, EQ_REGEX, '=')
+      yield eq_token
+      pos = Skip(lines, eq_token.end, r'\s*')
+
+      if SubString(lines, pos, pos + 2) != '[[':
+        exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp')
+        yield exp_token
+        pos = Cursor(exp_token.end.line + 1, 0)
+    elif found.token_type == '$for':
+      if prev_token_rstripped:
+        yield prev_token_rstripped
+      yield found
+      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
+      yield id_token
+      pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX)
+    elif found.token_type == '$range':
+      if prev_token_rstripped:
+        yield prev_token_rstripped
+      yield found
+      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
+      yield id_token
+      pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
+
+      dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..')
+      yield MakeToken(lines, pos, dots_pos, 'exp')
+      yield MakeToken(lines, dots_pos, dots_pos + 2, '..')
+      pos = dots_pos + 2
+      new_pos = Cursor(pos.line + 1, 0)
+      yield MakeToken(lines, pos, new_pos, 'exp')
+      pos = new_pos
+    elif found.token_type == '$':
+      if prev_token:
+        yield prev_token
+      yield found
+      exp_token = ParseExpTokenInParens(lines, found.end)
+      yield exp_token
+      pos = exp_token.end
+    elif (found.token_type == ']]' or found.token_type == '$if' or
+          found.token_type == '$elif' or found.token_type == '$else'):
+      if prev_token_rstripped:
+        yield prev_token_rstripped
+      yield found
+      pos = found.end
+    else:
+      if prev_token:
+        yield prev_token
+      yield found
+      pos = found.end
+
+
+def Tokenize(s):
+  """A generator that yields the tokens in the given string."""
+  if s != '':
+    lines = s.splitlines(True)
+    for token in TokenizeLines(lines, Cursor(0, 0)):
+      yield token
+
+
+class CodeNode:
+  def __init__(self, atomic_code_list=None):
+    self.atomic_code = atomic_code_list
+
+
+class VarNode:
+  def __init__(self, identifier=None, atomic_code=None):
+    self.identifier = identifier
+    self.atomic_code = atomic_code
+
+
+class RangeNode:
+  def __init__(self, identifier=None, exp1=None, exp2=None):
+    self.identifier = identifier
+    self.exp1 = exp1
+    self.exp2 = exp2
+
+
+class ForNode:
+  def __init__(self, identifier=None, sep=None, code=None):
+    self.identifier = identifier
+    self.sep = sep
+    self.code = code
+
+
+class ElseNode:
+  def __init__(self, else_branch=None):
+    self.else_branch = else_branch
+
+
+class IfNode:
+  def __init__(self, exp=None, then_branch=None, else_branch=None):
+    self.exp = exp
+    self.then_branch = then_branch
+    self.else_branch = else_branch
+
+
+class RawCodeNode:
+  def __init__(self, token=None):
+    self.raw_code = token
+
+
+class LiteralDollarNode:
+  def __init__(self, token):
+    self.token = token
+
+
+class ExpNode:
+  def __init__(self, token, python_exp):
+    self.token = token
+    self.python_exp = python_exp
+
+
+def PopFront(a_list):
+  head = a_list[0]
+  a_list[:1] = []
+  return head
+
+
+def PushFront(a_list, elem):
+  a_list[:0] = [elem]
+
+
+def PopToken(a_list, token_type=None):
+  token = PopFront(a_list)
+  if token_type is not None and token.token_type != token_type:
+    print 'ERROR: %s expected at %s' % (token_type, token.start)
+    print 'ERROR: %s found instead' % (token,)
+    sys.exit(1)
+
+  return token
+
+
+def PeekToken(a_list):
+  if not a_list:
+    return None
+
+  return a_list[0]
+
+
+def ParseExpNode(token):
+  python_exp = re.sub(r'([_A-Za-z]\w*)', r'self.GetValue("\1")', token.value)
+  return ExpNode(token, python_exp)
+
+
+def ParseElseNode(tokens):
+  def Pop(token_type=None):
+    return PopToken(tokens, token_type)
+
+  next = PeekToken(tokens)
+  if not next:
+    return None
+  if next.token_type == '$else':
+    Pop('$else')
+    Pop('[[')
+    code_node = ParseCodeNode(tokens)
+    Pop(']]')
+    return code_node
+  elif next.token_type == '$elif':
+    Pop('$elif')
+    exp = Pop('code')
+    Pop('[[')
+    code_node = ParseCodeNode(tokens)
+    Pop(']]')
+    inner_else_node = ParseElseNode(tokens)
+    return CodeNode([IfNode(ParseExpNode(exp), code_node, inner_else_node)])
+  elif not next.value.strip():
+    Pop('code')
+    return ParseElseNode(tokens)
+  else:
+    return None
+
+
+def ParseAtomicCodeNode(tokens):
+  def Pop(token_type=None):
+    return PopToken(tokens, token_type)
+
+  head = PopFront(tokens)
+  t = head.token_type
+  if t == 'code':
+    return RawCodeNode(head)
+  elif t == '$var':
+    id_token = Pop('id')
+    Pop('=')
+    next = PeekToken(tokens)
+    if next.token_type == 'exp':
+      exp_token = Pop()
+      return VarNode(id_token, ParseExpNode(exp_token))
+    Pop('[[')
+    code_node = ParseCodeNode(tokens)
+    Pop(']]')
+    return VarNode(id_token, code_node)
+  elif t == '$for':
+    id_token = Pop('id')
+    next_token = PeekToken(tokens)
+    if next_token.token_type == 'code':
+      sep_token = next_token
+      Pop('code')
+    else:
+      sep_token = None
+    Pop('[[')
+    code_node = ParseCodeNode(tokens)
+    Pop(']]')
+    return ForNode(id_token, sep_token, code_node)
+  elif t == '$if':
+    exp_token = Pop('code')
+    Pop('[[')
+    code_node = ParseCodeNode(tokens)
+    Pop(']]')
+    else_node = ParseElseNode(tokens)
+    return IfNode(ParseExpNode(exp_token), code_node, else_node)
+  elif t == '$range':
+    id_token = Pop('id')
+    exp1_token = Pop('exp')
+    Pop('..')
+    exp2_token = Pop('exp')
+    return RangeNode(id_token, ParseExpNode(exp1_token),
+                     ParseExpNode(exp2_token))
+  elif t == '$id':
+    return ParseExpNode(Token(head.start + 1, head.end, head.value[1:], 'id'))
+  elif t == '$($)':
+    return LiteralDollarNode(head)
+  elif t == '$':
+    exp_token = Pop('exp')
+    return ParseExpNode(exp_token)
+  elif t == '[[':
+    code_node = ParseCodeNode(tokens)
+    Pop(']]')
+    return code_node
+  else:
+    PushFront(tokens, head)
+    return None
+
+
+def ParseCodeNode(tokens):
+  atomic_code_list = []
+  while True:
+    if not tokens:
+      break
+    atomic_code_node = ParseAtomicCodeNode(tokens)
+    if atomic_code_node:
+      atomic_code_list.append(atomic_code_node)
+    else:
+      break
+  return CodeNode(atomic_code_list)
+
+
+def ParseToAST(pump_src_text):
+  """Convert the given Pump source text into an AST."""
+  tokens = list(Tokenize(pump_src_text))
+  code_node = ParseCodeNode(tokens)
+  return code_node
+
+
+class Env:
+  def __init__(self):
+    self.variables = []
+    self.ranges = []
+
+  def Clone(self):
+    clone = Env()
+    clone.variables = self.variables[:]
+    clone.ranges = self.ranges[:]
+    return clone
+
+  def PushVariable(self, var, value):
+    # If value looks like an int, store it as an int.
+    try:
+      int_value = int(value)
+      if ('%s' % int_value) == value:
+        value = int_value
+    except Exception:
+      pass
+    self.variables[:0] = [(var, value)]
+
+  def PopVariable(self):
+    self.variables[:1] = []
+
+  def PushRange(self, var, lower, upper):
+    self.ranges[:0] = [(var, lower, upper)]
+
+  def PopRange(self):
+    self.ranges[:1] = []
+
+  def GetValue(self, identifier):
+    for (var, value) in self.variables:
+      if identifier == var:
+        return value
+
+    print 'ERROR: meta variable %s is undefined.' % (identifier,)
+    sys.exit(1)
+
+  def EvalExp(self, exp):
+    try:
+      result = eval(exp.python_exp)
+    except Exception, e:
+      print 'ERROR: caught exception %s: %s' % (e.__class__.__name__, e)
+      print ('ERROR: failed to evaluate meta expression %s at %s' %
+             (exp.python_exp, exp.token.start))
+      sys.exit(1)
+    return result
+
+  def GetRange(self, identifier):
+    for (var, lower, upper) in self.ranges:
+      if identifier == var:
+        return (lower, upper)
+
+    print 'ERROR: range %s is undefined.' % (identifier,)
+    sys.exit(1)
+
+
+class Output:
+  def __init__(self):
+    self.string = ''
+
+  def GetLastLine(self):
+    index = self.string.rfind('\n')
+    if index < 0:
+      return ''
+
+    return self.string[index + 1:]
+
+  def Append(self, s):
+    self.string += s
+
+
+def RunAtomicCode(env, node, output):
+  if isinstance(node, VarNode):
+    identifier = node.identifier.value.strip()
+    result = Output()
+    RunAtomicCode(env.Clone(), node.atomic_code, result)
+    value = result.string
+    env.PushVariable(identifier, value)
+  elif isinstance(node, RangeNode):
+    identifier = node.identifier.value.strip()
+    lower = int(env.EvalExp(node.exp1))
+    upper = int(env.EvalExp(node.exp2))
+    env.PushRange(identifier, lower, upper)
+  elif isinstance(node, ForNode):
+    identifier = node.identifier.value.strip()
+    if node.sep is None:
+      sep = ''
+    else:
+      sep = node.sep.value
+    (lower, upper) = env.GetRange(identifier)
+    for i in range(lower, upper + 1):
+      new_env = env.Clone()
+      new_env.PushVariable(identifier, i)
+      RunCode(new_env, node.code, output)
+      if i != upper:
+        output.Append(sep)
+  elif isinstance(node, RawCodeNode):
+    output.Append(node.raw_code.value)
+  elif isinstance(node, IfNode):
+    cond = env.EvalExp(node.exp)
+    if cond:
+      RunCode(env.Clone(), node.then_branch, output)
+    elif node.else_branch is not None:
+      RunCode(env.Clone(), node.else_branch, output)
+  elif isinstance(node, ExpNode):
+    value = env.EvalExp(node)
+    output.Append('%s' % (value,))
+  elif isinstance(node, LiteralDollarNode):
+    output.Append('$')
+  elif isinstance(node, CodeNode):
+    RunCode(env.Clone(), node, output)
+  else:
+    print 'BAD'
+    print node
+    sys.exit(1)
+
+
+def RunCode(env, code_node, output):
+  for atomic_code in code_node.atomic_code:
+    RunAtomicCode(env, atomic_code, output)
+
+
+def IsSingleLineComment(cur_line):
+  return '//' in cur_line
+
+
+def IsInPreprocessorDirective(prev_lines, cur_line):
+  if cur_line.lstrip().startswith('#'):
+    return True
+  return prev_lines and prev_lines[-1].endswith('\\')
+
+
+def WrapComment(line, output):
+  loc = line.find('//')
+  before_comment = line[:loc].rstrip()
+  if before_comment == '':
+    indent = loc
+  else:
+    output.append(before_comment)
+    indent = len(before_comment) - len(before_comment.lstrip())
+  prefix = indent*' ' + '// '
+  max_len = 80 - len(prefix)
+  comment = line[loc + 2:].strip()
+  segs = [seg for seg in re.split(r'(\w+\W*)', comment) if seg != '']
+  cur_line = ''
+  for seg in segs:
+    if len((cur_line + seg).rstrip()) < max_len:
+      cur_line += seg
+    else:
+      if cur_line.strip() != '':
+        output.append(prefix + cur_line.rstrip())
+      cur_line = seg.lstrip()
+  if cur_line.strip() != '':
+    output.append(prefix + cur_line.strip())
+
+
+def WrapCode(line, line_concat, output):
+  indent = len(line) - len(line.lstrip())
+  prefix = indent*' '  # Prefix of the current line
+  max_len = 80 - indent - len(line_concat)  # Maximum length of the current line
+  new_prefix = prefix + 4*' '  # Prefix of a continuation line
+  new_max_len = max_len - 4  # Maximum length of a continuation line
+  # Prefers to wrap a line after a ',' or ';'.
+  segs = [seg for seg in re.split(r'([^,;]+[,;]?)', line.strip()) if seg != '']
+  cur_line = ''  # The current line without leading spaces.
+  for seg in segs:
+    # If the line is still too long, wrap at a space.
+    while cur_line == '' and len(seg.strip()) > max_len:
+      seg = seg.lstrip()
+      split_at = seg.rfind(' ', 0, max_len)
+      output.append(prefix + seg[:split_at].strip() + line_concat)
+      seg = seg[split_at + 1:]
+      prefix = new_prefix
+      max_len = new_max_len
+
+    if len((cur_line + seg).rstrip()) < max_len:
+      cur_line = (cur_line + seg).lstrip()
+    else:
+      output.append(prefix + cur_line.rstrip() + line_concat)
+      prefix = new_prefix
+      max_len = new_max_len
+      cur_line = seg.lstrip()
+  if cur_line.strip() != '':
+    output.append(prefix + cur_line.strip())
+
+
+def WrapPreprocessorDirective(line, output):
+  WrapCode(line, ' \\', output)
+
+
+def WrapPlainCode(line, output):
+  WrapCode(line, '', output)
+
+
+def IsMultiLineIWYUPragma(line):
+  return re.search(r'/\* IWYU pragma: ', line)
+
+
+def IsHeaderGuardIncludeOrOneLineIWYUPragma(line):
+  return (re.match(r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line) or
+          re.match(r'^#include\s', line) or
+          # Don't break IWYU pragmas, either; that causes iwyu.py problems.
+          re.search(r'// IWYU pragma: ', line))
+
+
+def WrapLongLine(line, output):
+  line = line.rstrip()
+  if len(line) <= 80:
+    output.append(line)
+  elif IsSingleLineComment(line):
+    if IsHeaderGuardIncludeOrOneLineIWYUPragma(line):
+      # The style guide made an exception to allow long header guard lines,
+      # includes and IWYU pragmas.
+      output.append(line)
+    else:
+      WrapComment(line, output)
+  elif IsInPreprocessorDirective(output, line):
+    if IsHeaderGuardIncludeOrOneLineIWYUPragma(line):
+      # The style guide made an exception to allow long header guard lines,
+      # includes and IWYU pragmas.
+      output.append(line)
+    else:
+      WrapPreprocessorDirective(line, output)
+  elif IsMultiLineIWYUPragma(line):
+    output.append(line)
+  else:
+    WrapPlainCode(line, output)
+
+
+def BeautifyCode(string):
+  lines = string.splitlines()
+  output = []
+  for line in lines:
+    WrapLongLine(line, output)
+  output2 = [line.rstrip() for line in output]
+  return '\n'.join(output2) + '\n'
+
+
+def ConvertFromPumpSource(src_text):
+  """Return the text generated from the given Pump source text."""
+  ast = ParseToAST(StripMetaComments(src_text))
+  output = Output()
+  RunCode(Env(), ast, output)
+  return BeautifyCode(output.string)
+
+
+def main(argv):
+  if len(argv) == 1:
+    print __doc__
+    sys.exit(1)
+
+  file_path = argv[-1]
+  output_str = ConvertFromPumpSource(file(file_path, 'r').read())
+  if file_path.endswith('.pump'):
+    output_file_path = file_path[:-5]
+  else:
+    output_file_path = '-'
+  if output_file_path == '-':
+    print output_str,
+  else:
+    output_file = file(output_file_path, 'w')
+    output_file.write('// This file was GENERATED by command:\n')
+    output_file.write('//     %s %s\n' %
+                      (os.path.basename(__file__), os.path.basename(file_path)))
+    output_file.write('// DO NOT EDIT BY HAND!!!\n\n')
+    output_file.write(output_str)
+    output_file.close()
+
+
+if __name__ == '__main__':
+  main(sys.argv)