#!/usr/pkg/bin/python
"""precursor to pfcompile, just enumerates tokens in .pf file

   purpose of this is to keep things extremely simple when bootstrapping;
   I was doing this with the RCS file but each time I change machines
   I lose that trail"""

Copyright = """
    pfcompile -- bootstrap compiler for postforth version 1
    Copyright (C) 2005  John Comeau <jc.jcomeau.com>

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
    """
errormessage = "Not all needed libraries found, upgrade or check path"
try:
 True # not defined in older Python releases
except:
 True, False = 1, 0
try:
 import sys, os, types, re, pwd
 sys.path.append(os.sep.join([pwd.getpwuid(os.geteuid())[5], 'lib', 'python']))
 from com.jcomeau import gpl, jclicense
except:
 try:
  sys.stderr.write("%s\n" % errormessage)
 except:
  print errormessage
 raise

# get name this program was called as
self = sys.argv[0].split(os.sep)[-1]
command = self.split('.')[0]  # chop any suffix (extension)
# now get name we gave it when we wrote it
originalself = Copyright.split()[0]

# globals and routines that should be in every program
# (yes, you could import them, but there are problems in that approach too)

def DebugPrint(*whatever):
 return False  # defined instead by pytest module, use that for debugging

def noTuples(*args):
 "reduces args to lists or scalars, helps with command-line testing"
 while type(args[0]) == types.TupleType: args = args[0]
 return args

def scalars(*args):
 "reduces args to scalars, helps with command-line testing"
 args = noTuples(args)
 while type(args[0]) == types.ListType: args = args[0]
 return args

# other globals, specific to this program
from com.jcomeau.arcane import countdict
postforth = {
 '/*': 'multilineComment',
 '//': 'comment',
 '\n': 'type',  # used as no-op
 '2#': 'skipnext',
 '10#': 'skipnext',
 '16#': 'skipnext',
}

def multilineComment(*args):
 "skip past next */"
 DebugPrint('processing multiline comment')
 args = scalars(args)
 tokenizer = args[0]
 while True:
  DebugPrint('skipping', tokenizer.list[tokenizer.pointer])
  token = tokenizer.getnext()
  if token == None or token.endswith('*/'): break

def comment(*args):
 "skip past following newline"
 DebugPrint('processing comment')
 args = scalars(args)
 tokenizer = args[0]
 while True:
  DebugPrint('skipping', tokenizer.list[tokenizer.pointer])
  token = tokenizer.getnext()
  if token == None or token.endswith('\n'): break

def skipnext(*args):
 args = scalars(args)
 tokenizer = args[0]
 DebugPrint('skipping', tokenizer.list[tokenizer.pointer])
 tokenizer.getnext()

class tokenizer:
 pointer = 0
 list = []

 def __init__(self, *args):
  args = scalars(args)
  sourcefile = args[0]
  source = map(str.split, open(sourcefile).readlines())
  for line in source:
   self.list.extend(line)
   self.list.append('\n')
  DebugPrint('list', self.list)

 def getnext(self):
  if self.pointer == len(self.list):
   return None
  else:
   next = self.list[self.pointer]
   self.pointer += 1
   return next

def pftokens(*args):
 """enumerate postFORTH tokens from program(s) passed in on command line

    using "inside knowledge" of postFORTH, find tokens and output
    as a list
 """
 DebugPrint("'pytest %s.%s' should show this; args: " % (
  self, originalself), args)
 args = scalars(args)
 tokenmap = dict()
 tokenmap.update(postforth)
 tokens = countdict()
 for sourcefile in args:
  pftokenizer = tokenizer(sourcefile)
  while True:
   token = pftokenizer.getnext()
   DebugPrint('token', token)
   if token == None: break
   if token in tokenmap.keys():
    eval(tokenmap[token])(pftokenizer)
   if token != '\n': tokens[token] += 1
 DebugPrint(tokens)
 print tokens.keys()

if __name__ == '__main__':
 # if this program was imported by another, the above test will fail,
 # and this following code won't be used...
 pftokens(sys.argv[1:])
else:
 # if you want something to be done on import, do it here; otherwise pass
 pass