var fs = require('fs')
var Lexer = require('lex')

function readFile(filePath) {
	return fs.readFileSync(filePath, { encoding: 'utf8' });
}

function countTokens(program) {
    var operators = [
        '+', '-', '*', '/', '%', '!=', '==', '<', '>', '<=', '>=', '=',
        '+=', '-=', '*=', '%=', '<<', '>>', '>>>', '<<=', '>>=',
        '>>>=', '&', '&=', '|', '|=', '&&', '||', '^', '^=', '(', ')',
        '[', ']', '{', '}', '!', '--', '++', '~', ',', ';', '.', ':'
    ].sort((x, y) => y.length - x.length)

    var row = 0
    var col = 0
    var tokenCount = 0

    function logToken(errType, tokenCount, val) {
        // console.log(errType, tokenCount, val)
    }

    var lexer = (new Lexer(function() { console.log('Error at ' + col + ':' + row + ':'); console.log(arguments); }))
    .addRule(/(\r\n|\n|\r)/, function() { this.reject = true; row++; col = 1; }, [])
    .addRule(/./, function() { this.reject = true; col++; }, [])
    .addRule(/\s+/, function() {}) // None
    .addRule(/<!--.*/, function() {}) // None
    .addRule(/function/, function(val) { tokenCount++; logToken('F', tokenCount, val)}) // Function
    .addRule(new RegExp('//.*'), function() {}) // Line Comment
    .addRule(new RegExp('/\\*.*?\\*/', 'm'), function() {}) // Multi-line Comment
    .addRule(/[\w$_][\w\d$_.]*[\w\d$_.]/, function(val) { tokenCount += (val.split('.').length * 2 - 1); val.split('.').map(x => logToken('D', tokenCount, x)) }) // Dotted Name
    .addRule(/[\w$_][\w\d$_]*/, function(val) { tokenCount++; logToken('N', tokenCount, val)}) // Name
    .addRule(/(?:0|[1-9]\d*)(\.\d+)/, function(val) { tokenCount++; logToken('0', tokenCount, val)}) // Number
    .addRule(new RegExp(operators.map(x => x.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|')), function(val) { tokenCount++; logToken('O', tokenCount, val)}) // Operator
    .addRule(/( \'(?:[^\'\\]*(?:\\.[^\'\\]*)*)\' | "(?:[^"\\]*(?:\\.[^"\\]*)*)" )/, function(val) { tokenCount++; logToken('S', tokenCount, val)}) // String

    lexer.setInput(program)
    lexer.lex()

    return tokenCount
}

program = readFile('src.js');
console.log(countTokens(program));