@ -1,11 +1,15 @@
#!/usr/bin/python
#!/usr/bin/python
from __future__ import print_function
import sys , os , re , difflib , unicodedata , errno , cgi
import sys , os , re , difflib , unicodedata , errno , cgi
from itertools import *
from itertools import *
diff_symbols = " -+=*&^ % $#@!~/ "
diff_symbols = " -+=*&^ % $#@!~/ "
diff_colors = [ ' red ' , ' green ' , ' blue ' ]
diff_colors = [ ' red ' , ' green ' , ' blue ' ]
if sys . version_info [ 0 ] > = 3 :
unichr = chr
class ColorFormatter :
class ColorFormatter :
class Null :
class Null :
@ -142,7 +146,7 @@ class ZipDiffer:
sys . stdout . writelines ( [ symbols [ i ] , l ] )
sys . stdout . writelines ( [ symbols [ i ] , l ] )
except IOError as e :
except IOError as e :
if e . errno != errno . EPIPE :
if e . errno != errno . EPIPE :
print >> sys . stderr , " %s : %s : %s " % ( sys . argv [ 0 ] , e . filename , e . strerror )
print ( " %s : %s : %s " % ( sys . argv [ 0 ] , e . filename , e . strerror ) , file = sys . stder r )
sys . exit ( 1 )
sys . exit ( 1 )
@ -215,7 +219,7 @@ class DiffSinks:
else :
else :
failed + = 1
failed + = 1
total = passed + failed
total = passed + failed
print " %d out of %d tests passed. %d failed ( %g %% ) " % ( passed , total , failed , 100. * failed / total )
print ( " %d out of %d tests passed. %d failed ( %g %% ) " % ( passed , total , failed , 100. * failed / total ) )
@staticmethod
@staticmethod
def print_ngrams ( f , ns = ( 1 , 2 , 3 ) ) :
def print_ngrams ( f , ns = ( 1 , 2 , 3 ) ) :
@ -240,7 +244,7 @@ class DiffSinks:
del importantgrams
del importantgrams
for ngram , stats in allgrams . iteritems ( ) :
for ngram , stats in allgrams . iteritems ( ) :
print " zscore: %9f failed: %6d passed: %6d ngram: < %s > " % ( stats . zscore ( allstats ) , stats . failed . count , stats . passed . count , ' , ' . join ( " U+ %04X " % u for u in ngram ) )
print ( " zscore: %9f failed: %6d passed: %6d ngram: < %s > " % ( stats . zscore ( allstats ) , stats . failed . count , stats . passed . count , ' , ' . join ( " U+ %04X " % u for u in ngram ) ) )
@ -310,7 +314,7 @@ class FilterHelpers:
def filter_printer_function ( filter_callback ) :
def filter_printer_function ( filter_callback ) :
def printer ( f ) :
def printer ( f ) :
for line in filter_callback ( f ) :
for line in filter_callback ( f ) :
print line
print ( line )
return printer
return printer
@staticmethod
@staticmethod
@ -344,7 +348,7 @@ class UtilMains:
def process_multiple_files ( callback , mnemonic = " FILE " ) :
def process_multiple_files ( callback , mnemonic = " FILE " ) :
if " --help " in sys . argv :
if " --help " in sys . argv :
print " Usage: %s %s ... " % ( sys . argv [ 0 ] , mnemonic )
print ( " Usage: %s %s ... " % ( sys . argv [ 0 ] , mnemonic ) )
sys . exit ( 1 )
sys . exit ( 1 )
try :
try :
@ -353,14 +357,14 @@ class UtilMains:
callback ( FileHelpers . open_file_or_stdin ( s ) )
callback ( FileHelpers . open_file_or_stdin ( s ) )
except IOError as e :
except IOError as e :
if e . errno != errno . EPIPE :
if e . errno != errno . EPIPE :
print >> sys . stderr , " %s : %s : %s " % ( sys . argv [ 0 ] , e . filename , e . strerror )
print ( " %s : %s : %s " % ( sys . argv [ 0 ] , e . filename , e . strerror ) , file = sys . stder r )
sys . exit ( 1 )
sys . exit ( 1 )
@staticmethod
@staticmethod
def process_multiple_args ( callback , mnemonic ) :
def process_multiple_args ( callback , mnemonic ) :
if len ( sys . argv ) == 1 or " --help " in sys . argv :
if len ( sys . argv ) == 1 or " --help " in sys . argv :
print " Usage: %s %s ... " % ( sys . argv [ 0 ] , mnemonic )
print ( " Usage: %s %s ... " % ( sys . argv [ 0 ] , mnemonic ) )
sys . exit ( 1 )
sys . exit ( 1 )
try :
try :
@ -368,7 +372,7 @@ class UtilMains:
callback ( s )
callback ( s )
except IOError as e :
except IOError as e :
if e . errno != errno . EPIPE :
if e . errno != errno . EPIPE :
print >> sys . stderr , " %s : %s : %s " % ( sys . argv [ 0 ] , e . filename , e . strerror )
print ( " %s : %s : %s " % ( sys . argv [ 0 ] , e . filename , e . strerror ) , file = sys . stder r )
sys . exit ( 1 )
sys . exit ( 1 )
@staticmethod
@staticmethod
@ -377,8 +381,8 @@ class UtilMains:
concat_separator = False ) :
concat_separator = False ) :
if " --help " in sys . argv :
if " --help " in sys . argv :
print " Usage: \n %s %s ... \n or: \n %s \n \n When called with no arguments, input is read from standard input. " \
print ( " Usage: \n %s %s ... \n or: \n %s \n \n When called with no arguments, input is read from standard input. " \
% ( sys . argv [ 0 ] , mnemonic , sys . argv [ 0 ] )
% ( sys . argv [ 0 ] , mnemonic , sys . argv [ 0 ] ) )
sys . exit ( 1 )
sys . exit ( 1 )
try :
try :
@ -389,15 +393,15 @@ class UtilMains:
break
break
if line [ - 1 ] == ' \n ' :
if line [ - 1 ] == ' \n ' :
line = line [ : - 1 ]
line = line [ : - 1 ]
print callback ( line )
print ( callback ( line ) )
else :
else :
args = sys . argv [ 1 : ]
args = sys . argv [ 1 : ]
if concat_separator != False :
if concat_separator != False :
args = [ concat_separator . join ( args ) ]
args = [ concat_separator . join ( args ) ]
print separator . join ( callback ( x ) for x in ( args ) )
print ( separator . join ( callback ( x ) for x in ( args ) ) )
except IOError as e :
except IOError as e :
if e . errno != errno . EPIPE :
if e . errno != errno . EPIPE :
print >> sys . stderr , " %s : %s : %s " % ( sys . argv [ 0 ] , e . filename , e . strerror )
print ( " %s : %s : %s " % ( sys . argv [ 0 ] , e . filename , e . strerror ) , file = sys . stder r )
sys . exit ( 1 )
sys . exit ( 1 )
@ -415,7 +419,9 @@ class Unicode:
@staticmethod
@staticmethod
def encode ( s ) :
def encode ( s ) :
return u ' ' . join ( unichr ( x ) for x in Unicode . parse ( s ) ) . encode ( ' utf-8 ' )
s = u ' ' . join ( unichr ( x ) for x in Unicode . parse ( s ) )
if sys . version_info [ 0 ] == 2 : s = s . encode ( ' utf-8 ' )
return s
shorthands = {
shorthands = {
" ZERO WIDTH NON-JOINER " : " ZWNJ " ,
" ZERO WIDTH NON-JOINER " : " ZWNJ " ,
@ -471,7 +477,7 @@ class Manifest:
if not os . path . exists ( s ) :
if not os . path . exists ( s ) :
if strict :
if strict :
print >> sys . stderr , " %s : %s does not exist " % ( sys . argv [ 0 ] , s )
print ( " %s : %s does not exist " % ( sys . argv [ 0 ] , s ) , file = sys . stderr )
sys . exit ( 1 )
sys . exit ( 1 )
return
return
@ -487,7 +493,7 @@ class Manifest:
yield p
yield p
except IOError :
except IOError :
if strict :
if strict :
print >> sys . stderr , " %s : %s does not exist " % ( sys . argv [ 0 ] , os . path . join ( s , " MANIFEST " ) )
print ( " %s : %s does not exist " % ( sys . argv [ 0 ] , os . path . join ( s , " MANIFEST " ) ) , file = sys . stderr )
sys . exit ( 1 )
sys . exit ( 1 )
return
return
else :
else :
@ -506,12 +512,12 @@ class Manifest:
dirnames . sort ( )
dirnames . sort ( )
filenames . sort ( )
filenames . sort ( )
ms = os . path . join ( dirpath , " MANIFEST " )
ms = os . path . join ( dirpath , " MANIFEST " )
print " GEN %s " % ms
print ( " GEN %s " % ms )
m = open ( ms , " w " )
m = open ( ms , " w " )
for f in filenames :
for f in filenames :
print >> m , f
print ( f , file = m )
for f in dirnames :
for f in dirnames :
print >> m , f
print ( f , file = m )
for f in dirnames :
for f in dirnames :
Manifest . update_recursive ( os . path . join ( dirpath , f ) )
Manifest . update_recursive ( os . path . join ( dirpath , f ) )