darwin/cogapp/cogapp.py

d775ee353a Oliv*0001 """ Cog content generation tool.
8fbfd1f382 Oliv*0002 """
                0003
d775ee353a Oliv*0004 import copy
                0005 import getopt
                0006 import glob
                0007 import io
                0008 import linecache
                0009 import os
                0010 import re
                0011 import shlex
                0012 import sys
                0013 import traceback
                0014 import types
8fbfd1f382 Oliv*0015
d775ee353a Oliv*0016 from .whiteutils import commonPrefix, reindentBlock, whitePrefix
                0017 from .utils import NumberedFileReader, Redirectable, change_dir, md5
8fbfd1f382 Oliv*0018
d775ee353a Oliv*0019 __version__ = "3.4.1"
8fbfd1f382 Oliv*0020
                0021 usage = """\
d775ee353a Oliv*0022 cog - generate content with inlined Python code.
8fbfd1f382 Oliv*0023
d775ee353a Oliv*0024 cog [OPTIONS] [INFILE | @FILELIST | &FILELIST] ...
8fbfd1f382 Oliv*0025
                0026 INFILE is the name of an input file, '-' will read from stdin.
                0027 FILELIST is the name of a text file containing file names or
d775ee353a Oliv*0028 other @FILELISTs.
                0029
                0030 For @FILELIST, paths in the file list are relative to the working
                0031 directory where cog was called.  For &FILELIST, paths in the file
                0032 list are relative to the file list location.
8fbfd1f382 Oliv*0033
                0034 OPTIONS:
                0035     -c          Checksum the output to protect it against accidental change.
                0036     -d          Delete the generator code from the output file.
                0037     -D name=val Define a global string available to your generator code.
                0038     -e          Warn if a file has no cog code in it.
                0039     -I PATH     Add PATH to the list of directories for data files and modules.
d775ee353a Oliv*0040     -n ENCODING Use ENCODING when reading and writing files.
8fbfd1f382 Oliv*0041     -o OUTNAME  Write the output to OUTNAME.
d775ee353a Oliv*0042     -p PROLOGUE Prepend the generator source with PROLOGUE. Useful to insert an
                0043                 import line. Example: -p "import math"
                0044     -P          Use print() instead of cog.outl() for code output.
8fbfd1f382 Oliv*0045     -r          Replace the input file with the output.
                0046     -s STRING   Suffix all generated output lines with STRING.
                0047     -U          Write the output with Unix newlines (only LF line-endings).
                0048     -w CMD      Use CMD if the output file needs to be made writable.
                0049                     A %s in the CMD will be filled with the filename.
                0050     -x          Excise all the generated output without running the generators.
d775ee353a Oliv*0051     -z          The end-output marker can be omitted, and is assumed at eof.
8fbfd1f382 Oliv*0052     -v          Print the version of cog and exit.
d775ee353a Oliv*0053     --check     Check that the files would not change if run again.
                0054     --markers='START END END-OUTPUT'
                0055                 The patterns surrounding cog inline instructions. Should
                0056                 include three values separated by spaces, the start, end,
                0057                 and end-output markers. Defaults to '[[[cog ]]] [[[end]]]'.
                0058     --verbosity=VERBOSITY
                0059                 Control the amount of output. 2 (the default) lists all files,
                0060                 1 lists only changed files, 0 lists no files.
8fbfd1f382 Oliv*0061     -h          Print this help.
                0062 """
                0063
                0064 class CogError(Exception):
                0065     """ Any exception raised by Cog.
                0066     """
                0067     def __init__(self, msg, file='', line=0):
                0068         if file:
d775ee353a Oliv*0069             super().__init__(f"{file}({line}): {msg}")
8fbfd1f382 Oliv*0070         else:
d775ee353a Oliv*0071             super().__init__(msg)
8fbfd1f382 Oliv*0072
                0073 class CogUsageError(CogError):
                0074     """ An error in usage of command-line arguments in cog.
                0075     """
d775ee353a Oliv*0076     pass
8fbfd1f382 Oliv*0077
                0078 class CogInternalError(CogError):
                0079     """ An error in the coding of Cog. Should never happen.
                0080     """
d775ee353a Oliv*0081     pass
8fbfd1f382 Oliv*0082
                0083 class CogGeneratedError(CogError):
                0084     """ An error raised by a user's cog generator.
                0085     """
d775ee353a Oliv*0086     pass
8fbfd1f382 Oliv*0087
d775ee353a Oliv*0088 class CogUserException(CogError):
                0089     """ An exception caught when running a user's cog generator.
                0090         The argument is the traceback message to print.
8fbfd1f382 Oliv*0091     """
d775ee353a Oliv*0092     pass
8fbfd1f382 Oliv*0093
d775ee353a Oliv*0094 class CogCheckFailed(CogError):
                0095     """ A --check failed.
                0096     """
                0097     pass
8fbfd1f382 Oliv*0098
                0099
                0100 class CogGenerator(Redirectable):
                0101     """ A generator pulled from a source file.
                0102     """
d775ee353a Oliv*0103     def __init__(self, options=None):
                0104         super().__init__()
8fbfd1f382 Oliv*0105         self.markers = []
                0106         self.lines = []
d775ee353a Oliv*0107         self.options = options or CogOptions()
8fbfd1f382 Oliv*0108
                0109     def parseMarker(self, l):
                0110         self.markers.append(l)
                0111
                0112     def parseLine(self, l):
                0113         self.lines.append(l.strip('\n'))
                0114
                0115     def getCode(self):
                0116         """ Extract the executable Python code from the generator.
                0117         """
                0118         # If the markers and lines all have the same prefix
                0119         # (end-of-line comment chars, for example),
                0120         # then remove it from all the lines.
                0121         prefIn = commonPrefix(self.markers + self.lines)
                0122         if prefIn:
                0123             self.markers = [ l.replace(prefIn, '', 1) for l in self.markers ]
                0124             self.lines = [ l.replace(prefIn, '', 1) for l in self.lines ]
                0125
                0126         return reindentBlock(self.lines, '')
                0127
d775ee353a Oliv*0128     def evaluate(self, cog, globals, fname):
8fbfd1f382 Oliv*0129         # figure out the right whitespace prefix for the output
                0130         prefOut = whitePrefix(self.markers)
                0131
                0132         intext = self.getCode()
                0133         if not intext:
                0134             return ''
                0135
d775ee353a Oliv*0136         prologue = "import " + cog.cogmodulename + " as cog\n"
                0137         if self.options.sPrologue:
                0138             prologue += self.options.sPrologue + '\n'
                0139         code = compile(prologue + intext, str(fname), 'exec')
8fbfd1f382 Oliv*0140
                0141         # Make sure the "cog" module has our state.
                0142         cog.cogmodule.msg = self.msg
                0143         cog.cogmodule.out = self.out
                0144         cog.cogmodule.outl = self.outl
                0145         cog.cogmodule.error = self.error
                0146
d775ee353a Oliv*0147         real_stdout = sys.stdout
                0148         if self.options.bPrintOutput:
                0149             sys.stdout = captured_stdout = io.StringIO()
                0150
8fbfd1f382 Oliv*0151         self.outstring = ''
d775ee353a Oliv*0152         try:
                0153             eval(code, globals)
                0154         except CogError:
                0155             raise
                0156         except:
                0157             typ, err, tb = sys.exc_info()
                0158             frames = (tuple(fr) for fr in traceback.extract_tb(tb.tb_next))
                0159             frames = find_cog_source(frames, prologue)
                0160             msg = "".join(traceback.format_list(frames))
                0161             msg += f"{typ.__name__}: {err}"
                0162             raise CogUserException(msg)
                0163         finally:
                0164             sys.stdout = real_stdout
                0165
                0166         if self.options.bPrintOutput:
                0167             self.outstring = captured_stdout.getvalue()
8fbfd1f382 Oliv*0168
                0169         # We need to make sure that the last line in the output
                0170         # ends with a newline, or it will be joined to the
                0171         # end-output line, ruining cog's idempotency.
                0172         if self.outstring and self.outstring[-1] != '\n':
                0173             self.outstring += '\n'
                0174
                0175 #        return reindentBlock(self.outstring, prefOut)
                0176         return self.outstring
                0177
                0178     def msg(self, s):
                0179         self.prout("Message: "+s)
                0180
                0181     def out(self, sOut='', dedent=False, trimblanklines=False):
                0182         """ The cog.out function.
                0183         """
                0184         if trimblanklines and ('\n' in sOut):
                0185             lines = sOut.split('\n')
                0186             if lines[0].strip() == '':
                0187                 del lines[0]
                0188             if lines and lines[-1].strip() == '':
                0189                 del lines[-1]
                0190             sOut = '\n'.join(lines)+'\n'
                0191         if dedent:
                0192             sOut = reindentBlock(sOut)
                0193         self.outstring += sOut
                0194
                0195     def outl(self, sOut='', **kw):
                0196         """ The cog.outl function.
                0197         """
                0198         self.out(sOut, **kw)
                0199         self.out('\n')
                0200
                0201     def error(self, msg='Error raised by cog generator.'):
                0202         """ The cog.error function.
                0203             Instead of raising standard python errors, cog generators can use
                0204             this function.  It will display the error without a scary Python
                0205             traceback.
                0206         """
                0207         raise CogGeneratedError(msg)
                0208
                0209
                0210 class CogOptions:
                0211     """ Options for a run of cog.
                0212     """
                0213     def __init__(self):
                0214         # Defaults for argument values.
                0215         self.args = []
                0216         self.includePath = []
                0217         self.defines = {}
                0218         self.bShowVersion = False
                0219         self.sMakeWritableCmd = None
                0220         self.bReplace = False
                0221         self.bNoGenerate = False
                0222         self.sOutputName = None
                0223         self.bWarnEmpty = False
                0224         self.bHashOutput = False
                0225         self.bDeleteCode = False
                0226         self.bEofCanBeEnd = False
                0227         self.sSuffix = None
                0228         self.bNewlines = False
d775ee353a Oliv*0229         self.sBeginSpec = '[[[cog'
                0230         self.sEndSpec = ']]]'
                0231         self.sEndOutput = '[[[end]]]'
                0232         self.sEncoding = "utf-8"
                0233         self.verbosity = 2
                0234         self.sPrologue = ''
                0235         self.bPrintOutput = False
                0236         self.bCheck = False
8fbfd1f382 Oliv*0237
                0238     def __eq__(self, other):
                0239         """ Comparison operator for tests to use.
                0240         """
                0241         return self.__dict__ == other.__dict__
                0242
                0243     def clone(self):
                0244         """ Make a clone of these options, for further refinement.
                0245         """
                0246         return copy.deepcopy(self)
                0247
                0248     def addToIncludePath(self, dirs):
                0249         """ Add directories to the include path.
                0250         """
                0251         dirs = dirs.split(os.pathsep)
                0252         self.includePath.extend(dirs)
                0253
                0254     def parseArgs(self, argv):
                0255         # Parse the command line arguments.
                0256         try:
d775ee353a Oliv*0257             opts, self.args = getopt.getopt(
                0258                 argv,
                0259                 'cdD:eI:n:o:rs:p:PUvw:xz',
                0260                 [
                0261                     'check',
                0262                     'markers=',
                0263                     'verbosity=',
                0264                 ]
                0265             )
8fbfd1f382 Oliv*0266         except getopt.error as msg:
                0267             raise CogUsageError(msg)
                0268
                0269         # Handle the command line arguments.
                0270         for o, a in opts:
                0271             if o == '-c':
                0272                 self.bHashOutput = True
                0273             elif o == '-d':
                0274                 self.bDeleteCode = True
                0275             elif o == '-D':
                0276                 if a.count('=') < 1:
                0277                     raise CogUsageError("-D takes a name=value argument")
                0278                 name, value = a.split('=', 1)
                0279                 self.defines[name] = value
                0280             elif o == '-e':
                0281                 self.bWarnEmpty = True
                0282             elif o == '-I':
d775ee353a Oliv*0283                 self.addToIncludePath(os.path.abspath(a))
                0284             elif o == '-n':
                0285                 self.sEncoding = a
8fbfd1f382 Oliv*0286             elif o == '-o':
                0287                 self.sOutputName = a
                0288             elif o == '-r':
                0289                 self.bReplace = True
                0290             elif o == '-s':
                0291                 self.sSuffix = a
d775ee353a Oliv*0292             elif o == '-p':
                0293                 self.sPrologue = a
                0294             elif o == '-P':
                0295                 self.bPrintOutput = True
8fbfd1f382 Oliv*0296             elif o == '-U':
                0297                 self.bNewlines = True
                0298             elif o == '-v':
                0299                 self.bShowVersion = True
                0300             elif o == '-w':
                0301                 self.sMakeWritableCmd = a
                0302             elif o == '-x':
                0303                 self.bNoGenerate = True
                0304             elif o == '-z':
                0305                 self.bEofCanBeEnd = True
d775ee353a Oliv*0306             elif o == '--check':
                0307                 self.bCheck = True
                0308             elif o == '--markers':
                0309                 self._parse_markers(a)
                0310             elif o == '--verbosity':
                0311                 self.verbosity = int(a)
8fbfd1f382 Oliv*0312             else:
                0313                 # Since getopt.getopt is given a list of possible flags,
                0314                 # this is an internal error.
d775ee353a Oliv*0315                 raise CogInternalError(f"Don't understand argument {o}")
                0316
                0317     def _parse_markers(self, val):
                0318         try:
                0319             self.sBeginSpec, self.sEndSpec, self.sEndOutput = val.split(" ")
                0320         except ValueError:
                0321             raise CogUsageError(
                0322                 f"--markers requires 3 values separated by spaces, could not parse {val!r}"
                0323             )
8fbfd1f382 Oliv*0324
                0325     def validate(self):
                0326         """ Does nothing if everything is OK, raises CogError's if it's not.
                0327         """
                0328         if self.bReplace and self.bDeleteCode:
                0329             raise CogUsageError("Can't use -d with -r (or you would delete all your source!)")
                0330
                0331         if self.bReplace and self.sOutputName:
                0332             raise CogUsageError("Can't use -o with -r (they are opposites)")
                0333
                0334
                0335 class Cog(Redirectable):
                0336     """ The Cog engine.
                0337     """
                0338     def __init__(self):
d775ee353a Oliv*0339         super().__init__()
8fbfd1f382 Oliv*0340         self.options = CogOptions()
d775ee353a Oliv*0341         self._fixEndOutputPatterns()
                0342         self.cogmodulename = "cog"
                0343         self.createCogModule()
                0344         self.bCheckFailed = False
8fbfd1f382 Oliv*0345
d775ee353a Oliv*0346     def _fixEndOutputPatterns(self):
                0347         end_output = re.escape(self.options.sEndOutput)
                0348         self.reEndOutput = re.compile(end_output + r"(?P<hashsect> *\(checksum: (?P<hash>[a-f0-9]+)\))")
                0349         self.sEndFormat = self.options.sEndOutput + " (checksum: %s)"
8fbfd1f382 Oliv*0350
                0351     def showWarning(self, msg):
d775ee353a Oliv*0352         self.prout(f"Warning: {msg}")
8fbfd1f382 Oliv*0353
                0354     def isBeginSpecLine(self, s):
d775ee353a Oliv*0355         return self.options.sBeginSpec in s
8fbfd1f382 Oliv*0356
                0357     def isEndSpecLine(self, s):
d775ee353a Oliv*0358         return self.options.sEndSpec in s and not self.isEndOutputLine(s)
8fbfd1f382 Oliv*0359
                0360     def isEndOutputLine(self, s):
d775ee353a Oliv*0361         return self.options.sEndOutput in s
8fbfd1f382 Oliv*0362
d775ee353a Oliv*0363     def createCogModule(self):
                0364         """ Make a cog "module" object so that imported Python modules
8fbfd1f382 Oliv*0365             can say "import cog" and get our state.
                0366         """
d775ee353a Oliv*0367         self.cogmodule = types.SimpleNamespace()
8fbfd1f382 Oliv*0368         self.cogmodule.path = []
                0369
                0370     def openOutputFile(self, fname):
                0371         """ Open an output file, taking all the details into account.
                0372         """
                0373         opts = {}
                0374         mode = "w"
d775ee353a Oliv*0375         opts['encoding'] = self.options.sEncoding
8fbfd1f382 Oliv*0376         if self.options.bNewlines:
d775ee353a Oliv*0377             opts["newline"] = "\n"
                0378         fdir = os.path.dirname(fname)
                0379         if os.path.dirname(fdir) and not os.path.exists(fdir):
                0380             os.makedirs(fdir)
8fbfd1f382 Oliv*0381         return open(fname, mode, **opts)
                0382
                0383     def openInputFile(self, fname):
d775ee353a Oliv*0384         """ Open an input file.
                0385         """
8fbfd1f382 Oliv*0386         if fname == "-":
                0387             return sys.stdin
                0388         else:
d775ee353a Oliv*0389             return open(fname, encoding=self.options.sEncoding)
8fbfd1f382 Oliv*0390
                0391     def processFile(self, fIn, fOut, fname=None, globals=None):
                0392         """ Process an input file object to an output file object.
                0393             fIn and fOut can be file objects, or file names.
                0394         """
                0395
                0396         sFileIn = fname or ''
                0397         sFileOut = fname or ''
                0398         fInToClose = fOutToClose = None
                0399         # Convert filenames to files.
d775ee353a Oliv*0400         if isinstance(fIn, (bytes, str)):
8fbfd1f382 Oliv*0401             # Open the input file.
                0402             sFileIn = fIn
                0403             fIn = fInToClose = self.openInputFile(fIn)
d775ee353a Oliv*0404         if isinstance(fOut, (bytes, str)):
8fbfd1f382 Oliv*0405             # Open the output file.
                0406             sFileOut = fOut
                0407             fOut = fOutToClose = self.openOutputFile(fOut)
                0408
                0409         try:
                0410             fIn = NumberedFileReader(fIn)
                0411
                0412             bSawCog = False
                0413
                0414             self.cogmodule.inFile = sFileIn
                0415             self.cogmodule.outFile = sFileOut
d775ee353a Oliv*0416             self.cogmodulename = 'cog_' + md5(sFileOut.encode()).hexdigest()
                0417             sys.modules[self.cogmodulename] = self.cogmodule
                0418             # if "import cog" explicitly done in code by user, note threading will cause clashes.
                0419             sys.modules['cog'] = self.cogmodule
8fbfd1f382 Oliv*0420
                0421             # The globals dict we'll use for this file.
                0422             if globals is None:
                0423                 globals = {}
                0424
                0425             # If there are any global defines, put them in the globals.
                0426             globals.update(self.options.defines)
                0427
                0428             # loop over generator chunks
                0429             l = fIn.readline()
                0430             while l:
                0431                 # Find the next spec begin
                0432                 while l and not self.isBeginSpecLine(l):
                0433                     if self.isEndSpecLine(l):
d775ee353a Oliv*0434                         raise CogError(
                0435                             f"Unexpected {self.options.sEndSpec!r}",
                0436                             file=sFileIn,
                0437                             line=fIn.linenumber(),
                0438                         )
8fbfd1f382 Oliv*0439                     if self.isEndOutputLine(l):
d775ee353a Oliv*0440                         raise CogError(
                0441                             f"Unexpected {self.options.sEndOutput!r}",
                0442                             file=sFileIn,
                0443                             line=fIn.linenumber(),
                0444                         )
8fbfd1f382 Oliv*0445                     fOut.write(l)
                0446                     l = fIn.readline()
                0447                 if not l:
                0448                     break
                0449                 if not self.options.bDeleteCode:
                0450                     fOut.write(l)
                0451
                0452                 # l is the begin spec
d775ee353a Oliv*0453                 gen = CogGenerator(options=self.options)
8fbfd1f382 Oliv*0454                 gen.setOutput(stdout=self.stdout)
                0455                 gen.parseMarker(l)
                0456                 firstLineNum = fIn.linenumber()
                0457                 self.cogmodule.firstLineNum = firstLineNum
                0458
                0459                 # If the spec begin is also a spec end, then process the single
                0460                 # line of code inside.
                0461                 if self.isEndSpecLine(l):
d775ee353a Oliv*0462                     beg = l.find(self.options.sBeginSpec)
                0463                     end = l.find(self.options.sEndSpec)
8fbfd1f382 Oliv*0464                     if beg > end:
                0465                         raise CogError("Cog code markers inverted",
                0466                             file=sFileIn, line=firstLineNum)
                0467                     else:
d775ee353a Oliv*0468                         sCode = l[beg+len(self.options.sBeginSpec):end].strip()
8fbfd1f382 Oliv*0469                         gen.parseLine(sCode)
                0470                 else:
                0471                     # Deal with an ordinary code block.
                0472                     l = fIn.readline()
                0473
                0474                     # Get all the lines in the spec
                0475                     while l and not self.isEndSpecLine(l):
                0476                         if self.isBeginSpecLine(l):
d775ee353a Oliv*0477                             raise CogError(
                0478                                 f"Unexpected {self.options.sBeginSpec!r}",
                0479                                 file=sFileIn,
                0480                                 line=fIn.linenumber(),
                0481                             )
8fbfd1f382 Oliv*0482                         if self.isEndOutputLine(l):
d775ee353a Oliv*0483                             raise CogError(
                0484                                 f"Unexpected {self.options.sEndOutput!r}",
                0485                                 file=sFileIn,
                0486                                 line=fIn.linenumber(),
                0487                             )
8fbfd1f382 Oliv*0488                         if not self.options.bDeleteCode:
                0489                             fOut.write(l)
                0490                         gen.parseLine(l)
                0491                         l = fIn.readline()
                0492                     if not l:
                0493                         raise CogError(
                0494                             "Cog block begun but never ended.",
                0495                             file=sFileIn, line=firstLineNum)
                0496
                0497                     if not self.options.bDeleteCode:
                0498                         fOut.write(l)
                0499                     gen.parseMarker(l)
                0500
                0501                 l = fIn.readline()
                0502
                0503                 # Eat all the lines in the output section.  While reading past
                0504                 # them, compute the md5 hash of the old output.
d775ee353a Oliv*0505                 previous = []
                0506                 hasher = md5()
8fbfd1f382 Oliv*0507                 while l and not self.isEndOutputLine(l):
                0508                     if self.isBeginSpecLine(l):
d775ee353a Oliv*0509                         raise CogError(
                0510                             f"Unexpected {self.options.sBeginSpec!r}",
                0511                             file=sFileIn,
                0512                             line=fIn.linenumber(),
                0513                         )
8fbfd1f382 Oliv*0514                     if self.isEndSpecLine(l):
d775ee353a Oliv*0515                         raise CogError(
                0516                             f"Unexpected {self.options.sEndSpec!r}",
                0517                             file=sFileIn,
                0518                             line=fIn.linenumber(),
                0519                         )
                0520                     previous.append(l)
                0521                     hasher.update(l.encode("utf-8"))
8fbfd1f382 Oliv*0522                     l = fIn.readline()
                0523                 curHash = hasher.hexdigest()
                0524
                0525                 if not l and not self.options.bEofCanBeEnd:
                0526                     # We reached end of file before we found the end output line.
d775ee353a Oliv*0527                     raise CogError(
                0528                         f"Missing {self.options.sEndOutput!r} before end of file.",
                0529                         file=sFileIn,
                0530                         line=fIn.linenumber(),
                0531                     )
8fbfd1f382 Oliv*0532
                0533                 # Make the previous output available to the current code
d775ee353a Oliv*0534                 self.cogmodule.previous = "".join(previous)
8fbfd1f382 Oliv*0535
                0536                 # Write the output of the spec to be the new output if we're
                0537                 # supposed to generate code.
d775ee353a Oliv*0538                 hasher = md5()
8fbfd1f382 Oliv*0539                 if not self.options.bNoGenerate:
d775ee353a Oliv*0540                     sFile = f"<cog {sFileIn}:{firstLineNum}>"
8fbfd1f382 Oliv*0541                     sGen = gen.evaluate(cog=self, globals=globals, fname=sFile)
                0542                     sGen = self.suffixLines(sGen)
d775ee353a Oliv*0543                     hasher.update(sGen.encode("utf-8"))
8fbfd1f382 Oliv*0544                     fOut.write(sGen)
                0545                 newHash = hasher.hexdigest()
                0546
                0547                 bSawCog = True
                0548
                0549                 # Write the ending output line
                0550                 hashMatch = self.reEndOutput.search(l)
                0551                 if self.options.bHashOutput:
                0552                     if hashMatch:
d775ee353a Oliv*0553                         oldHash = hashMatch['hash']
8fbfd1f382 Oliv*0554                         if oldHash != curHash:
                0555                             raise CogError("Output has been edited! Delete old checksum to unprotect.",
                0556                                 file=sFileIn, line=fIn.linenumber())
                0557                         # Create a new end line with the correct hash.
                0558                         endpieces = l.split(hashMatch.group(0), 1)
                0559                     else:
                0560                         # There was no old hash, but we want a new hash.
d775ee353a Oliv*0561                         endpieces = l.split(self.options.sEndOutput, 1)
8fbfd1f382 Oliv*0562                     l = (self.sEndFormat % newHash).join(endpieces)
                0563                 else:
                0564                     # We don't want hashes output, so if there was one, get rid of
                0565                     # it.
                0566                     if hashMatch:
d775ee353a Oliv*0567                         l = l.replace(hashMatch['hashsect'], '', 1)
8fbfd1f382 Oliv*0568
                0569                 if not self.options.bDeleteCode:
                0570                     fOut.write(l)
                0571                 l = fIn.readline()
                0572
                0573             if not bSawCog and self.options.bWarnEmpty:
d775ee353a Oliv*0574                 self.showWarning(f"no cog code found in {sFileIn}")
8fbfd1f382 Oliv*0575         finally:
                0576             if fInToClose:
                0577                 fInToClose.close()
                0578             if fOutToClose:
                0579                 fOutToClose.close()
                0580
                0581
                0582     # A regex for non-empty lines, used by suffixLines.
d775ee353a Oliv*0583     reNonEmptyLines = re.compile(r"^\s*\S+.*$", re.MULTILINE)
8fbfd1f382 Oliv*0584
                0585     def suffixLines(self, text):
                0586         """ Add suffixes to the lines in text, if our options desire it.
                0587             text is many lines, as a single string.
                0588         """
                0589         if self.options.sSuffix:
                0590             # Find all non-blank lines, and add the suffix to the end.
                0591             repl = r"\g<0>" + self.options.sSuffix.replace('\\', '\\\\')
                0592             text = self.reNonEmptyLines.sub(repl, text)
                0593         return text
                0594
                0595     def processString(self, sInput, fname=None):
                0596         """ Process sInput as the text to cog.
                0597             Return the cogged output as a string.
                0598         """
d775ee353a Oliv*0599         fOld = io.StringIO(sInput)
                0600         fNew = io.StringIO()
8fbfd1f382 Oliv*0601         self.processFile(fOld, fNew, fname=fname)
                0602         return fNew.getvalue()
                0603
                0604     def replaceFile(self, sOldPath, sNewText):
                0605         """ Replace file sOldPath with the contents sNewText
                0606         """
                0607         if not os.access(sOldPath, os.W_OK):
                0608             # Need to ensure we can write.
                0609             if self.options.sMakeWritableCmd:
                0610                 # Use an external command to make the file writable.
                0611                 cmd = self.options.sMakeWritableCmd.replace('%s', sOldPath)
d775ee353a Oliv*0612                 with os.popen(cmd) as cmdout:
                0613                     self.stdout.write(cmdout.read())
8fbfd1f382 Oliv*0614                 if not os.access(sOldPath, os.W_OK):
d775ee353a Oliv*0615                     raise CogError(f"Couldn't make {sOldPath} writable")
8fbfd1f382 Oliv*0616             else:
                0617                 # Can't write!
d775ee353a Oliv*0618                 raise CogError(f"Can't overwrite {sOldPath}")
8fbfd1f382 Oliv*0619         f = self.openOutputFile(sOldPath)
                0620         f.write(sNewText)
                0621         f.close()
                0622
                0623     def saveIncludePath(self):
                0624         self.savedInclude = self.options.includePath[:]
                0625         self.savedSysPath = sys.path[:]
                0626
                0627     def restoreIncludePath(self):
                0628         self.options.includePath = self.savedInclude
                0629         self.cogmodule.path = self.options.includePath
                0630         sys.path = self.savedSysPath
                0631
                0632     def addToIncludePath(self, includePath):
                0633         self.cogmodule.path.extend(includePath)
                0634         sys.path.extend(includePath)
                0635
                0636     def processOneFile(self, sFile):
                0637         """ Process one filename through cog.
                0638         """
                0639
                0640         self.saveIncludePath()
d775ee353a Oliv*0641         bNeedNewline = False
8fbfd1f382 Oliv*0642
                0643         try:
                0644             self.addToIncludePath(self.options.includePath)
                0645             # Since we know where the input file came from,
                0646             # push its directory onto the include path.
                0647             self.addToIncludePath([os.path.dirname(sFile)])
                0648
                0649             # How we process the file depends on where the output is going.
                0650             if self.options.sOutputName:
                0651                 self.processFile(sFile, self.options.sOutputName, sFile)
d775ee353a Oliv*0652             elif self.options.bReplace or self.options.bCheck:
8fbfd1f382 Oliv*0653                 # We want to replace the cog file with the output,
                0654                 # but only if they differ.
d775ee353a Oliv*0655                 verb = "Cogging" if self.options.bReplace else "Checking"
                0656                 if self.options.verbosity >= 2:
                0657                     self.prout(f"{verb} {sFile}", end="")
                0658                     bNeedNewline = True
8fbfd1f382 Oliv*0659
                0660                 try:
                0661                     fOldFile = self.openInputFile(sFile)
                0662                     sOldText = fOldFile.read()
                0663                     fOldFile.close()
                0664                     sNewText = self.processString(sOldText, fname=sFile)
                0665                     if sOldText != sNewText:
d775ee353a Oliv*0666                         if self.options.verbosity >= 1:
                0667                             if self.options.verbosity < 2:
                0668                                 self.prout(f"{verb} {sFile}", end="")
                0669                             self.prout("  (changed)")
                0670                             bNeedNewline = False
                0671                         if self.options.bReplace:
                0672                             self.replaceFile(sFile, sNewText)
                0673                         else:
                0674                             assert self.options.bCheck
                0675                             self.bCheckFailed = True
8fbfd1f382 Oliv*0676                 finally:
                0677                     # The try-finally block is so we can print a partial line
                0678                     # with the name of the file, and print (changed) on the
                0679                     # same line, but also make sure to break the line before
                0680                     # any traceback.
                0681                     if bNeedNewline:
                0682                         self.prout("")
                0683             else:
                0684                 self.processFile(sFile, self.stdout, sFile)
                0685         finally:
                0686             self.restoreIncludePath()
                0687
d775ee353a Oliv*0688     def processWildcards(self, sFile):
                0689         files = glob.glob(sFile)
                0690         if files:
                0691             for sMatchingFile in files:
                0692                 self.processOneFile(sMatchingFile)
                0693         else:
                0694             self.processOneFile(sFile)
                0695
8fbfd1f382 Oliv*0696     def processFileList(self, sFileList):
                0697         """ Process the files in a file list.
                0698         """
                0699         flist = self.openInputFile(sFileList)
                0700         lines = flist.readlines()
                0701         flist.close()
                0702         for l in lines:
                0703             # Use shlex to parse the line like a shell.
                0704             lex = shlex.shlex(l, posix=True)
                0705             lex.whitespace_split = True
                0706             lex.commenters = '#'
                0707             # No escapes, so that backslash can be part of the path
                0708             lex.escape = ''
                0709             args = list(lex)
                0710             if args:
                0711                 self.processArguments(args)
                0712
                0713     def processArguments(self, args):
                0714         """ Process one command-line.
                0715         """
                0716         saved_options = self.options
                0717         self.options = self.options.clone()
                0718
                0719         self.options.parseArgs(args[1:])
                0720         self.options.validate()
                0721
                0722         if args[0][0] == '@':
                0723             if self.options.sOutputName:
                0724                 raise CogUsageError("Can't use -o with @file")
                0725             self.processFileList(args[0][1:])
d775ee353a Oliv*0726         elif args[0][0] == '&':
                0727             if self.options.sOutputName:
                0728                 raise CogUsageError("Can't use -o with &file")
                0729             file_list = args[0][1:]
                0730             with change_dir(os.path.dirname(file_list)):
                0731                 self.processFileList(os.path.basename(file_list))
8fbfd1f382 Oliv*0732         else:
d775ee353a Oliv*0733             self.processWildcards(args[0])
8fbfd1f382 Oliv*0734
                0735         self.options = saved_options
                0736
                0737     def callableMain(self, argv):
                0738         """ All of command-line cog, but in a callable form.
                0739             This is used by main.
                0740             argv is the equivalent of sys.argv.
                0741         """
d775ee353a Oliv*0742         argv = argv[1:]
8fbfd1f382 Oliv*0743
                0744         # Provide help if asked for anywhere in the command line.
                0745         if '-?' in argv or '-h' in argv:
                0746             self.prerr(usage, end="")
                0747             return
                0748
                0749         self.options.parseArgs(argv)
                0750         self.options.validate()
d775ee353a Oliv*0751         self._fixEndOutputPatterns()
8fbfd1f382 Oliv*0752
                0753         if self.options.bShowVersion:
d775ee353a Oliv*0754             self.prout(f"Cog version {__version__}")
8fbfd1f382 Oliv*0755             return
                0756
                0757         if self.options.args:
                0758             for a in self.options.args:
                0759                 self.processArguments([a])
                0760         else:
                0761             raise CogUsageError("No files to process")
                0762
d775ee353a Oliv*0763         if self.bCheckFailed:
                0764             raise CogCheckFailed("Check failed")
                0765
8fbfd1f382 Oliv*0766     def main(self, argv):
                0767         """ Handle the command-line execution for cog.
                0768         """
                0769
                0770         try:
                0771             self.callableMain(argv)
                0772             return 0
                0773         except CogUsageError as err:
                0774             self.prerr(err)
d775ee353a Oliv*0775             self.prerr("(for help use -h)")
8fbfd1f382 Oliv*0776             return 2
                0777         except CogGeneratedError as err:
d775ee353a Oliv*0778             self.prerr(f"Error: {err}")
8fbfd1f382 Oliv*0779             return 3
d775ee353a Oliv*0780         except CogUserException as err:
                0781             self.prerr("Traceback (most recent call last):")
                0782             self.prerr(err.args[0])
                0783             return 4
                0784         except CogCheckFailed as err:
                0785             self.prerr(err)
                0786             return 5
8fbfd1f382 Oliv*0787         except CogError as err:
                0788             self.prerr(err)
                0789             return 1
                0790
d775ee353a Oliv*0791
                0792 def find_cog_source(frame_summary, prologue):
                0793     """Find cog source lines in a frame summary list, for printing tracebacks.
                0794
                0795     Arguments:
                0796         frame_summary: a list of 4-item tuples, as returned by traceback.extract_tb.
                0797         prologue: the text of the code prologue.
                0798
                0799     Returns
                0800         A list of 4-item tuples, updated to correct the cog entries.
                0801
                0802     """
                0803     prolines = prologue.splitlines()
                0804     for filename, lineno, funcname, source in frame_summary:
                0805         if not source:
                0806             m = re.search(r"^<cog ([^:]+):(\d+)>$", filename)
                0807             if m:
                0808                 if lineno <= len(prolines):
                0809                     filename = '<prologue>'
                0810                     source = prolines[lineno-1]
                0811                     lineno -= 1     # Because "import cog" is the first line in the prologue
                0812                 else:
                0813                     filename, coglineno = m.groups()
                0814                     coglineno = int(coglineno)
                0815                     lineno += coglineno - len(prolines)
                0816                     source = linecache.getline(filename, lineno).strip()
                0817         yield filename, lineno, funcname, source
                0818
                0819
                0820 def main():
                0821     """Main function for entry_points to use."""
                0822     return Cog().main(sys.argv)