468 lines
16 KiB
Python
468 lines
16 KiB
Python
#! py -3
|
|
"""
|
|
Append output and error files to Java files
|
|
"""
|
|
TODO = """
|
|
- Test to make sure that None files indeed have no output
|
|
- Collect all tests under single flag
|
|
"""
|
|
from pathlib import Path
|
|
import pprint
|
|
import textwrap
|
|
import os, sys, re
|
|
from itertools import chain
|
|
from sortedcontainers import SortedSet
|
|
from betools import CmdLine, visitDir, ruler, head
|
|
|
|
maxlinewidth = 59
|
|
examplePath = Path(r"C:\Users\Bruce\Dropbox\___OnJava\ExtractedExamples")
|
|
|
|
|
|
class JavaMain:
|
|
max_output_length = 32 # lines beyond which we flag this
|
|
maindef = re.compile("public\s+static\s+void\s+main")
|
|
leader = "_" * 8
|
|
ellipses = [leader + leader.join(["..."] * 4) + leader]
|
|
# ellipses = ["[...]".center(14, '_') * 4]
|
|
|
|
class JFile:
|
|
@staticmethod
|
|
def with_main(javaFilePath):
|
|
with javaFilePath.open() as doc:
|
|
code = doc.read()
|
|
if JavaMain.maindef.search(code) or "{Exec:" in code:
|
|
return JavaMain.JFile(javaFilePath, code)
|
|
return None
|
|
def __init__(self, javaFilePath, code):
|
|
self.javaFilePath = javaFilePath
|
|
self.code = code
|
|
self.lines = self.code.splitlines()
|
|
self.output_line = None
|
|
for line in self.lines:
|
|
if "} /* Output:" in line:
|
|
self.output_line = line
|
|
self.newcode = ""
|
|
|
|
@staticmethod
|
|
def create(javaFilePath):
|
|
j_file = JavaMain.JFile.with_main(javaFilePath)
|
|
if j_file is None:
|
|
return None
|
|
if "{ValidateByHand}" in j_file.code:
|
|
return None
|
|
if "/* Output: (None) */" in j_file.code:
|
|
return None
|
|
if "/* Output: (Execute to see)" in j_file.code:
|
|
return None
|
|
outfile = javaFilePath.with_name(javaFilePath.stem + "-output.txt")
|
|
errfile = javaFilePath.with_name(javaFilePath.stem + "-erroroutput.txt")
|
|
if outfile.exists() or errfile.exists():
|
|
return JavaMain(javaFilePath, j_file, outfile, errfile)
|
|
return None
|
|
|
|
def __init__(self, javaFilePath, j_file, outfile, errfile):
|
|
self.javaFilePath = javaFilePath
|
|
self.j_file = j_file
|
|
self.outfile = outfile
|
|
self.errfile = errfile
|
|
self.first_and_last = None
|
|
self.first_lines = None
|
|
self.long_output = False
|
|
|
|
ol = self.j_file.output_line
|
|
if ol:
|
|
if "(First and last" in ol:
|
|
self.first_and_last = int(ol.partition("(First and last")[2].split()[0])
|
|
elif "(First" in ol:
|
|
self.first_lines = int(ol.partition("(First")[2].split()[0])
|
|
|
|
result =""
|
|
if outfile.exists():
|
|
with outfile.open() as f:
|
|
out = f.read().strip()
|
|
if out:
|
|
if self.first_and_last:
|
|
lines = out.splitlines()
|
|
out = "\n".join(lines[:self.first_and_last] + JavaMain.ellipses + lines[-self.first_and_last:])
|
|
elif self.first_lines:
|
|
lines = out.splitlines()
|
|
out = "\n".join(lines[:self.first_lines] + [" " * 18 + "..."])
|
|
result += out + "\n"
|
|
if errfile.exists(): # Always include all of errfile
|
|
with errfile.open() as f:
|
|
err = f.read().strip()
|
|
if err:
|
|
result += "___[ Error Output ]___\n"
|
|
result += err
|
|
self.result = JavaMain.wrapOutput(result) + "\n"
|
|
|
|
if len(self.result.splitlines()) > JavaMain.max_output_length:
|
|
self.long_output = True
|
|
|
|
for line in self.j_file.lines:
|
|
if line.startswith("} ///:~"):
|
|
self.j_file.newcode += "} /* Output:\n"
|
|
self.j_file.newcode += self.result + "*///:~\n"
|
|
break
|
|
if line.startswith("} /* Output:"):
|
|
line = line.partition("*///:~")[0]
|
|
self.j_file.newcode += line + "\n"
|
|
self.j_file.newcode += self.result + "*///:~\n"
|
|
break
|
|
else:
|
|
self.j_file.newcode += line + "\n"
|
|
|
|
def new_code(self):
|
|
return self.j_file.newcode
|
|
|
|
@staticmethod
|
|
def wrapOutput(output):
|
|
"""
|
|
Wrap to line limit and perform other fixups for display and comparison
|
|
"""
|
|
output = output.replace('\0', "NUL")
|
|
lines = output.splitlines()
|
|
result = []
|
|
for line in lines:
|
|
result += textwrap.wrap(line.rstrip(), width=maxlinewidth)
|
|
return "\n".join(result)
|
|
|
|
def write_modified_file(self):
|
|
with self.j_file.javaFilePath.open('w') as modified:
|
|
modified.write(self.j_file.newcode)
|
|
|
|
|
|
@CmdLine('o')
|
|
def allOutputTagLines():
|
|
"""Shows all lines starting with } /*"""
|
|
allvariations = set()
|
|
os.chdir(str(examplePath))
|
|
for jfp in Path(".").rglob("*.java"):
|
|
with jfp.open() as code:
|
|
for line in code.readlines():
|
|
if line.startswith("} /*"):
|
|
allvariations.add(line)
|
|
pprint.pprint(allvariations)
|
|
|
|
|
|
@CmdLine('t')
|
|
def outputTagTypes():
|
|
"""Show different output tag variations"""
|
|
types = set()
|
|
os.chdir(str(examplePath))
|
|
for jfp in Path(".").rglob("*.java"):
|
|
jf = JavaMain.JFile.with_main(jfp)
|
|
if jf is None:
|
|
continue
|
|
if jf.output_line:
|
|
types.add(jf.output_line)
|
|
pprint.pprint(types)
|
|
|
|
|
|
@CmdLine('e')
|
|
def extractResults():
|
|
"""Test extraction of all results"""
|
|
os.chdir(str(examplePath))
|
|
with Path("AttachedResults.txt").open('w') as results:
|
|
for jfp in Path(".").rglob("*.java"):
|
|
j_main = JavaMain.create(jfp)
|
|
if j_main:
|
|
results.write(ruler(jfp))
|
|
outline = j_main.j_file.output_line
|
|
if outline:
|
|
results.write(outline + "\n")
|
|
results.write(j_main.result)
|
|
os.system("subl AttachedResults.txt")
|
|
|
|
#@CmdLine('n')
|
|
def noOutputFixup():
|
|
"""Attach "Output: (None)" lines to empty output files"""
|
|
os.chdir(str(examplePath))
|
|
# test = open("test.txt", 'w')
|
|
for jfp in Path(".").rglob("*.java"):
|
|
if "gui" in jfp.parts or "swt" in jfp.parts:
|
|
continue
|
|
jf = JavaMain.JFile.with_main(jfp)
|
|
if jf is None:
|
|
continue
|
|
if "{ValidateByHand}" in jf.code:
|
|
continue
|
|
if not jf.output_line:
|
|
if JavaMain.create(jfp):
|
|
continue
|
|
newcode = ""
|
|
for line in jf.lines:
|
|
if line.startswith("} ///:~"):
|
|
newcode += "} /* Output: (None) *///:~\n"
|
|
else:
|
|
newcode += line + "\n"
|
|
with jfp.open('w') as f:
|
|
f.write(newcode)
|
|
os.system("subl {}".format(jfp))
|
|
# test.write(ruler(jfp))
|
|
# test.write(newcode)
|
|
|
|
@CmdLine('v')
|
|
def viewAttachedFiles():
|
|
"""Sublime edit all files containing output in this directory and below"""
|
|
for java in Path(".").rglob("*.java"):
|
|
with java.open() as codefile:
|
|
code = codefile.read()
|
|
if "/* Output:" in code:
|
|
if "/* Output: (None)" in code:
|
|
continue
|
|
if "/* Output: (Execute to see)" in code:
|
|
continue
|
|
for n, line in enumerate(code.splitlines()):
|
|
if "/* Output:" in line:
|
|
# os.system("subl {}:{}".format(java, n))
|
|
os.system("subl {}".format(java))
|
|
continue
|
|
|
|
|
|
@CmdLine('x')
|
|
def showNulBytesInOutput():
|
|
"""Look for NUL bytes in output files`"""
|
|
os.chdir(str(examplePath))
|
|
for normal in Path(".").rglob("*-output.txt"):
|
|
with normal.open() as codeFile:
|
|
if "\0" in codeFile.read():
|
|
print(normal)
|
|
for errors in Path(".").rglob("*-erroroutput.txt"):
|
|
with errors.open() as codeFile:
|
|
if "\0" in codeFile.read():
|
|
print(normal)
|
|
|
|
|
|
@CmdLine('s')
|
|
def showJavaFiles():
|
|
"""Sublime edit all java files in this directory and below"""
|
|
for java in Path(".").rglob("*.java"):
|
|
os.system("subl {}".format(java))
|
|
|
|
|
|
# @CmdLine('w')
|
|
def boldWords():
|
|
"""
|
|
Create list of bolded words to be used as a Word dictionary
|
|
"""
|
|
from bs4 import BeautifulSoup
|
|
import codecs
|
|
import string
|
|
clean = lambda dirty: ''.join(filter(string.printable.__contains__, dirty))
|
|
def flense(word):
|
|
word = clean(word)
|
|
word = word.split('(')[0]
|
|
word = word.split('[')[0]
|
|
return word.strip()
|
|
|
|
os.chdir(str(examplePath / ".."))
|
|
spelldict = SortedSet()
|
|
with codecs.open(str(Path("OnJava.htm")),'r', encoding='utf-8', errors='ignore') as book:
|
|
soup = BeautifulSoup(book.read())
|
|
for b in soup.find_all("b"):
|
|
text = (" ".join(b.text.split())).strip()
|
|
if " " in text:
|
|
continue
|
|
text = flense(text)
|
|
if text:
|
|
spelldict.add(text)
|
|
|
|
with Path("BoldedWords.txt").open('w') as boldwords:
|
|
for word in spelldict:
|
|
if len(word):
|
|
if word[0] in string.ascii_letters:
|
|
boldwords.write(word + "\n")
|
|
|
|
|
|
@CmdLine('b')
|
|
def blankOutputFiles():
|
|
"""Show java files with expected output where there is none"""
|
|
find_output = re.compile(r"/\* Output:(.*)\*///:~", re.DOTALL)
|
|
os.chdir(str(examplePath))
|
|
for java in Path(".").rglob("*.java"):
|
|
with java.open() as codeFile:
|
|
output = find_output.search(codeFile.read())
|
|
if output:
|
|
# print(output.group(1))
|
|
if not output.group(1).strip():
|
|
print(java)
|
|
|
|
@CmdLine('u')
|
|
def unexpectedOutput():
|
|
"""Show java files with output where none was expected"""
|
|
os.chdir(str(examplePath))
|
|
for java in Path(".").rglob("*.java"):
|
|
with java.open() as codeFile:
|
|
if "/* Output: (None) */" in codeFile.read():
|
|
outfile = java.with_name(java.stem + "-output.txt")
|
|
errfile = java.with_name(java.stem + "-erroroutput.txt")
|
|
if outfile.exists():
|
|
if outfile.stat().st_size:
|
|
print("Unexpected output: {}".format(java))
|
|
if errfile.exists():
|
|
if errfile.stat().st_size:
|
|
print("Unexpected error output: {}".format(java))
|
|
|
|
exclude_files = [
|
|
r"concurrency\ActiveObjectDemo.java",
|
|
r"concurrency\AtomicityTest.java",
|
|
r"concurrency\CachedThreadPool.java",
|
|
r"concurrency\CountDownLatchDemo.java",
|
|
r"concurrency\DaemonFromFactory.java",
|
|
r"concurrency\DeadlockingDiningPhilosophers.java",
|
|
r"concurrency\FastSimulation.java",
|
|
r"concurrency\FixedDiningPhilosophers.java",
|
|
r"concurrency\FixedThreadPool.java",
|
|
r"concurrency\MoreBasicThreads.java",
|
|
r"concurrency\NIOInterruption.java",
|
|
r"concurrency\SelfManaged.java",
|
|
r"concurrency\SemaphoreDemo.java",
|
|
r"concurrency\SimpleDaemons.java",
|
|
r"concurrency\SleepingTask.java",
|
|
r"concurrency\ThreadLocalVariableHolder.java",
|
|
r"patterns\PaperScissorsRock.java",
|
|
r"patterns\recyclea\RecycleA.java",
|
|
r"patterns\visitor\BeeAndFlowers.java",
|
|
r"concurrency\EvenGenerator.java",
|
|
r"concurrency\GreenhouseScheduler.java",
|
|
r"concurrency\OrnamentalGarden.java",
|
|
r"concurrency\PipedIO.java",
|
|
r"concurrency\SimplePriorities.java",
|
|
r"concurrency\SimpleThread.java",
|
|
r"concurrency\ThreadVariations.java",
|
|
r"generics\DynamicProxyMixin.java",
|
|
r"logging\LoggingLevelManipulation.java",
|
|
r"logging\SimpleFilter.java",
|
|
r"annotations\AtUnitExample4.java",
|
|
r"concurrency\BankTellerSimulation.java",
|
|
r"concurrency\CarBuilder.java",
|
|
r"concurrency\ListComparisons.java",
|
|
r"concurrency\MapComparisons.java",
|
|
r"concurrency\ReaderWriterList.java",
|
|
r"concurrency\restaurant2\RestaurantWithQueues.java",
|
|
r"generics\Mixins.java",
|
|
r"io\LockingMappedFiles.java",
|
|
r"logging\ConfigureLogging.java",
|
|
r"logging\LoggingLevels.java",
|
|
r"operators\HelloDate.java",
|
|
r"annotations\AtUnitComposition.java",
|
|
r"annotations\AtUnitExample3.java",
|
|
r"annotations\AtUnitExternalTest.java",
|
|
r"annotations\HashSetTest.java",
|
|
r"annotations\UseCaseTracker.java",
|
|
r"concurrency\Interrupting.java",
|
|
r"concurrency\SerialNumberChecker.java",
|
|
r"concurrency\SimpleMicroBenchmark.java",
|
|
r"concurrency\SynchronizationComparisons.java",
|
|
r"concurrency\SyncObject.java",
|
|
r"containers\ListPerformance.java",
|
|
r"io\Logon.java",
|
|
r"logging\CustomHandler.java",
|
|
r"object\HelloDate.java",
|
|
r"annotations\AtUnitExample1.java",
|
|
r"annotations\AtUnitExample2.java",
|
|
r"concurrency\ExchangerDemo.java",
|
|
r"concurrency\ExplicitCriticalSection.java",
|
|
r"concurrency\Restaurant.java",
|
|
r"containers\MapPerformance.java",
|
|
r"containers\SetPerformance.java",
|
|
r"exceptions\LoggingExceptions.java",
|
|
r"logging\InfoLogging.java",
|
|
r"logging\InfoLogging2.java",
|
|
r"logging\LogToFile.java",
|
|
r"logging\LogToFile2.java",
|
|
r"logging\MultipleHandlers.java",
|
|
r"logging\MultipleHandlers2.java",
|
|
r"annotations\AtUnitExample5.java",
|
|
r"concurrency\Daemons.java",
|
|
r"concurrency\HorseRace.java",
|
|
r"concurrency\ToastOMatic.java",
|
|
r"enums\ConstantSpecificMethod.java",
|
|
r"exceptions\LoggingExceptions2.java",
|
|
r"io\MakeDirectories.java",
|
|
r"io\MappedIO.java",
|
|
r"io\PreferencesDemo.java",
|
|
r"logging\PrintableLogRecord.java",
|
|
r"references\Compete.java",
|
|
|
|
# Keep an eye on:
|
|
r"strings\JGrep.java",
|
|
]
|
|
|
|
@CmdLine('c')
|
|
def compare_output():
|
|
"""Compare attached and newly-generated output"""
|
|
TODO = """
|
|
- Could also compare number of lines
|
|
"""
|
|
ratio_target = 1.0
|
|
from difflib import SequenceMatcher
|
|
os.chdir(str(examplePath))
|
|
def generated_output(jfp):
|
|
j_main = JavaMain.create(jfp)
|
|
if not j_main:
|
|
return None
|
|
return j_main.result.strip()
|
|
def embedded_output(jfp):
|
|
find_output = re.compile(r"/\* (Output:.*)\*///:~", re.DOTALL)
|
|
with jfp.open() as java:
|
|
output = find_output.search(java.read())
|
|
assert output, "No embedded output: in {}".format(jfp)
|
|
return "\n".join(output.group(1).strip().splitlines()[1:])
|
|
if Path("CompareExclusions.txt").is_file():
|
|
Path("CompareExclusions.txt").unlink()
|
|
with Path("OutputComparisons.txt").open('w') as comparisions:
|
|
for jfp in Path(".").rglob("*.java"):
|
|
if "gui" in jfp.parts or "swt" in jfp.parts:
|
|
continue
|
|
if str(jfp) in exclude_files:
|
|
continue
|
|
generated = generated_output(jfp)
|
|
if generated is None:
|
|
continue
|
|
embedded = embedded_output(jfp)
|
|
comp = SequenceMatcher(None, embedded, generated)
|
|
ratio = comp.ratio()
|
|
if ratio < ratio_target:
|
|
print(jfp)
|
|
print("ratio: {}\n".format(ratio))
|
|
comparisions.write("\n" + ruler(jfp))
|
|
comparisions.write("ratio: {}\n".format(ratio))
|
|
comparisions.write(ruler("Attached"))
|
|
comparisions.write(embedded)
|
|
comparisions.write("\n" + ruler("Generated"))
|
|
comparisions.write(generated)
|
|
with Path("CompareExclusions.txt").open('a') as exclusions:
|
|
exclusions.write('r"' + str(jfp) + "\",\n")
|
|
if Path("CompareExclusions.txt").is_file():
|
|
os.system("ed CompareExclusions.txt")
|
|
if Path("OutputComparisons.txt").is_file():
|
|
os.system("ed OutputComparisons.txt")
|
|
|
|
|
|
|
|
@CmdLine('a')
|
|
def attachFiles():
|
|
"""Attach standard and error output to all files"""
|
|
os.chdir(str(examplePath))
|
|
test = open("AllFilesWithOutput.txt", 'w')
|
|
longOutput = open("LongOutput.txt", 'w')
|
|
for jfp in Path(".").rglob("*.java"):
|
|
if "gui" in jfp.parts or "swt" in jfp.parts:
|
|
continue
|
|
j_main = JavaMain.create(jfp)
|
|
if j_main is None:
|
|
continue
|
|
j_main.write_modified_file()
|
|
test.write(ruler())
|
|
test.write(j_main.new_code())
|
|
if j_main.long_output:
|
|
longOutput.write(ruler())
|
|
longOutput.write(j_main.new_code())
|
|
# os.system("subl AllFilesWithOutput.txt")
|
|
# os.system("subl LongOutput.txt")
|
|
|
|
if __name__ == '__main__': CmdLine.run()
|
|
|