Commit 0cb1f008 authored by Ludwig Fueracker's avatar Ludwig Fueracker
Browse files

refactored, improved mem usage

parent 7899e778
Pipeline #2611 passed with stage
in 30 seconds
......@@ -43,6 +43,8 @@ def validateRecords():
def validateHashes(recordList):
#TODO: also sort this, perhaps execute on fullRecords or check against sorted commitIDs
#also TODO: collect data from all files before validating (changing paths src and crt)
#TODO: validate return-code!
global errorCount, astDifferObjSameCount, missingCount
#TODO: this method assumes that all records are from the same object file
iterRecords = iter(recordList)
......@@ -125,17 +127,16 @@ def buildFullRecordTo(pathToFullRecordFile):
# print "read full record from " + pathToFullRecordFile
#else:
fullRecord = buildFullRecord()
f = open(pathToFullRecordFile, 'w')
try:
f.write(repr(fullRecord) + "\n")
except MemoryError as me:
print me
raise
finally:
print time.ctime()
f.close()
print "built full record, wrote to " + pathToFullRecordFile
# f = open(pathToFullRecordFile, 'w')
# try:
# f.write(repr(fullRecord) + "\n")
# except MemoryError as me:
# print me
# raise
# finally:
# print time.ctime()
# f.close()
# print "built full record, wrote to " + pathToFullRecordFile
return fullRecord
......@@ -165,6 +166,14 @@ def buildFullRecord():
objFilename = data['obj-file']
del data['obj-file']
# del everything I don't need
del data['return-code']
del data['element-hashes']
del data['project']
del data['processed-bytes']
del data['object-file-size']
dataNewKeys = {tr(k): v for k, v in data.items()}
fullRecord[commitID][tr('files')][objFilename] = dataNewKeys
......@@ -178,6 +187,40 @@ def getSortedCommitIDList(fullRecord):
################################################################################
def plotBuildTimeGraph(measuredBuildTimes, realClangHashBuildTimes, optimalClangHashBuildTimes, optimalBuildTimes): # times in ms
fig, ax = plt.subplots()
ax.plot(measuredBuildTimes, label='measured build time')
ax.plot(realClangHashBuildTimes, label='real clang-hash build time')
ax.plot(optimalClangHashBuildTimes, label='optimal clang-hash build time')
ax.plot(optimalBuildTimes, label='optimal build time')
lgd = ax.legend(loc='center left', bbox_to_anchor=(1, 0.5)) # legend on the right
plt.xlabel('commits')
plt.ylabel('time [ms]')
fig.savefig(pathToRecords + '/../buildTimes.png', bbox_extra_artists=(lgd,), bbox_inches='tight')
def plotBuildTimeCompositionGraph(parseTimes, hashTimes, compileTimes, diffToBuildTime): # times in s
fig, ax = plt.subplots()
ax.stackplot(np.arange(1, len(parseTimes)+1), # x axis
[parseTimes, hashTimes, compileTimes, diffToBuildTime],
colors=['#008800','#FF0000','#0000FF', '#000000'])
plt.xlim(1,len(parseTimes))
plt.xlabel('commits')
plt.ylabel('time [s]')
lgd = ax.legend([mpatches.Patch(color='#000000'),
mpatches.Patch(color='#0000FF'),
mpatches.Patch(color='#FF0000'),
mpatches.Patch(color='#008800')],
['remaining build time','compile time', 'hash time', 'parse time'],
loc='center left', bbox_to_anchor=(1, 0.5))
fig.savefig(pathToRecords + '/../buildTimeComposition.png', bbox_extra_artists=(lgd,), bbox_inches='tight')
def makeBuildTimeGraph(fullRecord):
sortedCommitIDs = getSortedCommitIDList(fullRecord)
iterCommits = iter(sortedCommitIDs)
......@@ -250,36 +293,9 @@ def makeBuildTimeGraph(fullRecord):
prevCommit = currentCommit
# plot build time graph
fig, ax = plt.subplots()
ax.plot(measuredBuildTimes, label='measured build time')
ax.plot(realClangHashBuildTimes, label='real clang-hash build time')
ax.plot(optimalClangHashBuildTimes, label='optimal clang-hash build time')
ax.plot(optimalBuildTimes, label='optimal build time')
lgd = ax.legend(loc='center left', bbox_to_anchor=(1, 0.5)) # legend on the right
plt.xlabel('commits')
plt.ylabel('time in ms')
fig.savefig(pathToRecords + '/../buildTimes.png', bbox_extra_artists=(lgd,), bbox_inches='tight')
plotBuildTimeGraph(measuredBuildTimes, realClangHashBuildTimes, optimalClangHashBuildTimes, optimalBuildTimes)
# plot build time graph
x = np.arange(1, len(parseTimes)+1)
fig, ax = plt.subplots()
ax.stackplot(x, [parseTimes, hashTimes, compileTimes, diffToBuildTime], colors=['#008800','#FF0000','#0000FF', '#000000'])
plt.xlim(1,len(parseTimes))
plt.xlabel('commits')
plt.ylabel('time in s')
lgd = ax.legend([mpatches.Patch(color='#000000'),
mpatches.Patch(color='#0000FF'),
mpatches.Patch(color='#FF0000'),
mpatches.Patch(color='#008800')],
['remaining build time','compile time', 'hash time', 'parse time'],
loc='center left', bbox_to_anchor=(1, 0.5))
fig.savefig(pathToRecords + '/../buildTimeComposition.png', bbox_extra_artists=(lgd,), bbox_inches='tight')
plotBuildTimeCompositionGraph(parseTimes, hashTimes, compileTimes, diffToBuildTime)
################################################################################
......@@ -341,7 +357,7 @@ def makeTimeHistograms(fullRecord):
width = 0.7 * (bins[1] - bins[0])
center = (bins[:-1] + bins[1:]) / 2
fig, ax = plt.subplots()
plt.xlabel('time')
plt.xlabel('time [ms]')
plt.ylabel('#files')
ax.bar(center, hist, align='center', width=width)
fig.savefig(pathToRecords + '/../parseTimeHistogram.png')#, bbox_extra_artists=(lgd,), bbox_inche s='tight')
......@@ -351,7 +367,7 @@ def makeTimeHistograms(fullRecord):
width = 0.7 * (bins[1] - bins[0])
center = (bins[:-1] + bins[1:]) / 2
fig, ax = plt.subplots()
plt.xlabel('time')
plt.xlabel('time [ms]')
plt.ylabel('#files')
ax.bar(center, hist, align='center', width=width)
fig.savefig(pathToRecords + '/../hashTimeHistogram.png')#, bbox_extra_artists=(lgd,), bbox_inche s='tight')
......@@ -361,7 +377,7 @@ def makeTimeHistograms(fullRecord):
width = 0.7 * (bins[1] - bins[0])
center = (bins[:-1] + bins[1:]) / 2
fig, ax = plt.subplots()
plt.xlabel('time')
plt.xlabel('time [ms]')
plt.ylabel('#files')
ax.bar(center, hist, align='center', width=width)
fig.savefig(pathToRecords + '/../compileTimeHistogram.png')#, bbox_extra_artists=(lgd,), bbox_inche s='tight')
......@@ -401,6 +417,9 @@ def makeChangesGraph(fullRecord):
if 'src/' + filename in prevFiles:
print "file %s changed place to src/" % filename
prevRecord = prevFiles['src/' + filename]
elif 'crt/' + filename in prevFiles:
print "file %s changed place to crt/" % filename
prevRecord = prevFiles['crt/' + filename]
else:
print "ERROR, MISSING FILE: %s not in prev, continue with next commit" % filename
continue
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment