renamed stuff, added outputs

evaluate_data_with_stop.py

@@ -16,36 +16,31 @@ PATH_TO_RECORDS = '' # gets set from command line parameter
 
 # data/record filenames
 INFO_EXTENSION = '.info'
-FULL_RECORD_FILENAME = 'fullRecord' + INFO_EXTENSION
+FULL_RECORD_FILENAME = 'full_record' + INFO_EXTENSION
 COMMIT_INFO_FILENAME = 'buildInfo_musl_with_stop' + INFO_EXTENSION
-BUILD_TIME_DATA_FILENAME = 'totalBuildTimes.csv'
-BUILD_TIME_FILENAME = 'totalBuildTimes.pdf'
-BUILD_TIME_DATA_HEADER = ['totalParseTimes', 'totalHashTimes', 'totalCompileTimes', 'diffToBuildTimes', 'totalBuildTimes']
+BUILD_TIME_DATA_FILENAME = 'total_build_times.csv'
+BUILD_TIME_FILENAME = 'total_build_times.pdf'
+BUILD_TIME_DATA_HEADER = ['total_parse_times', 'total_hash_times', 'total_compile_times', 'diff_to_build_times', 'total_build_times']
 
 
 def abs_path(filename):
-    """Prepends the absolute path to the filename.
-    """
+    """Prepends the absolute path to the filename."""
     return PATH_TO_RECORDS + '/../' + filename
 
 
-def getListOfFiles(directory):
+def get_list_of_files(directory):
     for root, dirnames, filenames in os.walk(directory):
         for filename in fnmatch.filter(filenames, '*' + INFO_EXTENSION):
             yield os.path.join(root, filename)
 
 
-errorCount = 0
-astDifferObjSameCount = 0
-missingCount = 0    
-
 ################################################################################
 #
 #
 ################################################################################
 
-
-keyTranslationToNr = {
+def build_key_translation_dict():
+    key_translation_to_nr = {
         'start-time':       0,
         'hash-start-time':  1,
         'object-hash':      2,
@@ -67,61 +62,65 @@ keyTranslationToNr = {
         'insertions':       18,
         'deletions':        19,
         'run_id':           20
-}
-keyTranslationFromNr = {v: k for k, v in keyTranslationToNr.items()}
+    }
+    key_translation_from_nr = {v: k for k, v in key_translation_to_nr.items()}
+
+    key_translation_dict = key_translation_to_nr.copy()
+    key_translation_dict.update(key_translation_from_nr)
 
-keyTranslation = keyTranslationToNr.copy()
-keyTranslation.update(keyTranslationFromNr)
+    return key_translation_dict
 
 
+key_translation = build_key_translation_dict()
+
 def tr(key):
     """lookup key translation (both directions)"""
-    return keyTranslation[key]
+    return key_translation[key]
 
 
-def buildFullRecordTo(pathToFullRecordFile):
+def build_full_record_to(path_to_full_record_file):
     """structure of full record:
     {commitID: {'build-time': time, files: {filename: {record}, filename: {record}}}}
     """
-    fullRecord = buildFullRecord()   
+    full_record = build_full_record()
     if DO_PRINT_RECORDS:
-        f = open(pathToFullRecordFile, 'w')
+        f = open(path_to_full_record_file, 'w')
         try:
-            f.write(repr(fullRecord) + "\n")
+            f.write(repr(full_record) + "\n")
         except MemoryError as me:
             print me
             raise
         finally:
             print time.ctime()
             f.close()
-        print "built full record, wrote to " + pathToFullRecordFile
+        print "built full record, wrote to " + path_to_full_record_file
 
-    return fullRecord
+    return full_record
 
 
-def buildFullRecord():
+def build_full_record():
     """Builds a complete record from all the single hash records.
     The records are grouped by the commitIDs
     """
-    fullRecord = {}
-    with open(abs_path(COMMIT_INFO_FILENAME), 'r') as commitInfoFile:
-        commitInfo = eval(commitInfoFile.read())
-        for run_id in commitInfo:
+    full_record = {}
+    with open(abs_path(COMMIT_INFO_FILENAME), 'r') as commit_infoFile:
+        commit_info = eval(commit_infoFile.read())
+        for run_id in commit_info:
             if not isinstance(run_id, int): # dict also contains key 'commit-hash'
                 continue;
-            currentRecord = {}
-            currentRecord[tr('filename')] = commitInfo[run_id]['filename']
-            currentRecord[tr('build-time')] = commitInfo[run_id]['build-time']
-            currentRecord[tr('files')] = {}
-            fullRecord[run_id] = currentRecord 
-            
-    for recordFilename in getListOfFiles(PATH_TO_RECORDS):
-        for line in open(recordFilename):
+            current_record = {}
+            current_record[tr('filename')] = commit_info[run_id]['filename']
+            current_record[tr('build-time')] = commit_info[run_id]['build-time']
+            current_record[tr('files')] = {}
+            full_record[run_id] = current_record
+            
+    for record_filename in get_list_of_files(PATH_TO_RECORDS):
+        for line in open(record_filename):
             data = eval(line)
 #            commitID = data['commit-hash']
 #            del data['commit-hash']
             
-            objFilename = data['obj-file']
+            obj_filename = data['obj-file']
             del data['obj-file']
 
             
@@ -134,115 +133,124 @@ def buildFullRecord():
             
             run_id = data['run_id']
 
-            dataNewKeys = {tr(k): v for k, v in data.items()} 
-            fullRecord[run_id][tr('files')][objFilename] = dataNewKeys
+            data_new_keys = {tr(k): v for k, v in data.items()}
+            full_record[run_id][tr('files')][obj_filename] = data_new_keys
 
-    return fullRecord
+    return full_record
 
 
 ################################################################################
 
-def write_to_csv(data, columnNames, filename):
-    with open(filename, "w") as csvFile:
-        writer = csv.writer(csvFile)
-        writer.writerow(columnNames)
+def write_to_csv(data, column_names, filename):
+    with open(filename, "w") as csv_file:
+        writer = csv.writer(csv_file)
+        writer.writerow(column_names)
         for line in data:
             writer.writerow(line)
 
 
-def printAvg(data, name):
+def print_avg(data, name):
     print 'avg %s: %f' % (name, sum(data)/float(len(data)))
 
 ################################################################################
 
-parseColor, hashColor, compileColor, remainColor = ('#FFFF66','#FF0000','#3399FF','#008800')
+parse_color, hash_color, compile_color, remain_color = ('#FFFF66','#FF0000','#3399FF','#008800')
 
 def plot_build_time_graph1(data):
-    plotBuildTimeCompositionGraph(data[0], data[1], data[2], data[3])
+    plot_build_time_composition_graph(data[0], data[1], data[2], data[3])
 
 
-def plotBuildTimeCompositionGraph(parseTimes, hashTimes, compileTimes, diffToBuildTime): # times in ms
+def plot_build_time_composition_graph(parse_times, hash_times, compile_times, diff_to_build_time): # times in ms
     fig, ax = plt.subplots()
 
-    ax.stackplot(np.arange(1, len(parseTimes)+1), # x axis
-                 [parseTimes, hashTimes, compileTimes,
-                #diffToBuildTime
-                ], colors=[parseColor,hashColor,compileColor,
-                 #   remainColor
+    ax.stackplot(np.arange(1, len(parse_times)+1), # x axis
+                 [parse_times, hash_times, compile_times,
+                #diff_to_build_time
+                ], colors=[parse_color,hash_color,compile_color,
+                 #   remain_color
                 ], edgecolor='none')
-    plt.xlim(1,len(parseTimes))
+    plt.xlim(1,len(parse_times))
     plt.xlabel('commits')
     plt.ylabel('time [s]')
     ax.set_yscale('log')
-    lgd = ax.legend([#mpatches.Patch(color=remainColor),
-                     mpatches.Patch(color=compileColor),
-                     mpatches.Patch(color=hashColor),
-                     mpatches.Patch(color=parseColor)],
+    lgd = ax.legend([#mpatches.Patch(color=remain_color),
+                     mpatches.Patch(color=compile_color),
+                     mpatches.Patch(color=hash_color),
+                     mpatches.Patch(color=parse_color)],
                     [#'remaining build time',
                     'compile time', 'hash time', 'parse time'],
                     loc='center left', bbox_to_anchor=(1, 0.5))
     fig.savefig(abs_path(BUILD_TIME_FILENAME), bbox_extra_artists=(lgd,), bbox_inches='tight')
 
-    printAvg(parseTimes, 'parse')
-    printAvg(hashTimes, 'hash')
-    printAvg(compileTimes, 'compile')
-    printAvg(diffToBuildTime, 'remainder')
+    print "\n-----------------"
+    print "average total times per build:"
+    print_avg(parse_times, 'parse')
+    print_avg(hash_times, 'hash')
+    print_avg(compile_times, 'compile')
+    print_avg(diff_to_build_time, 'remainder')
+    print ""
+    print "average times if header/source file touched"
+    print "-----------------\n"
 
 
 
 ################################################################################
 
-def makeGraphs(fullRecord):
+def make_graphs(full_record):
     # data for build time graphs
-    totalParseTimes = []
-    totalHashTimes = []
-    totalCompileTimes = []
-    totalBuildTimes = []
-    diffToBuildTimes = []
-
-#    freshBuildRecord = fullRecord[0]
-    for run_id in fullRecord:
+    total_parse_times = []
+    total_hash_times = []
+    total_compile_times = []
+    total_build_times = []
+    diff_to_build_times = []
+
+    parse_times_header_touched = []
+    parse_times_source_touched = []
+
+
+#    freshBuildRecord = full_record[0]
+    for run_id in full_record:
         if run_id < 2: # skip fresh build (and also 1st, seems to be buggy...)
             continue
     
-        currentRecord = fullRecord[run_id]
-        currentFiles = currentRecord[tr('files')]
-        filesChanged = len(currentFiles) # count changed files per run #TODO!
+        current_record = full_record[run_id]
+        current_files = current_record[tr('files')]
+        files_changed = len(current_files) # count changed files per run #TODO!
 
-        print currentRecord[tr('filename')]
+        print current_record[tr('filename')]
 
-        totalParseDuration = 0
-        totalHashDuration = 0
-        totalCompileDuration = 0
-        for filename in currentFiles: # deal with first commit
-#            if tr('ast-hash') not in currentFiles[filename].keys():
+        total_parse_duration = 0
+        total_hash_duration = 0
+        total_compile_duration = 0
+        for filename in current_files: # deal with first commit
+#            if tr('ast-hash') not in current_files[filename].keys():
 #                print "error: missing AST hash for file %s" % filename
 #                continue
-            currentFileRecord = currentFiles[filename]
-            totalParseDuration += currentFileRecord[tr('parse-duration')]
-            totalHashDuration += currentFileRecord[tr('hash-duration')] 
-            totalCompileDuration += currentFileRecord[tr('compile-duration')]
+            current_file_record = current_files[filename]
+            total_parse_duration += current_file_record[tr('parse-duration')]
+            total_hash_duration += current_file_record[tr('hash-duration')]
+            total_compile_duration += current_file_record[tr('compile-duration')]
      
-#        if totalParseDuration == 0:# or (totalCompileDuration/1e6) > 500000:
+#        if total_parse_duration == 0:# or (total_compile_duration/1e6) > 500000:
 #            continue
   
-        totalParseTimes.append(totalParseDuration / 1e6) # nano to milli
-        totalHashTimes.append(totalHashDuration / 1e6)
-        totalCompileTimes.append(totalCompileDuration / 1e6)
-        buildTime = currentRecord[tr('build-time')]
-        totalBuildTimes.append(buildTime / 1e6)
-        diffToBuildTimes.append((buildTime - totalParseDuration - totalHashDuration - totalCompileDuration) / 1e6)
+        total_parse_times.append(total_parse_duration / 1e6) # nano to milli
+        total_hash_times.append(total_hash_duration / 1e6)
+        total_compile_times.append(total_compile_duration / 1e6)
+        build_time = current_record[tr('build-time')]
+        total_build_times.append(build_time / 1e6)
+        diff_to_build_times.append((build_time - total_parse_duration - total_hash_duration - total_compile_duration) / 1e6)
 
  
-        print 'run_id %d, #filesChanged: %d' % (run_id, filesChanged)
+        print 'run_id %d, #files_changed: %d' % (run_id, files_changed)
 
-    printAvg(totalBuildTimes, 'total')
+    print_avg(total_build_times, 'total')
 
     # save data to csv files
-    buildTimeData = np.column_stack((totalParseTimes, totalHashTimes, totalCompileTimes, diffToBuildTimes, totalBuildTimes))
-    write_to_csv(buildTimeData, BUILD_TIME_DATA_HEADER, abs_path(BUILD_TIME_DATA_FILENAME))
+    build_time_data = np.column_stack((total_parse_times, total_hash_times, total_compile_times, diff_to_build_times, total_build_times))
+    write_to_csv(build_time_data, BUILD_TIME_DATA_HEADER, abs_path(BUILD_TIME_DATA_FILENAME))
 
-    plotBuildTimeCompositionGraph(totalParseTimes, totalHashTimes, totalCompileTimes, diffToBuildTimes)
+    plot_build_time_composition_graph(total_parse_times, total_hash_times, total_compile_times, diff_to_build_times)
     
 ################################################################################
 """functions for reading data from the csv files to skip full record building"""
@@ -250,7 +258,7 @@ def makeGraphs(fullRecord):
 def csv_files_are_existing():
     return os.path.isfile(abs_path(BUILD_TIME_DATA_FILENAME))
 
-def read_from_csv(filename, columnNames):
+def read_from_csv(filename, column_names):
     data = []
     with open(filename) as csv_file:
         reader = csv.reader(csv_file)
@@ -291,10 +299,10 @@ if (len(sys.argv) > 1):
 #        read_csv_data_and_plot_graphs()
 #        print "finished graphs at %s" % time.ctime()
 #    else:
-    full_record = buildFullRecordTo(path_to_full_record_file)
+    full_record = build_full_record_to(path_to_full_record_file)
     print "finished building/loading full record at %s" % time.ctime()
 
-    makeGraphs(full_record)
+    make_graphs(full_record)
     print "finished graphs at %s" % time.ctime()
 
     print "Finished at %s" % time.ctime()