... |
... |
@@ -22,18 +22,18 @@ CSS_STYLE = """ |
22
|
22
|
}
|
23
|
23
|
.col1 {
|
24
|
24
|
background-color: #eee;
|
25
|
|
- }
|
|
25
|
+ }
|
|
26
|
+
|
26
|
27
|
|
27
|
|
-
|
28
|
28
|
.highlight {
|
29
|
29
|
background-color: #0a0;
|
30
|
30
|
}
|
31
|
31
|
</style>
|
32
|
32
|
"""
|
33
|
33
|
OBJ_DIR = sys.argv[1]
|
34
|
|
-BASELINE_DIR = os.path.join(OBJ_DIR,"baseline")
|
35
|
|
-BENCHMARK_DIR = os.path.join(OBJ_DIR,"benchmark")
|
36
|
|
-BENCHMARK_HTML = os.path.join(OBJ_DIR,"benchmark.html")
|
|
34
|
+BASELINE_DIR = os.path.join(OBJ_DIR, "baseline")
|
|
35
|
+BENCHMARK_DIR = os.path.join(OBJ_DIR, "benchmark")
|
|
36
|
+BENCHMARK_HTML = os.path.join(OBJ_DIR, "benchmark.html")
|
37
|
37
|
|
38
|
38
|
FONT_COUNT = 5
|
39
|
39
|
|
... |
... |
@@ -41,11 +41,9 @@ WARNING_SAME_COMMIT = "Warning: Baseline and Benchmark have the same commit ID!" |
41
|
41
|
INFO_1 = "* Average time for single iteration. Smaller values are better."
|
42
|
42
|
INFO_2 = "* If a value in the 'Iterations' column is given as '<i>x | y</i>', values <i>x</i> and <i>y</i> give the number of iterations in the baseline and the benchmark test, respectively."
|
43
|
43
|
|
44
|
|
-
|
45
|
|
-
|
46
|
44
|
|
47
|
45
|
def main():
|
48
|
|
- """Entry point for theq script"""
|
|
46
|
+ """Entry point for theq script"""
|
49
|
47
|
with open(BENCHMARK_HTML, "w") as html_file:
|
50
|
48
|
write_to_html(html_file, "<html>\n<head>\n")
|
51
|
49
|
write_to_html(html_file, CSS_STYLE)
|
... |
... |
@@ -65,7 +63,7 @@ def main(): |
65
|
63
|
|
66
|
64
|
# Generate total results table
|
67
|
65
|
generate_total_results_table(html_file, BASELINE_DIR, BENCHMARK_DIR)
|
68
|
|
-
|
|
66
|
+
|
69
|
67
|
# Generate results tables
|
70
|
68
|
for filename in os.listdir(BASELINE_DIR):
|
71
|
69
|
if filename.endswith(".txt") and not filename == "info.txt":
|
... |
... |
@@ -75,11 +73,11 @@ def main(): |
75
|
73
|
generate_results_table(
|
76
|
74
|
html_file, baseline_results, benchmark_results, filename
|
77
|
75
|
)
|
78
|
|
-
|
79
|
|
-
|
|
76
|
+
|
80
|
77
|
write_to_html(html_file, "<center>Freetype Benchmark</center>\n")
|
81
|
78
|
write_to_html(html_file, "</body>\n</html>\n")
|
82
|
79
|
|
|
80
|
+
|
83
|
81
|
def write_to_html(html_file, content):
|
84
|
82
|
"""Write content to html file"""
|
85
|
83
|
html_file.write(content)
|
... |
... |
@@ -94,7 +92,7 @@ def read_file(file_path): |
94
|
92
|
def parse_info_file(info_file):
|
95
|
93
|
"""Get info from info.txt file and return as list"""
|
96
|
94
|
info = read_file(info_file)
|
97
|
|
- info[1] = '<a href="">"{}{}">{}</a>\n'.format(GITLAB_URL, info[1].strip(), info[1][:8])
|
|
95
|
+ info[1] = f'<a href="">"{GITLAB_URL}{info[1].strip()}">{info[1][:8]}</a>\n'
|
98
|
96
|
return info
|
99
|
97
|
|
100
|
98
|
|
... |
... |
@@ -111,92 +109,113 @@ def generate_info_table(html_file, baseline_info, benchmark_info): |
111
|
109
|
):
|
112
|
110
|
write_to_html(
|
113
|
111
|
html_file,
|
114
|
|
- '<tr><td class="col1">{}</td><td>{}</td><td>{}</td></tr>\n'.format(
|
115
|
|
- info, baseline_line.strip(), benchmark_line.strip()
|
116
|
|
- ),
|
|
112
|
+ f'<tr><td class="col1">{info}</td><td>{baseline_line.strip()}</td><td>{benchmark_line.strip()}</td></tr>\n'
|
117
|
113
|
)
|
118
|
114
|
write_to_html(html_file, "</table><br/>")
|
119
|
115
|
write_to_html(html_file, f"<p>{INFO_1}</p>")
|
120
|
116
|
write_to_html(html_file, f"<p>{INFO_2}</p>")
|
121
|
|
-
|
|
117
|
+
|
122
|
118
|
|
123
|
119
|
def generate_total_results_table(html_file, baseline_dir, benchmark_dir):
|
124
|
120
|
"""Prepare total results table for html"""
|
125
|
|
-
|
|
121
|
+
|
126
|
122
|
# This dictionary will store aggregated results.
|
127
|
|
- test_results = {test: {"baseline": 0, "benchmark": 0, "n_baseline": 0, "n_benchmark": 0} for test in [
|
128
|
|
- "Load", "Load_Advances (Normal)", "Load_Advances (Fast)", "Load_Advances (Unscaled)", "Render",
|
129
|
|
- "Get_Glyph", "Get_Char_Index", "Iterate CMap", "New_Face", "Embolden", "Stroke", "Get_BBox",
|
130
|
|
- "Get_CBox", "New_Face & load glyph(s)"
|
131
|
|
- ]}
|
132
|
|
-
|
|
123
|
+ test_results = {
|
|
124
|
+ test: {"baseline": 0, "benchmark": 0, "n_baseline": 0, "n_benchmark": 0}
|
|
125
|
+ for test in [
|
|
126
|
+ "Load",
|
|
127
|
+ "Load_Advances (Normal)",
|
|
128
|
+ "Load_Advances (Fast)",
|
|
129
|
+ "Load_Advances (Unscaled)",
|
|
130
|
+ "Render",
|
|
131
|
+ "Get_Glyph",
|
|
132
|
+ "Get_Char_Index",
|
|
133
|
+ "Iterate CMap",
|
|
134
|
+ "New_Face",
|
|
135
|
+ "Embolden",
|
|
136
|
+ "Stroke",
|
|
137
|
+ "Get_BBox",
|
|
138
|
+ "Get_CBox",
|
|
139
|
+ "New_Face & load glyph(s)",
|
|
140
|
+ ]
|
|
141
|
+ }
|
|
142
|
+
|
133
|
143
|
total_time = 0
|
134
|
|
-
|
|
144
|
+
|
135
|
145
|
for filename in os.listdir(baseline_dir):
|
136
|
146
|
if filename.endswith(".txt") and not filename == "info.txt":
|
137
|
|
-
|
138
|
147
|
baseline_results = read_file(os.path.join(baseline_dir, filename))
|
139
|
148
|
benchmark_results = read_file(os.path.join(benchmark_dir, filename))
|
140
|
|
-
|
141
|
|
- for baseline_line, benchmark_line in zip(baseline_results, benchmark_results):
|
142
|
|
-
|
|
149
|
+
|
|
150
|
+ for baseline_line, benchmark_line in zip(
|
|
151
|
+ baseline_results, benchmark_results
|
|
152
|
+ ):
|
143
|
153
|
if baseline_line.startswith("Total time:"):
|
144
|
154
|
baseline_match = re.match(r"Total time: (\d+)s", baseline_line)
|
145
|
155
|
benchmark_match = re.match(r"Total time: (\d+)s", benchmark_line)
|
146
|
|
-
|
|
156
|
+
|
147
|
157
|
if baseline_match and benchmark_match:
|
148
|
158
|
total_time += int(baseline_match.group(1))
|
149
|
159
|
total_time += int(benchmark_match.group(1))
|
150
|
160
|
|
151
|
|
-
|
152
|
161
|
if baseline_line.startswith(" "):
|
153
|
|
- baseline_match = re.match(r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", baseline_line)
|
154
|
|
- benchmark_match = re.match(r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", benchmark_line)
|
155
|
|
-
|
|
162
|
+ baseline_match = re.match(
|
|
163
|
+ r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", baseline_line
|
|
164
|
+ )
|
|
165
|
+ benchmark_match = re.match(
|
|
166
|
+ r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s",
|
|
167
|
+ benchmark_line,
|
|
168
|
+ )
|
|
169
|
+
|
156
|
170
|
if baseline_match and benchmark_match:
|
157
|
171
|
test = baseline_match.group(1).strip()
|
158
|
172
|
baseline_value = float(baseline_match.group(2))
|
159
|
173
|
benchmark_value = float(benchmark_match.group(2))
|
160
|
174
|
baseline_n = int(baseline_match.group(3))
|
161
|
175
|
benchmark_n = int(benchmark_match.group(3))
|
162
|
|
-
|
|
176
|
+
|
163
|
177
|
# Aggregate the results
|
164
|
178
|
if test in test_results:
|
165
|
179
|
test_results[test]["baseline"] += baseline_value
|
166
|
180
|
test_results[test]["benchmark"] += benchmark_value
|
167
|
181
|
test_results[test]["n_baseline"] += baseline_n
|
168
|
182
|
test_results[test]["n_benchmark"] += benchmark_n
|
169
|
|
-
|
170
|
|
-
|
171
|
|
-
|
|
183
|
+
|
172
|
184
|
# Writing to HTML
|
173
|
185
|
write_to_html(html_file, "<h2>Total Results</h2>\n")
|
174
|
186
|
write_to_html(html_file, '<table border="1">\n')
|
175
|
187
|
write_to_html(
|
176
|
188
|
html_file,
|
177
|
|
- '<tr><th>Test</th><th>Iterations</th><th>* Baseline (µs)</th>\
|
178
|
|
- <th>* Benchmark (µs)</th><th>Difference (%)</th></tr>\n'
|
|
189
|
+ "<tr><th>Test</th><th>Iterations</th><th>* Baseline (µs)</th>\
|
|
190
|
+ <th>* Benchmark (µs)</th><th>Difference (%)</th></tr>\n",
|
179
|
191
|
)
|
180
|
192
|
|
181
|
|
- total_baseline = total_benchmark = total_diff = total_n_baseline = total_n_benchmark = 0
|
182
|
|
-
|
|
193
|
+ total_baseline = total_benchmark = total_n_baseline = total_n_benchmark = 0
|
|
194
|
+
|
183
|
195
|
for test, values in test_results.items():
|
184
|
|
-
|
185
|
196
|
baseline = values["baseline"] / FONT_COUNT
|
186
|
197
|
benchmark = values["benchmark"] / FONT_COUNT
|
187
|
198
|
n_baseline = values["n_baseline"] / FONT_COUNT
|
188
|
199
|
n_benchmark = values["n_benchmark"] / FONT_COUNT
|
189
|
|
-
|
190
|
|
- n_display = f"{n_baseline:.0f} | {n_benchmark:.0f}" if n_baseline != n_benchmark else int(n_baseline)
|
191
|
|
-
|
192
|
|
- diff = ((baseline - benchmark) / baseline) * 100 if not (baseline - benchmark) == 0 else 0
|
|
200
|
+
|
|
201
|
+ n_display = (
|
|
202
|
+ f"{n_baseline:.0f} | {n_benchmark:.0f}"
|
|
203
|
+ if n_baseline != n_benchmark
|
|
204
|
+ else int(n_baseline)
|
|
205
|
+ )
|
|
206
|
+
|
|
207
|
+ diff = (
|
|
208
|
+ ((baseline - benchmark) / baseline) * 100
|
|
209
|
+ if not (baseline - benchmark) == 0
|
|
210
|
+ else 0
|
|
211
|
+ )
|
193
|
212
|
|
194
|
213
|
# Calculate for total row
|
195
|
214
|
total_baseline += baseline
|
196
|
215
|
total_benchmark += benchmark
|
197
|
216
|
total_n_baseline += n_baseline
|
198
|
217
|
total_n_benchmark += n_benchmark
|
199
|
|
-
|
|
218
|
+
|
200
|
219
|
# Check which value is smaller for color highlighting
|
201
|
220
|
baseline_color = "highlight" if baseline <= benchmark else ""
|
202
|
221
|
benchmark_color = "highlight" if benchmark <= baseline else ""
|
... |
... |
@@ -205,21 +224,15 @@ def generate_total_results_table(html_file, baseline_dir, benchmark_dir): |
205
|
224
|
html_file,
|
206
|
225
|
f'<tr><td class="col1">{test}</td><td>{n_display}</td>\
|
207
|
226
|
<td class="{baseline_color}">{baseline:.1f}</td>\
|
208
|
|
- <td class="{benchmark_color}">{benchmark:.1f}</td><td>{diff:.1f}</td></tr>\n'
|
|
227
|
+ <td class="{benchmark_color}">{benchmark:.1f}</td><td>{diff:.1f}</td></tr>\n',
|
209
|
228
|
)
|
210
|
|
-
|
211
|
|
-
|
212
|
229
|
|
213
|
|
- total_diff = ((total_baseline - total_benchmark) / total_baseline) * 100 if not (total_baseline - total_benchmark) == 0 else 0
|
214
|
|
- total_n_display = f"{total_n_baseline} | {total_n_benchmark}" if total_n_baseline != total_n_benchmark else str(total_n_baseline)
|
215
|
|
-
|
216
|
230
|
write_to_html(
|
217
|
231
|
html_file,
|
218
|
|
- f'<tr><td class="col1">Total duration for all tests:</td><td class="col1" colspan="4">{total_time:.0f} s</td>'
|
|
232
|
+ f'<tr><td class="col1">Total duration for all tests:</td><td class="col1" colspan="4">{total_time:.0f} s</td>',
|
219
|
233
|
)
|
220
|
|
-
|
221
|
|
- write_to_html(html_file,'</table>\n')
|
222
|
234
|
|
|
235
|
+ write_to_html(html_file, "</table>\n")
|
223
|
236
|
|
224
|
237
|
|
225
|
238
|
def generate_results_table(html_file, baseline_results, benchmark_results, filename):
|
... |
... |
@@ -230,32 +243,27 @@ def generate_results_table(html_file, baseline_results, benchmark_results, filen |
230
|
243
|
if line.startswith("ftbench results for font")
|
231
|
244
|
][0]
|
232
|
245
|
|
233
|
|
- write_to_html(html_file, "<h3>Results for {}</h2>\n".format(fontname))
|
|
246
|
+ write_to_html(html_file, f"<h3>Results for {fontname}</h2>\n")
|
234
|
247
|
write_to_html(html_file, '<table border="1">\n')
|
235
|
248
|
write_to_html(
|
236
|
249
|
html_file,
|
237
|
|
- '<tr><th>Test</th><th>Iterations</th>\
|
238
|
|
- <th>* <a href="">"{}.txt">Baseline</a> (µs)</th>\
|
239
|
|
- <th>* <a href="">"{}.txt">Benchmark</a> (µs)</th>\
|
240
|
|
- <th>Difference (%)</th></tr>\n'.format(
|
241
|
|
- os.path.join("./baseline/", filename[:-4]),
|
242
|
|
- os.path.join("./benchmark/", filename[:-4]),
|
243
|
|
- ),
|
|
250
|
+ f'<tr><th>Test</th><th>Iterations</th>\
|
|
251
|
+ <th>* <a href="">"{ os.path.join("./baseline/", filename[:-4])}.txt">Baseline</a> (µs)</th>\
|
|
252
|
+ <th>* <a href="">"{ os.path.join("./benchmark/", filename[:-4])}.txt">Benchmark</a> (µs)</th>\
|
|
253
|
+ <th>Difference (%)</th></tr>\n'
|
244
|
254
|
)
|
245
|
255
|
|
246
|
|
- total_n = total_difference = total_time = 0
|
|
256
|
+ total_n = total_time = 0
|
247
|
257
|
|
248
|
258
|
for baseline_line, benchmark_line in zip(baseline_results, benchmark_results):
|
249
|
|
-
|
250
|
259
|
if baseline_line.startswith("Total time:"):
|
251
|
260
|
baseline_match = re.match(r"Total time: (\d+)s", baseline_line)
|
252
|
261
|
benchmark_match = re.match(r"Total time: (\d+)s", benchmark_line)
|
253
|
|
-
|
|
262
|
+
|
254
|
263
|
if baseline_match and benchmark_match:
|
255
|
264
|
total_time += int(baseline_match.group(1))
|
256
|
265
|
total_time += int(benchmark_match.group(1))
|
257
|
266
|
|
258
|
|
-
|
259
|
267
|
if baseline_line.startswith(" "):
|
260
|
268
|
baseline_match = re.match(
|
261
|
269
|
r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", baseline_line
|
... |
... |
@@ -269,42 +277,42 @@ def generate_results_table(html_file, baseline_results, benchmark_results, filen |
269
|
277
|
benchmark_value = float(benchmark_match.group(2))
|
270
|
278
|
|
271
|
279
|
percentage_diff = (
|
272
|
|
- (baseline_value - benchmark_value) / baseline_value
|
273
|
|
- ) * 100 if not (baseline_value - benchmark_value) == 0 else 0
|
|
280
|
+ ((baseline_value - benchmark_value) / baseline_value) * 100
|
|
281
|
+ if not (baseline_value - benchmark_value) == 0
|
|
282
|
+ else 0
|
|
283
|
+ )
|
274
|
284
|
|
275
|
285
|
baseline_n = baseline_match.group(3)
|
276
|
286
|
benchmark_n = benchmark_match.group(3)
|
277
|
|
-
|
278
|
287
|
|
279
|
288
|
n = (
|
280
|
289
|
baseline_n
|
281
|
290
|
if baseline_n == benchmark_n
|
282
|
291
|
else baseline_n + " | " + benchmark_n
|
283
|
292
|
)
|
284
|
|
-
|
285
|
|
-
|
286
|
293
|
|
287
|
294
|
total_n += int(baseline_n)
|
288
|
295
|
total_n += int(benchmark_n)
|
289
|
|
-
|
290
|
|
-
|
|
296
|
+
|
291
|
297
|
# Check which value is smaller for color highlighting
|
292
|
|
- baseline_color = "highlight" if baseline_value <= benchmark_value else ""
|
293
|
|
- benchmark_color = "highlight" if benchmark_value <= baseline_value else ""
|
|
298
|
+ baseline_color = (
|
|
299
|
+ "highlight" if baseline_value <= benchmark_value else ""
|
|
300
|
+ )
|
|
301
|
+ benchmark_color = (
|
|
302
|
+ "highlight" if benchmark_value <= baseline_value else ""
|
|
303
|
+ )
|
294
|
304
|
|
295
|
|
-
|
296
|
305
|
write_to_html(
|
297
|
306
|
html_file,
|
298
|
307
|
f'<tr><td class="col1">{baseline_match.group(1)}</td><td>{n}</td>\
|
299
|
|
- <td class="{baseline_color}">{baseline_value:.1f}</td><td class="{benchmark_color}">{benchmark_value:.1f}</td><td>{percentage_diff:.1f}</td></tr>\n'
|
300
|
|
- )
|
301
|
|
-
|
|
308
|
+ <td class="{baseline_color}">{baseline_value:.1f}</td><td class="{benchmark_color}">{benchmark_value:.1f}</td><td>{percentage_diff:.1f}</td></tr>\n',
|
|
309
|
+ )
|
|
310
|
+
|
302
|
311
|
write_to_html(
|
303
|
312
|
html_file,
|
304
|
|
- f'<tr><td class="col1">Total duration for the font:</td><td class="col1" colspan="4">{total_time:.0f} s</td></table>\n'
|
|
313
|
+ f'<tr><td class="col1">Total duration for the font:</td><td class="col1" colspan="4">{total_time:.0f} s</td></table>\n',
|
305
|
314
|
)
|
306
|
315
|
|
307
|
|
-
|
308
|
316
|
|
309
|
317
|
if __name__ == "__main__":
|
310
|
318
|
main() |