From 09535adfe5f09ccee5af540bf438d50a7bfcda56 Mon Sep 17 00:00:00 2001 From: Linchenn <40653845+Linchenn@users.noreply.github.com> Date: Wed, 4 Jan 2023 13:33:52 -0800 Subject: [PATCH] Add averageTimeExclFirst to benchmark_util.timeInference (#7231) FEATURE --- e2e/benchmarks/benchmark_util.js | 43 ++++--- .../browserstack-benchmark/app_node_test.js | 28 +++-- .../benchmark_models.js | 12 +- .../benchmark_results.html | 2 +- .../browserstack-benchmark/index.js | 112 +++++++++--------- 5 files changed, 104 insertions(+), 93 deletions(-) diff --git a/e2e/benchmarks/benchmark_util.js b/e2e/benchmarks/benchmark_util.js index 12d44db9e..ce89923cb 100644 --- a/e2e/benchmarks/benchmark_util.js +++ b/e2e/benchmarks/benchmark_util.js @@ -109,8 +109,8 @@ function generateInputFromDef(inputDefs, isForGraphModel = false) { generatedRaw.dispose(); } else { throw new Error( - `The ${inputDef.dtype} dtype of '${inputDef.name}' input ` + - `at model.inputs[${inputDefIndex}] is not supported.`); + `The ${inputDef.dtype} dtype of '${inputDef.name}' input ` + + `at model.inputs[${inputDefIndex}] is not supported.`); } tensorArray.push(inputTensor); }); @@ -162,8 +162,8 @@ function getPredictFnForModel(model, input) { predict = () => model.predict(input); } else { throw new Error( - 'Predict function was not found. Please provide a tf.GraphModel or ' + - 'tf.LayersModel'); + 'Predict function was not found. Please provide a tf.GraphModel or ' + + 'tf.LayersModel'); } return predict; } @@ -175,6 +175,8 @@ function getPredictFnForModel(model, input) { * about the model's inference time: * - `times`: an array of inference time for each inference * - `averageTime`: the average time of all inferences + * - `averageTimeExclFirst`: the average time of all inferences except the + * first. * - `minTime`: the minimum time of all inferences * - `maxTime`: the maximum time of all inferences * @@ -211,6 +213,8 @@ async function timeModelInference(model, input, numRuns = 1) { * time: * - `times`: an array of inference time for each inference * - `averageTime`: the average time of all inferences + * - `averageTimeExclFirst`: the average time of all inferences except the + * first. * - `minTime`: the minimum time of all inferences * - `maxTime`: the maximum time of all inferences * @@ -237,8 +241,8 @@ async function timeModelInference(model, input, numRuns = 1) { async function timeInference(predict, numRuns = 1) { if (typeof predict !== 'function') { throw new Error( - 'The first parameter should be a function, while ' + - `a(n) ${typeof predict} is found.`); + 'The first parameter should be a function, while ' + + `a(n) ${typeof predict} is found.`); } const times = []; @@ -254,11 +258,15 @@ async function timeInference(predict, numRuns = 1) { } const averageTime = times.reduce((acc, curr) => acc + curr, 0) / times.length; + const averageTimeExclFirst = times.length > 1 ? + times.slice(1).reduce((acc, curr) => acc + curr, 0) / (times.length - 1) : + 'NA'; const minTime = Math.min(...times); const maxTime = Math.max(...times); const timeInfo = { times, averageTime, + averageTimeExclFirst, minTime, maxTime @@ -352,9 +360,9 @@ async function downloadValuesFromTensorContainer(tensorContainer) { * @param numProfiles The number of rounds for profiling the inference process. */ async function profileModelInference( - model, input, isTflite = false, numProfiles = 1) { + model, input, isTflite = false, numProfiles = 1) { const predict = isTflite ? () => tfliteModel.predict(input) : - getPredictFnForModel(model, input); + getPredictFnForModel(model, input); return profileInference(predict, isTflite, numProfiles); } @@ -392,8 +400,8 @@ async function profileModelInference( async function profileInference(predict, isTflite = false, numProfiles = 1) { if (typeof predict !== 'function') { throw new Error( - 'The first parameter should be a function, while ' + - `a(n) ${typeof predict} is found.`); + 'The first parameter should be a function, while ' + + `a(n) ${typeof predict} is found.`); } let kernelInfo = {}; @@ -431,7 +439,7 @@ async function profileInference(predict, isTflite = false, numProfiles = 1) { kernelInfo.kernels[i].kernelTimeMs = totalTimeMs / kernelInfos.length; } kernelInfo.kernels = - kernelInfo.kernels.sort((a, b) => b.kernelTimeMs - a.kernelTimeMs); + kernelInfo.kernels.sort((a, b) => b.kernelTimeMs - a.kernelTimeMs); kernelInfo.aggregatedKernels = aggregateKernelTime(kernelInfo.kernels); return kernelInfo; } @@ -451,13 +459,13 @@ function aggregateKernelTime(kernels) { aggregatedKernelTime[kernel.name] = kernel.kernelTimeMs; } else { aggregatedKernelTime[kernel.name] = - oldAggregatedKernelTime + kernel.kernelTimeMs; + oldAggregatedKernelTime + kernel.kernelTimeMs; } }); return Object.entries(aggregatedKernelTime) - .map(([name, timeMs]) => ({name, timeMs})) - .sort((a, b) => b.timeMs - a.timeMs); + .map(([name, timeMs]) => ({ name, timeMs })) + .sort((a, b) => b.timeMs - a.timeMs); } /** @@ -512,7 +520,7 @@ async function setEnvFlags(flagConfig) { return true; } else if (typeof flagConfig !== 'object') { throw new Error( - `An object is expected, while a(n) ${typeof flagConfig} is found.`); + `An object is expected, while a(n) ${typeof flagConfig} is found.`); } // Check the validation of flags and values. @@ -523,9 +531,8 @@ async function setEnvFlags(flagConfig) { } if (TUNABLE_FLAG_VALUE_RANGE_MAP[flag].indexOf(flagConfig[flag]) === -1) { throw new Error( - `${flag} value is expected to be in the range [${ - TUNABLE_FLAG_VALUE_RANGE_MAP[flag]}], while ${flagConfig[flag]}` + - ' is found.'); + `${flag} value is expected to be in the range [${TUNABLE_FLAG_VALUE_RANGE_MAP[flag]}], while ${flagConfig[flag]}` + + ' is found.'); } } diff --git a/e2e/benchmarks/browserstack-benchmark/app_node_test.js b/e2e/benchmarks/browserstack-benchmark/app_node_test.js index 61d97985c..a8abc7d4d 100644 --- a/e2e/benchmarks/browserstack-benchmark/app_node_test.js +++ b/e2e/benchmarks/browserstack-benchmark/app_node_test.js @@ -30,37 +30,41 @@ describe('test app.js cli', () => { mockResults = { 'iPhone_XS_1': { timeInfo: { - times: [216.00000000000045], - averageTime: 216.00000000000045, + times: [218.00000000000045, 216.00000000000045], + averageTime: 217.00000000000045, + averageTimeExclFirst: 216.00000000000045, minTime: 216.00000000000045, - maxTime: 216.00000000000045 + maxTime: 218.00000000000045 }, tabId: 'iPhone_XS_1' }, 'Samsung_Galaxy_S20_1': { timeInfo: { - times: [428.89999999897555], - averageTime: 428.89999999897555, + times: [428.89999999897555, 430.89999999897555], + averageTime: 429.89999999897555, + averageTimeExclFirst: 430.89999999897555, minTime: 428.89999999897555, - maxTime: 428.89999999897555 + maxTime: 430.89999999897555 }, tabId: 'Samsung_Galaxy_S20_1' }, 'Windows_10_1': { timeInfo: { - times: [395.8500000001095], - averageTime: 395.8500000001095, + times: [395.8500000001095, 397.8500000001095], + averageTime: 396.8500000001095, + averageTimeExclFirst: 397.8500000001095, minTime: 395.8500000001095, - maxTime: 395.8500000001095 + maxTime: 397.8500000001095 }, tabId: 'Windows_10_1' }, 'OS_X_Catalina_1': { timeInfo: { - times: [176.19500000728294], - averageTime: 176.19500000728294, + times: [178.19500000728294, 176.19500000728294], + averageTime: 177.19500000728294, + averageTimeExclFirst: 176.19500000728294, minTime: 176.19500000728294, - maxTime: 176.19500000728294 + maxTime: 178.19500000728294 }, tabId: 'OS_X_Catalina_1' } diff --git a/e2e/benchmarks/browserstack-benchmark/benchmark_models.js b/e2e/benchmarks/browserstack-benchmark/benchmark_models.js index 293c72ae3..8c4216435 100644 --- a/e2e/benchmarks/browserstack-benchmark/benchmark_models.js +++ b/e2e/benchmarks/browserstack-benchmark/benchmark_models.js @@ -49,7 +49,7 @@ async function getBenchmarkSummary(timeInfo, memoryInfo, modelName = 'model') { const benchmarkSummary = ` benchmark the ${modelName} on ${envSummary} 1st inference time: ${printTime(timeInfo.times[0])} - Average inference time (${numRuns} runs): ${printTime(timeInfo.averageTime)} + Subsequent average inference time (${numRuns} runs): ${printTime(timeInfo.averageTimeExclFirst)} Best inference time: ${printTime(timeInfo.minTime)} Peak memory: ${printMemory(memoryInfo.peakBytes)} `; @@ -85,7 +85,7 @@ async function benchmarkModel(benchmarkParameters) { memoryInfo = await profileModelInference(model, input); } - return {timeInfo, memoryInfo}; + return { timeInfo, memoryInfo }; } async function benchmarkCodeSnippet(benchmarkParameters) { @@ -97,7 +97,7 @@ async function benchmarkCodeSnippet(benchmarkParameters) { if (predict == null) { throw new Error( - 'predict function is suppoed to be defined in codeSnippet.'); + 'predict function is suppoed to be defined in codeSnippet.'); } // Warm up. @@ -107,7 +107,7 @@ async function benchmarkCodeSnippet(benchmarkParameters) { timeInfo = await timeInference(predict, benchmarkParameters.numRuns); memoryInfo = await profileInference(predict); - return {timeInfo, memoryInfo}; + return { timeInfo, memoryInfo }; } describe('BrowserStack benchmark', () => { @@ -135,11 +135,11 @@ describe('BrowserStack benchmark', () => { // Get GPU hardware info. resultObj.gpuInfo = - targetBackend === 'webgl' ? (await getRendererInfo()) : 'MISS'; + targetBackend === 'webgl' ? (await getRendererInfo()) : 'MISS'; // Report results. console.log( - `${JSON.stringify(resultObj)}`); + `${JSON.stringify(resultObj)}`); } catch (error) { console.log(`${error}`); } diff --git a/e2e/benchmarks/browserstack-benchmark/benchmark_results.html b/e2e/benchmarks/browserstack-benchmark/benchmark_results.html index 342fc274d..de8c342e8 100644 --- a/e2e/benchmarks/browserstack-benchmark/benchmark_results.html +++ b/e2e/benchmarks/browserstack-benchmark/benchmark_results.html @@ -86,7 +86,7 @@ limitations under the License. } structuredBenchmarkResults[tableName][deviceName][benchmarkTargetName] - = benchmarkReocrd?.value?.timeInfo?.averageTime; + = benchmarkReocrd?.value?.timeInfo?.averageTimeExclFirst; } return structuredBenchmarkResults; } diff --git a/e2e/benchmarks/browserstack-benchmark/index.js b/e2e/benchmarks/browserstack-benchmark/index.js index 2dd2b3bf3..bf5ee9574 100644 --- a/e2e/benchmarks/browserstack-benchmark/index.js +++ b/e2e/benchmarks/browserstack-benchmark/index.js @@ -16,7 +16,7 @@ */ const TUNABLE_BROWSER_FIELDS = - ['os', 'os_version', 'browser', 'browser_version', 'device']; + ['os', 'os_version', 'browser', 'browser_version', 'device']; const WAITING_STATUS_COLOR = '#AAAAAA'; const COMPLETE_STATUS_COLOR = '#357edd'; const ERROR_STATUS_COLOR = '#e8564b'; @@ -48,9 +48,9 @@ const state = { numRuns: 10, backend: 'webgl', setupCodeSnippetEnv: - 'const img = tf.randomUniform([1, 240, 240, 3], 0, 1000); const filter = tf.randomUniform([3, 3, 3, 3], 0, 1000);', + 'const img = tf.randomUniform([1, 240, 240, 3], 0, 1000); const filter = tf.randomUniform([3, 3, 3, 3], 0, 1000);', codeSnippet: - 'predict = () => { return tf.conv2d(img, filter, 2, \'same\');};' + 'predict = () => { return tf.conv2d(img, filter, 2, \'same\');};' }, /** @@ -69,7 +69,7 @@ const state = { addBrowser: () => { // Add browser config to `state.browsers` array. - state.browsers.push({...state.browser}); + state.browsers.push({ ...state.browser }); // Enable the benchmark button. benchmarkButton.__li.style.pointerEvents = ''; @@ -84,8 +84,8 @@ const state = { removeBrowser: index => { if (index >= state.browsers.length) { throw new Error( - `Invalid index ${index}, while the state.browsers only ` + - `has ${state.browsers.length} items.`); + `Invalid index ${index}, while the state.browsers only ` + + `has ${state.browsers.length} items.`); } // Remove the browser from the `state.browsers` array. @@ -127,7 +127,7 @@ const state = { browserTabIdConfigMap[tabId] = browser; }); - const benchmark = {...state.benchmark}; + const benchmark = { ...state.benchmark }; if (state.benchmark.model !== 'custom') { delete benchmark['modelUrl']; } @@ -180,7 +180,7 @@ function constructBrowserTree(browsersArray) { // Route through non-leaf nodes. for (let fieldIndex = 0; fieldIndex <= TUNABLE_BROWSER_FIELDS.length - 2; - fieldIndex++) { + fieldIndex++) { const fieldName = TUNABLE_BROWSER_FIELDS[fieldIndex]; if (currentNode[browser[fieldName]] == null) { currentNode[browser[fieldName]] = {}; @@ -190,14 +190,14 @@ function constructBrowserTree(browsersArray) { // Set the full configuration as the leaf node. const leafFieldName = - TUNABLE_BROWSER_FIELDS[TUNABLE_BROWSER_FIELDS.length - 1]; + TUNABLE_BROWSER_FIELDS[TUNABLE_BROWSER_FIELDS.length - 1]; const leafFieldValue = browser[leafFieldName]; if (currentNode[leafFieldValue] == null) { currentNode[leafFieldValue] = browser; } else { console.warn( - `The browser ${browser} shares the same ` + - 'configuration with another browser.'); + `The browser ${browser} shares the same ` + + 'configuration with another browser.'); } }); return browserTreeRoot; @@ -213,7 +213,7 @@ function constructBrowserTree(browsersArray) { * @param {object} currentNode */ function updateFollowingFields( - currentFieldIndex, currentFieldValue, currentNode) { + currentFieldIndex, currentFieldValue, currentNode) { const nextFieldIndex = currentFieldIndex + 1; if (nextFieldIndex === TUNABLE_BROWSER_FIELDS.length) { return; @@ -231,7 +231,7 @@ function updateFollowingFields( // Update the options for the next field. const nextFieldController = browserSettingControllers[nextFieldIndex].options( - nextFieldAvailableValues); + nextFieldAvailableValues); // When updating options for a dat.gui controller, a new controller instacne // will be created, so we need to bind the event again and record the new @@ -266,7 +266,7 @@ function updateFollowingFields( function showBrowserField(fieldIndex, currentNode) { const fieldName = TUNABLE_BROWSER_FIELDS[fieldIndex]; const fieldController = - browserFolder.add(state.browser, fieldName, Object.keys(currentNode)); + browserFolder.add(state.browser, fieldName, Object.keys(currentNode)); fieldController.onFinishChange(() => { const newValue = state.browser[fieldName]; @@ -312,7 +312,7 @@ function drawTunableBrowserSummaryTable(summaryTabId, browsers) { // Whenever a browser configuration is removed, this table will be re-drawn, // so the index (the argument for state.removeBrowser) will be re-assigned. const removeBrowserButtonElement = - ``; + ``; row.push(removeBrowserButtonElement); values.push(row); @@ -321,9 +321,9 @@ function drawTunableBrowserSummaryTable(summaryTabId, browsers) { const surface = { name: 'Browsers to benchmark', tab: summaryTabId, - styles: {width: '100%'} + styles: { width: '100%' } }; - tfvis.render.table(surface, {headers, values}); + tfvis.render.table(surface, { headers, values }); } /** @@ -351,9 +351,9 @@ function drawUntunableBrowserSummaryTable(summaryTabId, browserTabIdConfigMap) { const surface = { name: 'Browsers to benchmark', tab: summaryTabId, - styles: {width: '100%'} + styles: { width: '100%' } }; - tfvis.render.table(surface, {headers, values}); + tfvis.render.table(surface, { headers, values }); } function initVisor() { @@ -364,7 +364,7 @@ function initVisor() { // Bind an event to visor's 'Maximize/Minimize' button. const visorFullScreenButton = - tfvis.visor().el.getElementsByTagName('button')[0]; + tfvis.visor().el.getElementsByTagName('button')[0]; const guiCloseButton = document.getElementsByClassName('close-button')[0]; const originalGuiWidth = gui.domElement.style.width; @@ -499,7 +499,7 @@ function setTabStatus(tabId, status) { */ function addLoaderElement(tabId) { const surface = tfvis.visor().surface( - {name: 'Benchmark Summary', tab: tabId, styles: {width: '100%'}}); + { name: 'Benchmark Summary', tab: tabId, styles: { width: '100%' } }); const loaderElement = document.createElement('div'); loaderElement.className = 'loader'; loaderElement.id = `${tabId}-loader`; @@ -523,7 +523,7 @@ function drawBenchmarkResultSummaryTable(benchmarkResult) { const headers = ['Field', 'Value']; const values = []; - const {timeInfo, memoryInfo, tabId} = benchmarkResult; + const { timeInfo, memoryInfo, tabId } = benchmarkResult; const timeArray = benchmarkResult.timeInfo.times; const numRuns = timeArray.length; @@ -533,8 +533,8 @@ function drawBenchmarkResultSummaryTable(benchmarkResult) { values.push(['2nd inference time', printTime(timeArray[1])]); } values.push([ - `Average inference time (${numRuns} runs)`, - printTime(timeInfo.averageTime) + `Average inference time (${numRuns} runs) except the first`, + printTime(timeInfo.averageTimeExclFirst) ]); values.push(['Best time', printTime(timeInfo.minTime)]); values.push(['Worst time', printTime(timeInfo.maxTime)]); @@ -552,9 +552,9 @@ function drawBenchmarkResultSummaryTable(benchmarkResult) { const surface = { name: 'Benchmark Summary', tab: tabId, - styles: {width: '100%'} + styles: { width: '100%' } }; - tfvis.render.table(surface, {headers, values}); + tfvis.render.table(surface, { headers, values }); } async function drawInferenceTimeLineChart(benchmarkResult) { @@ -571,34 +571,34 @@ async function drawInferenceTimeLineChart(benchmarkResult) { if (index === 0) { return; } - values.push({x: index + 1, y: time}); + values.push({ x: index + 1, y: time }); }); const surface = { name: `2nd - ${inferenceTimeArray.length}st Inference Time`, tab: tabId, - styles: {width: '100%'} + styles: { width: '100%' } }; - const data = {values}; + const data = { values }; const drawOptions = - {zoomToFit: true, xLabel: '', yLabel: 'time (ms)', xType: 'ordinal'}; + { zoomToFit: true, xLabel: '', yLabel: 'time (ms)', xType: 'ordinal' }; await tfvis.render.linechart(surface, data, drawOptions); // Whenever resize the parent div element, re-draw the chart canvas. try { const originalCanvasHeight = tfvis.visor() - .surface(surface) - .drawArea.getElementsByTagName('canvas')[0] - .height; + .surface(surface) + .drawArea.getElementsByTagName('canvas')[0] + .height; const labelElement = tfvis.visor().surface(surface).label; new ResizeObserver(() => { // Keep the height of chart/canvas unchanged. tfvis.visor() - .surface(surface) - .drawArea.getElementsByTagName('canvas')[0] - .height = originalCanvasHeight; + .surface(surface) + .drawArea.getElementsByTagName('canvas')[0] + .height = originalCanvasHeight; tfvis.render.linechart(surface, data, drawOptions); }).observe(labelElement); } catch (e) { @@ -618,9 +618,9 @@ function drawBrowserSettingTable(tabId, browserConf) { const surface = { name: 'Browser Setting', tab: tabId, - styles: {width: '100%'} + styles: { width: '100%' } }; - tfvis.render.table(surface, {headers, values}); + tfvis.render.table(surface, { headers, values }); } function drawBenchmarkParameterTable(tabId) { @@ -636,9 +636,9 @@ function drawBenchmarkParameterTable(tabId) { const surface = { name: 'Benchmark Parameter', tab: tabId, - styles: {width: '100%'} + styles: { width: '100%' } }; - tfvis.render.table(surface, {headers, values}); + tfvis.render.table(surface, { headers, values }); } function showModelSelection() { @@ -646,21 +646,21 @@ function showModelSelection() { let modelUrlController = null; modelFolder - .add( - state.benchmark, 'model', [...Object.keys(benchmarks), 'codeSnippet']) - .name('model name') - .onChange(async model => { - if (model === 'custom') { - if (modelUrlController === null) { - modelUrlController = modelFolder.add(state.benchmark, 'modelUrl'); - modelUrlController.domElement.querySelector('input').placeholder = - 'https://your-domain.com/model-path/model.json'; - } - } else if (modelUrlController != null) { - modelFolder.remove(modelUrlController); - modelUrlController = null; + .add( + state.benchmark, 'model', [...Object.keys(benchmarks), 'codeSnippet']) + .name('model name') + .onChange(async model => { + if (model === 'custom') { + if (modelUrlController === null) { + modelUrlController = modelFolder.add(state.benchmark, 'modelUrl'); + modelUrlController.domElement.querySelector('input').placeholder = + 'https://your-domain.com/model-path/model.json'; } - }); + } else if (modelUrlController != null) { + modelFolder.remove(modelUrlController); + modelUrlController = null; + } + }); modelFolder.open(); return modelFolder; } @@ -688,7 +688,7 @@ function printMemory(bytes) { } function onPageLoad() { - gui = new dat.gui.GUI({width: 400}); + gui = new dat.gui.GUI({ width: 400 }); gui.domElement.id = 'gui'; socket = io(); @@ -714,7 +714,7 @@ function onPageLoad() { // Enable users to benchmark. addingBrowserButton = - browserFolder.add(state, 'addBrowser').name('Add browser'); + browserFolder.add(state, 'addBrowser').name('Add browser'); benchmarkButton = gui.add(state, 'run').name('Run benchmark'); // Disable the 'Run benchmark' button until a browser is added.