Revert "Switch low bw audio test to histograms."

This reverts commit 71c9a18f25.

Reason for revert: Relevant python code isn't pulled when the test runs on swarming.

Original change's description:
> Switch low bw audio test to histograms.
> 
> Also requires a recipe change so the results processor switches to
> histogram mode when this CL is landed.
> 
> Bug: chromium:1029452
> Change-Id: Ic09deefc3f4f9d7a82ffeafeb5209fcfc361aece
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171683
> Reviewed-by: Artem Titov <titovartem@webrtc.org>
> Commit-Queue: Patrik Höglund <phoglund@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#30884}

TBR=phoglund@webrtc.org,titovartem@webrtc.org

Change-Id: I6b3645ff939943a21185a1a1c8c5a0877e29db8c
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: chromium:1029452
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171692
Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
Commit-Queue: Patrik Höglund <phoglund@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30893}
This commit is contained in:
Patrik Höglund 2020-03-26 06:46:20 +00:00 committed by Commit Bot
parent ac80481c48
commit ad502fa727
3 changed files with 32 additions and 60 deletions

View file

@ -206,6 +206,7 @@ if (rtc_include_tests) {
if (is_android) { if (is_android) {
deps += [ "//testing/android/native_test:native_test_native_code" ] deps += [ "//testing/android/native_test:native_test_native_code" ]
} }
data = [ data = [
"../resources/voice_engine/audio_tiny16.wav", "../resources/voice_engine/audio_tiny16.wav",
"../resources/voice_engine/audio_tiny48.wav", "../resources/voice_engine/audio_tiny48.wav",
@ -222,8 +223,6 @@ if (rtc_include_tests) {
"../resources/voice_engine/audio_tiny16.wav", "../resources/voice_engine/audio_tiny16.wav",
"../resources/voice_engine/audio_tiny48.wav", "../resources/voice_engine/audio_tiny48.wav",
] ]
data_deps =
[ "//third_party/catapult/tracing/tracing/proto:histogram_proto" ]
if (is_win) { if (is_win) {
data += [ "${root_out_dir}/low_bandwidth_audio_test.exe" ] data += [ "${root_out_dir}/low_bandwidth_audio_test.exe" ]
} else { } else {

View file

@ -16,6 +16,7 @@ output files will be performed.
import argparse import argparse
import collections import collections
import json
import logging import logging
import os import os
import re import re
@ -57,7 +58,7 @@ def _ParseArgs():
parser.add_argument('--num-retries', default='0', parser.add_argument('--num-retries', default='0',
help='Number of times to retry the test on Android.') help='Number of times to retry the test on Android.')
parser.add_argument('--isolated_script_test_perf_output', default=None, parser.add_argument('--isolated_script_test_perf_output', default=None,
help='Path to store perf results in histogram proto format.') help='Path to store perf results in chartjson format.')
parser.add_argument('--extra-test-args', default=[], action='append', parser.add_argument('--extra-test-args', default=[], action='append',
help='Extra args to path to the test binary.') help='Extra args to path to the test binary.')
@ -169,7 +170,7 @@ def _RunPesq(executable_path, reference_file, degraded_file,
if match: if match:
raw_mos, _ = match.groups() raw_mos, _ = match.groups()
return {'pesq_mos': (raw_mos, 'unitless')} return {'pesq_mos': (raw_mos, 'score')}
else: else:
logging.error('PESQ: %s', out.splitlines()[-1]) logging.error('PESQ: %s', out.splitlines()[-1])
return {} return {}
@ -195,66 +196,41 @@ def _RunPolqa(executable_path, reference_file, degraded_file):
return {} return {}
mos_lqo, = match.groups() mos_lqo, = match.groups()
return {'polqa_mos_lqo': (mos_lqo, 'unitless')} return {'polqa_mos_lqo': (mos_lqo, 'score')}
def _MergeInPerfResultsFromCcTests(histograms, run_perf_results_file): def _AddChart(charts, metric, test_name, value, units):
from tracing.value import histogram_set chart = charts.setdefault(metric, {})
chart[test_name] = {
"type": "scalar",
"value": value,
"units": units,
}
cc_histograms = histogram_set.HistogramSet()
def _AddRunPerfResults(charts, run_perf_results_file):
with open(run_perf_results_file, 'rb') as f: with open(run_perf_results_file, 'rb') as f:
contents = f.read() per_run_perf_results = json.load(f)
if not contents: if 'charts' not in per_run_perf_results:
return return
for metric, cases in per_run_perf_results['charts'].items():
cc_histograms.ImportProto(contents) chart = charts.setdefault(metric, {})
for case_name, case_value in cases.items():
histograms.Merge(cc_histograms) if case_name in chart:
logging.error('Overriding results for %s/%s', metric, case_name)
chart[case_name] = case_value
Analyzer = collections.namedtuple('Analyzer', ['name', 'func', 'executable', Analyzer = collections.namedtuple('Analyzer', ['name', 'func', 'executable',
'sample_rate_hz']) 'sample_rate_hz'])
def _ConfigurePythonPath(args):
script_dir = os.path.dirname(os.path.realpath(__file__))
checkout_root = os.path.abspath(
os.path.join(script_dir, os.pardir, os.pardir))
sys.path.insert(0, os.path.join(checkout_root, 'third_party', 'catapult',
'tracing'))
sys.path.insert(0, os.path.join(checkout_root, 'third_party', 'protobuf',
'python'))
# The low_bandwidth_audio_perf_test gn rule will build the protobuf stub for
# python, so put it in the path for this script before we attempt to import
# it.
histogram_proto_path = os.path.join(
args.build_dir, 'pyproto', 'tracing', 'tracing', 'proto')
sys.path.insert(0, histogram_proto_path)
# Fail early in case the proto hasn't been built.
from tracing.proto import histogram_proto
if not histogram_proto.HAS_PROTO:
raise ImportError('Could not find histogram_pb2. You need to build the '
'low_bandwidth_audio_perf_test target before invoking '
'this script. Expected to find '
'histogram_pb2.py in %s.' % histogram_proto_path)
def main(): def main():
# pylint: disable=W0101 # pylint: disable=W0101
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
args = _ParseArgs() args = _ParseArgs()
_ConfigurePythonPath(args)
# Import catapult modules here after configuring the pythonpath.
from tracing.value import histogram_set
from tracing.value.diagnostics import reserved_infos
from tracing.value.diagnostics import generic_set
pesq_path, polqa_path = _GetPathToTools() pesq_path, polqa_path = _GetPathToTools()
if pesq_path is None: if pesq_path is None:
return 1 return 1
@ -274,14 +250,14 @@ def main():
if polqa_path and _RunPolqa(polqa_path, example_path, example_path): if polqa_path and _RunPolqa(polqa_path, example_path, example_path):
analyzers.append(Analyzer('polqa', _RunPolqa, polqa_path, 48000)) analyzers.append(Analyzer('polqa', _RunPolqa, polqa_path, 48000))
histograms = histogram_set.HistogramSet() charts = {}
for analyzer in analyzers: for analyzer in analyzers:
# Start the test executable that produces audio files. # Start the test executable that produces audio files.
test_process = subprocess.Popen( test_process = subprocess.Popen(
_LogCommand(test_command + [ _LogCommand(test_command + [
'--sample_rate_hz=%d' % analyzer.sample_rate_hz, '--sample_rate_hz=%d' % analyzer.sample_rate_hz,
'--test_case_prefix=%s' % analyzer.name, '--test_case_prefix=%s' % analyzer.name
'--write_histogram_proto_json'
] + args.extra_test_args), ] + args.extra_test_args),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
perf_results_file = None perf_results_file = None
@ -303,12 +279,9 @@ def main():
analyzer_results = analyzer.func(analyzer.executable, analyzer_results = analyzer.func(analyzer.executable,
reference_file, degraded_file) reference_file, degraded_file)
for metric, (value, units) in analyzer_results.items(): for metric, (value, units) in analyzer_results.items():
hist = histograms.CreateHistogram(metric, units, [value]) # Output a result for the perf dashboard.
user_story = generic_set.GenericSet([test_name])
hist.diagnostics[reserved_infos.STORIES.name] = user_story
# Output human readable results.
print 'RESULT %s: %s= %s %s' % (metric, test_name, value, units) print 'RESULT %s: %s= %s %s' % (metric, test_name, value, units)
_AddChart(charts, metric, test_name, value, units)
if args.remove: if args.remove:
os.remove(reference_file) os.remove(reference_file)
@ -318,13 +291,13 @@ def main():
if perf_results_file: if perf_results_file:
perf_results_file = _GetFile(perf_results_file, out_dir, move=True, perf_results_file = _GetFile(perf_results_file, out_dir, move=True,
android=args.android, adb_prefix=adb_prefix) android=args.android, adb_prefix=adb_prefix)
_MergeInPerfResultsFromCcTests(histograms, perf_results_file) _AddRunPerfResults(charts, perf_results_file)
if args.remove: if args.remove:
os.remove(perf_results_file) os.remove(perf_results_file)
if args.isolated_script_test_perf_output: if args.isolated_script_test_perf_output:
with open(args.isolated_script_test_perf_output, 'wb') as f: with open(args.isolated_script_test_perf_output, 'w') as f:
f.write(histograms.AsProto().SerializeToString()) json.dump({"format_version": "1.0", "charts": charts}, f)
return test_process.wait() return test_process.wait()

View file

@ -105,7 +105,7 @@ std::string AudioOutputFile() {
std::string PerfResultsOutputFile() { std::string PerfResultsOutputFile() {
return webrtc::test::OutputPath() + "PCLowBandwidth_perf_" + return webrtc::test::OutputPath() + "PCLowBandwidth_perf_" +
FileSampleRateSuffix() + ".pb"; FileSampleRateSuffix() + ".json";
} }
void LogTestResults() { void LogTestResults() {