mirror of
https://github.com/mollyim/webrtc.git
synced 2025-05-17 23:57:59 +01:00
Revert "Switch low bw audio test to histograms."
This reverts commit 71c9a18f25
.
Reason for revert: Relevant python code isn't pulled when the test runs on swarming.
Original change's description:
> Switch low bw audio test to histograms.
>
> Also requires a recipe change so the results processor switches to
> histogram mode when this CL is landed.
>
> Bug: chromium:1029452
> Change-Id: Ic09deefc3f4f9d7a82ffeafeb5209fcfc361aece
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171683
> Reviewed-by: Artem Titov <titovartem@webrtc.org>
> Commit-Queue: Patrik Höglund <phoglund@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#30884}
TBR=phoglund@webrtc.org,titovartem@webrtc.org
Change-Id: I6b3645ff939943a21185a1a1c8c5a0877e29db8c
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: chromium:1029452
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171692
Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
Commit-Queue: Patrik Höglund <phoglund@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30893}
This commit is contained in:
parent
ac80481c48
commit
ad502fa727
3 changed files with 32 additions and 60 deletions
|
@ -206,6 +206,7 @@ if (rtc_include_tests) {
|
||||||
if (is_android) {
|
if (is_android) {
|
||||||
deps += [ "//testing/android/native_test:native_test_native_code" ]
|
deps += [ "//testing/android/native_test:native_test_native_code" ]
|
||||||
}
|
}
|
||||||
|
|
||||||
data = [
|
data = [
|
||||||
"../resources/voice_engine/audio_tiny16.wav",
|
"../resources/voice_engine/audio_tiny16.wav",
|
||||||
"../resources/voice_engine/audio_tiny48.wav",
|
"../resources/voice_engine/audio_tiny48.wav",
|
||||||
|
@ -222,8 +223,6 @@ if (rtc_include_tests) {
|
||||||
"../resources/voice_engine/audio_tiny16.wav",
|
"../resources/voice_engine/audio_tiny16.wav",
|
||||||
"../resources/voice_engine/audio_tiny48.wav",
|
"../resources/voice_engine/audio_tiny48.wav",
|
||||||
]
|
]
|
||||||
data_deps =
|
|
||||||
[ "//third_party/catapult/tracing/tracing/proto:histogram_proto" ]
|
|
||||||
if (is_win) {
|
if (is_win) {
|
||||||
data += [ "${root_out_dir}/low_bandwidth_audio_test.exe" ]
|
data += [ "${root_out_dir}/low_bandwidth_audio_test.exe" ]
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -16,6 +16,7 @@ output files will be performed.
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import collections
|
import collections
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
@ -57,7 +58,7 @@ def _ParseArgs():
|
||||||
parser.add_argument('--num-retries', default='0',
|
parser.add_argument('--num-retries', default='0',
|
||||||
help='Number of times to retry the test on Android.')
|
help='Number of times to retry the test on Android.')
|
||||||
parser.add_argument('--isolated_script_test_perf_output', default=None,
|
parser.add_argument('--isolated_script_test_perf_output', default=None,
|
||||||
help='Path to store perf results in histogram proto format.')
|
help='Path to store perf results in chartjson format.')
|
||||||
parser.add_argument('--extra-test-args', default=[], action='append',
|
parser.add_argument('--extra-test-args', default=[], action='append',
|
||||||
help='Extra args to path to the test binary.')
|
help='Extra args to path to the test binary.')
|
||||||
|
|
||||||
|
@ -169,7 +170,7 @@ def _RunPesq(executable_path, reference_file, degraded_file,
|
||||||
if match:
|
if match:
|
||||||
raw_mos, _ = match.groups()
|
raw_mos, _ = match.groups()
|
||||||
|
|
||||||
return {'pesq_mos': (raw_mos, 'unitless')}
|
return {'pesq_mos': (raw_mos, 'score')}
|
||||||
else:
|
else:
|
||||||
logging.error('PESQ: %s', out.splitlines()[-1])
|
logging.error('PESQ: %s', out.splitlines()[-1])
|
||||||
return {}
|
return {}
|
||||||
|
@ -195,66 +196,41 @@ def _RunPolqa(executable_path, reference_file, degraded_file):
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
mos_lqo, = match.groups()
|
mos_lqo, = match.groups()
|
||||||
return {'polqa_mos_lqo': (mos_lqo, 'unitless')}
|
return {'polqa_mos_lqo': (mos_lqo, 'score')}
|
||||||
|
|
||||||
|
|
||||||
def _MergeInPerfResultsFromCcTests(histograms, run_perf_results_file):
|
def _AddChart(charts, metric, test_name, value, units):
|
||||||
from tracing.value import histogram_set
|
chart = charts.setdefault(metric, {})
|
||||||
|
chart[test_name] = {
|
||||||
|
"type": "scalar",
|
||||||
|
"value": value,
|
||||||
|
"units": units,
|
||||||
|
}
|
||||||
|
|
||||||
cc_histograms = histogram_set.HistogramSet()
|
|
||||||
|
def _AddRunPerfResults(charts, run_perf_results_file):
|
||||||
with open(run_perf_results_file, 'rb') as f:
|
with open(run_perf_results_file, 'rb') as f:
|
||||||
contents = f.read()
|
per_run_perf_results = json.load(f)
|
||||||
if not contents:
|
if 'charts' not in per_run_perf_results:
|
||||||
return
|
return
|
||||||
|
for metric, cases in per_run_perf_results['charts'].items():
|
||||||
cc_histograms.ImportProto(contents)
|
chart = charts.setdefault(metric, {})
|
||||||
|
for case_name, case_value in cases.items():
|
||||||
histograms.Merge(cc_histograms)
|
if case_name in chart:
|
||||||
|
logging.error('Overriding results for %s/%s', metric, case_name)
|
||||||
|
chart[case_name] = case_value
|
||||||
|
|
||||||
|
|
||||||
Analyzer = collections.namedtuple('Analyzer', ['name', 'func', 'executable',
|
Analyzer = collections.namedtuple('Analyzer', ['name', 'func', 'executable',
|
||||||
'sample_rate_hz'])
|
'sample_rate_hz'])
|
||||||
|
|
||||||
|
|
||||||
def _ConfigurePythonPath(args):
|
|
||||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
|
||||||
checkout_root = os.path.abspath(
|
|
||||||
os.path.join(script_dir, os.pardir, os.pardir))
|
|
||||||
|
|
||||||
sys.path.insert(0, os.path.join(checkout_root, 'third_party', 'catapult',
|
|
||||||
'tracing'))
|
|
||||||
sys.path.insert(0, os.path.join(checkout_root, 'third_party', 'protobuf',
|
|
||||||
'python'))
|
|
||||||
|
|
||||||
# The low_bandwidth_audio_perf_test gn rule will build the protobuf stub for
|
|
||||||
# python, so put it in the path for this script before we attempt to import
|
|
||||||
# it.
|
|
||||||
histogram_proto_path = os.path.join(
|
|
||||||
args.build_dir, 'pyproto', 'tracing', 'tracing', 'proto')
|
|
||||||
sys.path.insert(0, histogram_proto_path)
|
|
||||||
|
|
||||||
# Fail early in case the proto hasn't been built.
|
|
||||||
from tracing.proto import histogram_proto
|
|
||||||
if not histogram_proto.HAS_PROTO:
|
|
||||||
raise ImportError('Could not find histogram_pb2. You need to build the '
|
|
||||||
'low_bandwidth_audio_perf_test target before invoking '
|
|
||||||
'this script. Expected to find '
|
|
||||||
'histogram_pb2.py in %s.' % histogram_proto_path)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
# pylint: disable=W0101
|
# pylint: disable=W0101
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
args = _ParseArgs()
|
args = _ParseArgs()
|
||||||
|
|
||||||
_ConfigurePythonPath(args)
|
|
||||||
|
|
||||||
# Import catapult modules here after configuring the pythonpath.
|
|
||||||
from tracing.value import histogram_set
|
|
||||||
from tracing.value.diagnostics import reserved_infos
|
|
||||||
from tracing.value.diagnostics import generic_set
|
|
||||||
|
|
||||||
pesq_path, polqa_path = _GetPathToTools()
|
pesq_path, polqa_path = _GetPathToTools()
|
||||||
if pesq_path is None:
|
if pesq_path is None:
|
||||||
return 1
|
return 1
|
||||||
|
@ -274,14 +250,14 @@ def main():
|
||||||
if polqa_path and _RunPolqa(polqa_path, example_path, example_path):
|
if polqa_path and _RunPolqa(polqa_path, example_path, example_path):
|
||||||
analyzers.append(Analyzer('polqa', _RunPolqa, polqa_path, 48000))
|
analyzers.append(Analyzer('polqa', _RunPolqa, polqa_path, 48000))
|
||||||
|
|
||||||
histograms = histogram_set.HistogramSet()
|
charts = {}
|
||||||
|
|
||||||
for analyzer in analyzers:
|
for analyzer in analyzers:
|
||||||
# Start the test executable that produces audio files.
|
# Start the test executable that produces audio files.
|
||||||
test_process = subprocess.Popen(
|
test_process = subprocess.Popen(
|
||||||
_LogCommand(test_command + [
|
_LogCommand(test_command + [
|
||||||
'--sample_rate_hz=%d' % analyzer.sample_rate_hz,
|
'--sample_rate_hz=%d' % analyzer.sample_rate_hz,
|
||||||
'--test_case_prefix=%s' % analyzer.name,
|
'--test_case_prefix=%s' % analyzer.name
|
||||||
'--write_histogram_proto_json'
|
|
||||||
] + args.extra_test_args),
|
] + args.extra_test_args),
|
||||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||||
perf_results_file = None
|
perf_results_file = None
|
||||||
|
@ -303,12 +279,9 @@ def main():
|
||||||
analyzer_results = analyzer.func(analyzer.executable,
|
analyzer_results = analyzer.func(analyzer.executable,
|
||||||
reference_file, degraded_file)
|
reference_file, degraded_file)
|
||||||
for metric, (value, units) in analyzer_results.items():
|
for metric, (value, units) in analyzer_results.items():
|
||||||
hist = histograms.CreateHistogram(metric, units, [value])
|
# Output a result for the perf dashboard.
|
||||||
user_story = generic_set.GenericSet([test_name])
|
|
||||||
hist.diagnostics[reserved_infos.STORIES.name] = user_story
|
|
||||||
|
|
||||||
# Output human readable results.
|
|
||||||
print 'RESULT %s: %s= %s %s' % (metric, test_name, value, units)
|
print 'RESULT %s: %s= %s %s' % (metric, test_name, value, units)
|
||||||
|
_AddChart(charts, metric, test_name, value, units)
|
||||||
|
|
||||||
if args.remove:
|
if args.remove:
|
||||||
os.remove(reference_file)
|
os.remove(reference_file)
|
||||||
|
@ -318,13 +291,13 @@ def main():
|
||||||
if perf_results_file:
|
if perf_results_file:
|
||||||
perf_results_file = _GetFile(perf_results_file, out_dir, move=True,
|
perf_results_file = _GetFile(perf_results_file, out_dir, move=True,
|
||||||
android=args.android, adb_prefix=adb_prefix)
|
android=args.android, adb_prefix=adb_prefix)
|
||||||
_MergeInPerfResultsFromCcTests(histograms, perf_results_file)
|
_AddRunPerfResults(charts, perf_results_file)
|
||||||
if args.remove:
|
if args.remove:
|
||||||
os.remove(perf_results_file)
|
os.remove(perf_results_file)
|
||||||
|
|
||||||
if args.isolated_script_test_perf_output:
|
if args.isolated_script_test_perf_output:
|
||||||
with open(args.isolated_script_test_perf_output, 'wb') as f:
|
with open(args.isolated_script_test_perf_output, 'w') as f:
|
||||||
f.write(histograms.AsProto().SerializeToString())
|
json.dump({"format_version": "1.0", "charts": charts}, f)
|
||||||
|
|
||||||
return test_process.wait()
|
return test_process.wait()
|
||||||
|
|
||||||
|
|
|
@ -105,7 +105,7 @@ std::string AudioOutputFile() {
|
||||||
|
|
||||||
std::string PerfResultsOutputFile() {
|
std::string PerfResultsOutputFile() {
|
||||||
return webrtc::test::OutputPath() + "PCLowBandwidth_perf_" +
|
return webrtc::test::OutputPath() + "PCLowBandwidth_perf_" +
|
||||||
FileSampleRateSuffix() + ".pb";
|
FileSampleRateSuffix() + ".json";
|
||||||
}
|
}
|
||||||
|
|
||||||
void LogTestResults() {
|
void LogTestResults() {
|
||||||
|
|
Loading…
Reference in a new issue