Add option to sim to collect performance data

bin/build-call_sim-cli

@@ -88,7 +88,7 @@ export MACOSX_DEPLOYMENT_TARGET="10.15"
         gn gen -C "${OUTPUT_DIR}"/debug "--args=${WEBRTC_ARGS}"
         ninja -C "${OUTPUT_DIR}"/debug
     else
-        gn gen -C "${OUTPUT_DIR}"/release "--args=${WEBRTC_ARGS} is_debug=false"
+        gn gen -C "${OUTPUT_DIR}"/release "--args=${WEBRTC_ARGS} is_debug=false symbol_level=1"
         ninja -C "${OUTPUT_DIR}"/release
     fi
 )
@@ -100,6 +100,8 @@ export MACOSX_DEPLOYMENT_TARGET="10.15"
         OUTPUT_DIR="${OUTPUT_DIR}" cargo build --package ringrtc --bin call_sim-cli --features=call_sim
         echo "Can run with target/debug/call_sim-cli"
     else
+        # Build with debug line tables, but not full debug info.
+        export CARGO_PROFILE_RELEASE_DEBUG=1
         OUTPUT_DIR="${OUTPUT_DIR}" cargo build --package ringrtc --bin call_sim-cli --features=call_sim --release
         echo "Can run with target/release/call_sim-cli"
     fi

call_sim/docker/ringrtc/Dockerfile

@@ -6,6 +6,9 @@
 FROM ubuntu:24.04
 
 RUN apt-get update \
-    && apt-get install -y curl iproute2 iperf iptables iputils-ping protobuf-compiler libpulse-dev
+    && apt-get install -y curl iproute2 iperf iptables iputils-ping protobuf-compiler libpulse-dev linux-tools-generic linux-tools-common build-essential \
+    && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
+    &&  . ~/.cargo/env \
+    && cargo install --features=bin addr2line
 
 COPY target/release/call_sim-cli /usr/local/bin/

call_sim/src/docker.rs

@@ -12,6 +12,7 @@ use chrono::DateTime;
 use futures_util::stream::TryStreamExt;
 use itertools::Itertools;
 use std::process::Stdio;
+use std::time::Duration;
 use tokio::fs::OpenOptions;
 use tokio::io::{stdout, AsyncWriteExt};
 use tokio::process::Command;
@@ -678,16 +679,36 @@ pub async fn start_cli(
     remote_call_config: &CallConfig,
     client_profile: &ClientProfile,
     call_type: &CallTypeConfig,
+    profile: bool,
 ) -> Result<()> {
     println!("Starting cli for `{}`", name);
     let log_file_arg = format!("/report/{}.log", name);
     let input_file_arg = format!("/media/{}", media_io.audio_input_file);
     let output_file_arg = format!("/report/{}", media_io.audio_output_file);
 
-    let mut args = [
+    let mut args = ["exec", "-d", name].map(String::from).to_vec();
-        "exec",
+
-        "-d",
+    if profile {
-        name,
+        let perf_arg = format!("--output=/report/{}.perf", name);
+
+        args.extend_from_slice(
+            &[
+                "perf",
+                "record",
+                "-e",
+                "cycles",
+                "--call-graph=dwarf",
+                "-F",
+                "99",
+                "--user-callchains",
+                &perf_arg,
+            ]
+            .map(String::from),
+        );
+    }
+
+    args.extend_from_slice(
+        &[
             "call_sim-cli",
             "--name",
             name,
@@ -698,8 +719,8 @@ pub async fn start_cli(
             "--output-file",
             &output_file_arg,
         ]
-    .map(String::from)
+        .map(String::from),
-    .to_vec();
+    );
 
     args.push("--stats-interval-secs".to_string());
     args.push(format!("{}", call_config.stats_interval_secs));
@@ -862,6 +883,38 @@ pub async fn start_cli(
     Ok(())
 }
 
+pub async fn finish_perf(a: &str, b: &str) -> Result<()> {
+    for client in [a, b] {
+        let mut exited = false;
+        for _ in 0..60 {
+            let status = Command::new("docker")
+                .args(["exec", client, "pgrep", "perf"])
+                .spawn()?
+                .wait()
+                .await?;
+            if !status.success() {
+                // if we couldn't find it, it exited; otherwise keep waiting.
+                exited = true;
+                let perf_command = format!(
+                    "perf report --percent-limit=5 --call-graph=2 -i /report/{}.perf \
+                    --addr2line=/root/.cargo/bin/addr2line > /report/{}.perf.txt 2>&1",
+                    client, client
+                );
+                let _ = Command::new("docker")
+                    .args(["exec", client, "sh", "-c", &perf_command])
+                    .spawn()?
+                    .wait()
+                    .await?;
+
+                break;
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+        println!("{} perf exited? {}", client, exited);
+    }
+    Ok(())
+}
+
 pub async fn convert_raw_to_wav(
     location: &str,
     raw_file: &str,

call_sim/src/main.rs

@@ -70,6 +70,10 @@ struct Args {
     /// If None, then uses the first group in the list
     #[arg(long)]
     group_name: Option<String>,
+
+    /// Run `perf record` on the clients
+    #[arg(long)]
+    profile: bool,
 }
 
 // Set these two values when running call sim group calls. The Auth Key is used to generate profiles
@@ -640,6 +644,7 @@ async fn main() -> Result<()> {
             &test_set_name,
             client_profiles.clone(),
             call_type_config,
+            args.profile,
         )?;
         match test_set_name.as_str() {
             "minimal_example" => run_minimal_example(test).await?,

call_sim/src/test.rs

@@ -29,9 +29,9 @@ use crate::common::{
 use crate::docker::{
     analyze_video, analyze_visqol_mos, clean_network, clean_up, convert_mp4_to_yuv,
     convert_raw_to_wav, convert_wav_to_16khz_mono, convert_yuv_to_mp4, create_network,
-    emulate_network_change, emulate_network_start, generate_spectrogram, get_signaling_server_logs,
+    emulate_network_change, emulate_network_start, finish_perf, generate_spectrogram,
-    get_turn_server_logs, start_cli, start_client, start_signaling_server, start_tcp_dump,
+    get_signaling_server_logs, get_turn_server_logs, start_cli, start_client,
-    start_turn_server, DockerStats,
+    start_signaling_server, start_tcp_dump, start_turn_server, DockerStats,
 };
 use crate::report::{AnalysisReport, AnalysisReportMos, Report};
 use crate::{
@@ -154,6 +154,9 @@ pub struct Test {
     // information even if we change the reference media in the future.
     sounds: HashMap<String, Sound>,
     videos: HashMap<String, Video>,
+
+    // Whether to run `perf record` (and report)
+    profile: bool,
 }
 
 pub struct MediaFileIo {
@@ -171,6 +174,7 @@ impl Test {
         set_name: &str,
         client_profiles: Vec<ClientProfile>,
         call_type: CallTypeConfig,
+        profile: bool,
     ) -> Result<Self> {
         let time_started = chrono::Local::now();
 
@@ -205,6 +209,7 @@ impl Test {
 
             client_profiles,
             call_type,
+            profile,
         })
     }
 
@@ -284,6 +289,7 @@ impl Test {
                 &test_case_config.client_b_config,
                 &self.client_profiles[0],
                 &self.call_type,
+                self.profile,
             )
             .await?;
 
@@ -299,6 +305,7 @@ impl Test {
                 &test_case_config.client_a_config,
                 &self.client_profiles[1],
                 &self.call_type,
+                self.profile,
             )
             .await?;
 
@@ -787,6 +794,17 @@ impl Test {
             .await
         {
             Ok(_) => {
+                if self.profile {
+                    // allow perf to finish and collect reports.
+                    println!("waiting for perf... ");
+                    if let Err(e) =
+                        finish_perf(test_case.client_a.name, test_case.client_b.name).await
+                    {
+                        println!("couldn't wait for perf {:?}", e);
+                    }
+                    println!("... done");
+                }
+
                 // For debugging, dump the signaling_server logs.
                 get_signaling_server_logs(&test_case.test_path).await?;