Implement (optional) buffering

README.md

@@ -321,6 +321,24 @@ For example, you could capture the video within [OBS].
 [OBS]: https://obsproject.com/fr
 
 
+#### Buffering
+
+It is possible to add buffering. This increases latency but reduces jitter (see
+#2464).
+
+The option is available for display buffering:
+
+```bash
+scrcpy --display-buffer=50  # add 50 ms buffering for display
+```
+
+and V4L2 sink:
+
+```bash
+scrcpy --v4l2-buffer=500    # add 500 ms buffering for v4l2 sink
+```
+
+
 ### Connection
 
 #### Wireless

app/meson.build

@@ -2,6 +2,7 @@ src = [
     'src/main.c',
     'src/adb.c',
     'src/cli.c',
+    'src/clock.c',
     'src/compat.c',
     'src/control_msg.c',
     'src/controller.c',
@@ -10,6 +11,7 @@ src = [
     'src/event_converter.c',
     'src/file_handler.c',
     'src/fps_counter.c',
+    'src/frame_buffer.c',
     'src/input_manager.c',
     'src/opengl.c',
     'src/receiver.c',
@@ -25,6 +27,7 @@ src = [
     'src/util/process.c',
     'src/util/str_util.c',
     'src/util/thread.c',
+    'src/util/tick.c',
 ]
 
 if host_machine.system() == 'windows'
@@ -165,6 +168,10 @@ if get_option('buildtype') == 'debug'
             'src/cli.c',
             'src/util/str_util.c',
         ]],
+        ['test_clock', [
+            'tests/test_clock.c',
+            'src/clock.c',
+        ]],
         ['test_control_msg_serialize', [
             'tests/test_control_msg_serialize.c',
             'src/control_msg.c',

app/scrcpy.1

@@ -56,6 +56,12 @@ The list of possible display ids can be listed by "adb shell dumpsys display"
 
 Default is 0.
 
+.TP
+.BI "\-\-display\-buffer ms
+Add a buffering delay (in milliseconds) before displaying. This increases latency to compensate for jitter.
+
+Default is 0 (no buffering).
+
 .TP
 .BI "\-\-encoder " name
 Use a specific MediaCodec encoder (must be a H.264 encoder).
@@ -191,6 +197,14 @@ Output to v4l2loopback device.
 
 It requires to lock the video orientation (see \fB\-\-lock\-video\-orientation\fR).
 
+.TP
+.BI "\-\-v4l2-buffer " ms
+Add a buffering delay (in milliseconds) before pushing frames. This increases latency to compensate for jitter.
+
+This option is similar to \fB\-\-display\-buffer\fR, but specific to V4L2 sink.
+
+Default is 0 (no buffering).
+
 .TP
 .BI "\-V, \-\-verbosity " value
 Set the log level ("verbose", "debug", "info", "warn" or "error").

app/src/cli.c

@@ -55,6 +55,12 @@ scrcpy_print_usage(const char *arg0) {
         "\n"
         "        Default is 0.\n"
         "\n"
+        "    --display-buffer ms\n"
+        "        Add a buffering delay (in milliseconds) before displaying.\n"
+        "        This increases latency to compensate for jitter.\n"
+        "\n"
+        "        Default is 0 (no buffering).\n"
+        "\n"
         "    --encoder name\n"
         "        Use a specific MediaCodec encoder (must be a H.264 encoder).\n"
         "\n"
@@ -182,6 +188,15 @@ scrcpy_print_usage(const char *arg0) {
         "        It requires to lock the video orientation (see\n"
         "        --lock-video-orientation).\n"
         "\n"
+        "    --v4l2-buffer ms\n"
+        "        Add a buffering delay (in milliseconds) before pushing\n"
+        "        frames. This increases latency to compensate for jitter.\n"
+        "\n"
+        "        This option is similar to --display-buffer, but specific to\n"
+        "        V4L2 sink.\n"
+        "\n"
+        "        Default is 0 (no buffering).\n"
+        "\n"
 #endif
         "    -V, --verbosity value\n"
         "        Set the log level (verbose, debug, info, warn or error).\n"
@@ -392,6 +407,19 @@ parse_max_fps(const char *s, uint16_t *max_fps) {
     return true;
 }
 
+static bool
+parse_buffering_time(const char *s, sc_tick *tick) {
+    long value;
+    bool ok = parse_integer_arg(s, &value, false, 0, 0x7FFFFFFF,
+                                "buffering time");
+    if (!ok) {
+        return false;
+    }
+
+    *tick = SC_TICK_FROM_MS(value);
+    return true;
+}
+
 static bool
 parse_lock_video_orientation(const char *s,
                              enum sc_lock_video_orientation *lock_mode) {
@@ -689,6 +717,8 @@ guess_record_format(const char *filename) {
 #define OPT_ENCODER_NAME           1025
 #define OPT_POWER_OFF_ON_CLOSE     1026
 #define OPT_V4L2_SINK              1027
+#define OPT_DISPLAY_BUFFER         1028
+#define OPT_V4L2_BUFFER            1029
 
 bool
 scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
@@ -700,6 +730,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
         {"disable-screensaver",    no_argument,       NULL,
                                                   OPT_DISABLE_SCREENSAVER},
         {"display",                required_argument, NULL, OPT_DISPLAY_ID},
+        {"display-buffer",         required_argument, NULL, OPT_DISPLAY_BUFFER},
         {"encoder",                required_argument, NULL, OPT_ENCODER_NAME},
         {"force-adb-forward",      no_argument,       NULL,
                                                   OPT_FORCE_ADB_FORWARD},
@@ -732,6 +763,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
         {"turn-screen-off",        no_argument,       NULL, 'S'},
 #ifdef HAVE_V4L2
         {"v4l2-sink",              required_argument, NULL, OPT_V4L2_SINK},
+        {"v4l2-buffer",            required_argument, NULL, OPT_V4L2_BUFFER},
 #endif
         {"verbosity",              required_argument, NULL, 'V'},
         {"version",                no_argument,       NULL, 'v'},
@@ -917,10 +949,20 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
             case OPT_POWER_OFF_ON_CLOSE:
                 opts->power_off_on_close = true;
                 break;
+            case OPT_DISPLAY_BUFFER:
+                if (!parse_buffering_time(optarg, &opts->display_buffer)) {
+                    return false;
+                }
+                break;
 #ifdef HAVE_V4L2
             case OPT_V4L2_SINK:
                 opts->v4l2_device = optarg;
                 break;
+            case OPT_V4L2_BUFFER:
+                if (!parse_buffering_time(optarg, &opts->v4l2_buffer)) {
+                    return false;
+                }
+                break;
 #endif
             default:
                 // getopt prints the error message on stderr
@@ -941,6 +983,11 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
              "See --lock-video-orientation.");
         opts->lock_video_orientation = SC_LOCK_VIDEO_ORIENTATION_INITIAL;
     }
+
+    if (opts->v4l2_buffer && !opts->v4l2_device) {
+        LOGE("V4L2 buffer value without V4L2 sink\n");
+        return false;
+    }
 #else
     if (!opts->display && !opts->record_filename) {
         LOGE("-N/--no-display requires screen recording (-r/--record)");

app/src/clock.c

@@ -0,0 +1,111 @@
+#include "clock.h"
+
+#include "util/log.h"
+
+#define SC_CLOCK_NDEBUG // comment to debug
+
+void
+sc_clock_init(struct sc_clock *clock) {
+    clock->count = 0;
+    clock->head = 0;
+    clock->left_sum.system = 0;
+    clock->left_sum.stream = 0;
+    clock->right_sum.system = 0;
+    clock->right_sum.stream = 0;
+}
+
+// Estimate the affine function f(stream) = slope * stream + offset
+static void
+sc_clock_estimate(struct sc_clock *clock,
+                  double *out_slope, sc_tick *out_offset) {
+    assert(clock->count > 1); // two points are necessary
+
+    struct sc_clock_point left_avg = {
+        .system = clock->left_sum.system / (clock->count / 2),
+        .stream = clock->left_sum.stream / (clock->count / 2),
+    };
+    struct sc_clock_point right_avg = {
+        .system = clock->right_sum.system / ((clock->count + 1) / 2),
+        .stream = clock->right_sum.stream / ((clock->count + 1) / 2),
+    };
+
+    double slope = (double) (right_avg.system - left_avg.system)
+                 / (right_avg.stream - left_avg.stream);
+
+    if (clock->count < SC_CLOCK_RANGE) {
+        /* The first frames are typically received and decoded with more delay
+         * than the others, causing a wrong slope estimation on start. To
+         * compensate, assume an initial slope of 1, then progressively use the
+         * estimated slope. */
+        slope = (clock->count * slope + (SC_CLOCK_RANGE - clock->count))
+              / SC_CLOCK_RANGE;
+    }
+
+    struct sc_clock_point global_avg = {
+        .system = (clock->left_sum.system + clock->right_sum.system)
+                 / clock->count,
+        .stream = (clock->left_sum.stream + clock->right_sum.stream)
+                 / clock->count,
+    };
+
+    sc_tick offset = global_avg.system - (sc_tick) (global_avg.stream * slope);
+
+    *out_slope = slope;
+    *out_offset = offset;
+}
+
+void
+sc_clock_update(struct sc_clock *clock, sc_tick system, sc_tick stream) {
+    struct sc_clock_point *point = &clock->points[clock->head];
+
+    if (clock->count == SC_CLOCK_RANGE || clock->count & 1) {
+        // One point passes from the right sum to the left sum
+
+        unsigned mid;
+        if (clock->count == SC_CLOCK_RANGE) {
+            mid = (clock->head + SC_CLOCK_RANGE / 2) % SC_CLOCK_RANGE;
+        } else {
+            // Only for the first frames
+            mid = clock->count / 2;
+        }
+
+        struct sc_clock_point *mid_point = &clock->points[mid];
+        clock->left_sum.system += mid_point->system;
+        clock->left_sum.stream += mid_point->stream;
+        clock->right_sum.system -= mid_point->system;
+        clock->right_sum.stream -= mid_point->stream;
+    }
+
+    if (clock->count == SC_CLOCK_RANGE) {
+        // The current point overwrites the previous value in the circular
+        // array, update the left sum accordingly
+        clock->left_sum.system -= point->system;
+        clock->left_sum.stream -= point->stream;
+    } else {
+        ++clock->count;
+    }
+
+    point->system = system;
+    point->stream = stream;
+
+    clock->right_sum.system += system;
+    clock->right_sum.stream += stream;
+
+    clock->head = (clock->head + 1) % SC_CLOCK_RANGE;
+
+    if (clock->count > 1) {
+        // Update estimation
+        sc_clock_estimate(clock, &clock->slope, &clock->offset);
+
+#ifndef SC_CLOCK_NDEBUG
+        LOGD("Clock estimation: %g * pts + %" PRItick,
+             clock->slope, clock->offset);
+#endif
+    }
+}
+
+sc_tick
+sc_clock_to_system_time(struct sc_clock *clock, sc_tick stream) {
+    assert(clock->count > 1); // sc_clock_update() must have been called
+    return (sc_tick) (stream * clock->slope) + clock->offset;
+}

app/src/clock.h

@@ -0,0 +1,70 @@
+#ifndef SC_CLOCK_H
+#define SC_CLOCK_H
+
+#include "common.h"
+
+#include <assert.h>
+
+#include "util/tick.h"
+
+#define SC_CLOCK_RANGE 32
+static_assert(!(SC_CLOCK_RANGE & 1), "SC_CLOCK_RANGE must be even");
+
+struct sc_clock_point {
+    sc_tick system;
+    sc_tick stream;
+};
+
+/**
+ * The clock aims to estimate the affine relation between the stream (device)
+ * time and the system time:
+ *
+ *     f(stream) = slope * stream + offset
+ *
+ * To that end, it stores the SC_CLOCK_RANGE last clock points (the timestamps
+ * of a frame expressed both in stream time and system time) in a circular
+ * array.
+ *
+ * To estimate the slope, it splits the last SC_CLOCK_RANGE points into two
+ * sets of SC_CLOCK_RANGE/2 points, and compute their centroid ("average
+ * point"). The slope of the estimated affine function is that of the line
+ * passing through these two points.
+ *
+ * To estimate the offset, it computes the centroid of all the SC_CLOCK_RANGE
+ * points. The resulting affine function passes by this centroid.
+ *
+ * With a circular array, the rolling sums (and average) are quick to compute.
+ * In practice, the estimation is stable and the evolution is smooth.
+ */
+struct sc_clock {
+    // Circular array
+    struct sc_clock_point points[SC_CLOCK_RANGE];
+
+    // Number of points in the array (count <= SC_CLOCK_RANGE)
+    unsigned count;
+
+    // Index of the next point to write
+    unsigned head;
+
+    // Sum of the first count/2 points
+    struct sc_clock_point left_sum;
+
+    // Sum of the last (count+1)/2 points
+    struct sc_clock_point right_sum;
+
+    // Estimated slope and offset
+    // (computed on sc_clock_update(), used by sc_clock_to_system_time())
+    double slope;
+    sc_tick offset;
+};
+
+void
+sc_clock_init(struct sc_clock *clock);
+
+void
+sc_clock_update(struct sc_clock *clock, sc_tick system, sc_tick stream);
+
+sc_tick
+sc_clock_to_system_time(struct sc_clock *clock, sc_tick stream);
+
+#endif