fftools/ffmpeg_sched: prevent demuxers from getting stuck

When the furthest-behind stream is being fed by a demuxer that is also
feeding packets to a choked filter graph, we need to unchoke that filter
graph to prevent the demuxer from getting stuck trying to write packets to
the choked filter graph.

This situation can also apply recursively - if the demuxer is also writing
to a filtergraph that is also reading from a choked demuxer, there is a
similar deadlock.

Solve all such deadlocks by just brute-force recursively unchoking all
nodes that can somehow prevent this demuxer from writing packets. This
should normally not result in any change in behavior, unless audio/video
streams are badly desynchronized, in which case it may result in extra
memory usage from the too-far-ahead stream buffering packets inside the
muxer. (But this is, of course, preferable to a deadlock)

Fixes: https://code.ffmpeg.org/FFmpeg/FFmpeg/issues/20611

fftools/ffmpeg_sched.c

@@ -1282,14 +1282,59 @@ int sch_mux_sub_heartbeat_add(Scheduler *sch, unsigned mux_idx, unsigned stream_
     return 0;
 }
 
+static void unchoke_for_stream(Scheduler *sch, SchedulerNode src);
+
+// Unchoke any filter graphs that are downstream of this node, to prevent it
+// from getting stuck trying to push data to a full queue
+static void unchoke_downstream(Scheduler *sch, SchedulerNode *dst)
+{
+    SchFilterGraph *fg;
+    SchDec *dec;
+    SchEnc *enc;
+    switch (dst->type) {
+    case SCH_NODE_TYPE_DEC:
+        dec = &sch->dec[dst->idx];
+        for (int i = 0; i < dec->nb_outputs; i++)
+            unchoke_downstream(sch, dec->outputs[i].dst);
+        break;
+    case SCH_NODE_TYPE_ENC:
+        enc = &sch->enc[dst->idx];
+        for (int i = 0; i < enc->nb_dst; i++)
+            unchoke_downstream(sch, &enc->dst[i]);
+        break;
+    case SCH_NODE_TYPE_MUX:
+        // muxers are never choked
+        break;
+    case SCH_NODE_TYPE_FILTER_IN:
+        fg = &sch->filters[dst->idx];
+        if (fg->best_input == fg->nb_inputs) {
+            fg->waiter.choked_next = 0;
+        } else {
+            // ensure that this filter graph is not stuck waiting for
+            // input from a different upstream demuxer
+            unchoke_for_stream(sch, fg->inputs[fg->best_input].src);
+        }
+        break;
+    default:
+        av_unreachable("Invalid destination node type?");
+        break;
+    }
+}
+
 static void unchoke_for_stream(Scheduler *sch, SchedulerNode src)
 {
     while (1) {
         SchFilterGraph *fg;
+        SchDemux *demux;
         switch (src.type) {
         case SCH_NODE_TYPE_DEMUX:
             // fed directly by a demuxer (i.e. not through a filtergraph)
-            sch->demux[src.idx].waiter.choked_next = 0;
+            demux = &sch->demux[src.idx];
+            if (demux->waiter.choked_next == 0)
+                return; // prevent infinite loop
+            demux->waiter.choked_next = 0;
+            for (int i = 0; i < demux->nb_streams; i++)
+                unchoke_downstream(sch, demux->streams[i].dst);
             return;
         case SCH_NODE_TYPE_DEC:
             src = sch->dec[src.idx].src;