[mediacapture-transform] Track transferability requirement adds complexity to some common use cases (#116)

guidou has just created a new issue for https://github.com/w3c/mediacapture-transform:

== Track transferability requirement adds complexity to some common use cases ==
In many use cases it is better to keep all tracks on Window and use the worker just for the processing.

Currently, mediacapture-transform requires transferring the track to be processed to the worker, which adds complexity to these use cases.

Consider the use case of a before/after preview window where the original camera track is shown together with the processed track. With the spec as currently written, something like this is needed (omitting some variable declarations, for simplicity):

```
// main.html
function setUpWorker() {
  worker = new Worker("worker.js");
  worker.onmessage = async event => {
    if (event.data.command == "processed-track") {
      let stream = new MediaStream([event.data.param]);
      processedVideo.srcObject = stream;
      await processedVideo.play()
    } else if (event.data.command == "stopped-track") {
      track.stop();
    }
  }
}

async function startCapture() {
    let stream = await navigator.mediaDevices.getUserMedia({video:true});
    originalVideo.srcObject = stream;
    await originalVideo.play()
    track = stream.getVideoTracks()[0];
    let cloneTrack = track.clone();
    worker.postMessage({command: "process-track", param: cloneTrack}, [cloneTrack]);
}

function stopCapture() {
  worker.postMessage({command: "stop-track"}, [track]);
}

// worker.js
onmessage = event => {
    if (event.data.command == "process-track") {
        track = event.data.param;
        let processor = new MediaStreamTrackProcessor({track});
        let generator = new VideoTrackGenerator();
        processor.readable.pipeThrough(GetTransform()).pipeTo(generator.writable);
        postMessage({command: "processed-track", param: generator.track}, [generator.track]);
    } else if (event.data.command == "stop-track") {
        track.stop();
        postMessage({command: "stopped-track"});
    }
}
```

Compare to how it would be without forcing track transferability:
```
// main.js
function setUpWorker() {
  worker = new Worker("worker.js");
}

async function startCapture() {
    let stream = await navigator.mediaDevices.getUserMedia({video:true});
    originalVideo.srcObject = stream;
    await originalVideo.play();
    track = stream.getVideoTracks()[0];
    processedTrack = MediaDevices.createVideoTrackGeneratorAndProcessor(worker, track);
    processedVideo.srcObject = new MediaStream([processedTrack]);
    await processedVideo.play();
}

function stopCapture() {
  track.stop();
  processedTrack.stop();
}

// worker.js
oncapturetransform = event => {
    event.processor.readable.pipeThrough(GetTransform()).pipeTo(event.generator.writable);
}
```

Even though it is a minimal demo, the difference in complexity is considerable. Forcing track transferability requires cloning the camera track and a communication protocol between Worker and Window to keep the state consistent between them, all of which is unnecessary if the tracks can be kept on Window. 

Note that this issue is not about advocating for removing track transferability. Use cases where transferring tracks to a worker makes sense can continue to be supported without problem even if this issue is addressed. 

A runnable implementation of this demo (spec version) is available at https://guidou.github.io/mct/spec-main.html

Source code for the spec and proposed variants is available at https://github.com/guidou/guidou.github.io/tree/master/mct

Please view or discuss this issue at https://github.com/w3c/mediacapture-transform/issues/116 using your GitHub account


-- 
Sent via github-notify-ml as configured in https://github.com/w3c/github-notify-ml-config

Received on Tuesday, 19 November 2024 15:50:16 UTC