- From: guest271314 via GitHub <sysbot+gh@w3.org>
- Date: Fri, 21 Aug 2020 01:06:37 +0000
- To: public-webrtc-logs@w3.org
While the relevant parties are here, where is the most suitable venue to formally propose
`Response.mediaStream()` and `Response.mediaStreamTrack()`?
For example, given PCM, we can use `AudioWorkletProcessor` and `MediaStreamAudioSourceDestinatioNode` to do something like
```
<doctype html>
<html>
<body>
<button>start</button><button>stop</button>
<script>
if (globalThis.gc) {
// gc();
}
let controller, signal;
// throws memory allocation error, use multiple Memory instances
// const memory = new WebAssembly.Memory({initial: 8062, maximum: 9675, shared: true});
const [start, stop] = document.querySelectorAll('button');
start.onclick = async e => {
class AudioWorkletProcessor {}
class AudioWorkletNativeFileStream extends AudioWorkletProcessor {
constructor(options) {
super(options);
this.byteSize = 512 * 344 * 60 * 50;
this.memory = new Uint8Array(this.byteSize);
Object.assign(this, options.processorOptions);
this.port.onmessage = this.appendBuffers.bind(this);
}
async appendBuffers({ data: readable }) {
Object.assign(this, { readable });
const source = {
write: (value, c) => {
if (this.totalBytes < this.byteSize) {
this.memory.set(value, this.readOffset);
this.readOffset = this.readOffset + value.buffer.byteLength;
this.totalBytes = this.readOffset;
} else {
console.log(
value.buffer.byteLength,
this.readOffset,
this.totalBytes
);
}
if (this.totalBytes >= (384 * 512) / 2 && !this.started) {
this.started = true;
this.port.postMessage({ start: this.started });
}
},
close() {
console.log('stopped');
},
};
try {
await this.readable.pipeTo(new WritableStream(source));
} catch (e) {
console.warn(e);
console.log(this.writeOffset, this.totalBytes);
this.endOfStream();
}
}
endOfStream() {
this.port.postMessage({
ended: true,
currentTime,
currentFrame,
readOffset: this.readOffset,
readIndex: this.readIndex,
writeOffset: this.writeOffet,
writeIndex: this.writeIndex,
totalBytes: this.totalBytes,
});
}
process(inputs, outputs) {
const channels = outputs.flat();
if (
this.writeOffset >= this.totalBytes ||
this.totalBytes === this.byteSize
) {
console.log(this);
this.endOfStream();
return false;
}
const uint8 = new Uint8Array(512);
try {
for (let i = 0; i < 512; i++, this.writeOffset++) {
if (this.writeOffset === this.byteSize) {
break;
}
uint8[i] = this.memory[this.writeOffset];
// ++this.writeOffset;
}
const uint16 = new Uint16Array(uint8.buffer);
// based on int16ToFloat32 function at https://stackoverflow.com/a/35248852
for (let i = 0, j = 0, n = 1; i < uint16.length; i++) {
const int = uint16[i];
// If the high bit is on, then it is a negative number, and actually counts backwards.
const float =
int >= 0x8000 ? -(0x10000 - int) / 0x8000 : int / 0x7fff;
// interleave
channels[(n = ++n % 2)][!n ? j++ : j - 1] = float;
}
} catch (e) {
console.error(e);
}
return true;
}
}
// register processor in AudioWorkletGlobalScope
function registerProcessor(name, processorCtor) {
return `${processorCtor};\nregisterProcessor('${name}', ${processorCtor.name});`;
}
const worklet = URL.createObjectURL(
new Blob(
[
registerProcessor(
'audio-worklet-native-file-stream',
AudioWorkletNativeFileStream
),
],
{ type: 'text/javascript' }
)
);
const ac = new AudioContext({
latencyHint: 1,
sampleRate: 44100,
numberOfChannels: 2,
});
ac.onstatechange = e => console.log(ac.state);
if (ac.state === 'running') {
await ac.suspend();
}
await ac.audioWorklet.addModule(worklet);
const aw = new AudioWorkletNode(
ac,
'audio-worklet-native-file-stream',
{
numberOfInputs: 1,
numberOfOutputs: 2,
channelCount: 2,
processorOptions: {
totalBytes: 0,
readIndex: 0,
writeIndex: 0,
readOffset: 0,
writeOffset: 0,
done: false,
ended: false,
started: false,
},
}
);
aw.onprocessorerror = e => {
console.error(e);
console.trace();
};
// comment MediaStream, MediaStreamTrack creation
const msd = new MediaStreamAudioDestinationNode(ac);
const { stream } = msd;
const [track] = stream.getAudioTracks();
aw.connect(msd);
// aw.connect(ac.destination);
const recorder = new MediaRecorder(stream);
recorder.ondataavailable = e =>
console.log(URL.createObjectURL(e.data));
aw.port.onmessage = async e => {
console.log(e.data);
if (
e.data.start &&
ac.state === 'suspended'
) {
await ac.resume();
recorder.start();
} else {
if (recorder.state === 'recording') {
recorder.stop();
track.stop();
aw.disconnect();
msd.disconnect();
await ac.suspend();
console.log(track);
// gc();
}
}
};
try {
start.disabled = true;
controller = new AbortController();
signal = controller.signal;
const { body: readable } = await fetch(
'http://localhost:8000?start=true',
{
cache: 'no-store',
signal,
}
);
aw.port.postMessage(readable, [readable]);
} catch (e) {
console.warn(e);
} finally {
}
};
stop.onclick = e => {
controller.abort();
start.disabled = false;
};
</script>
</body>
</html>
```
which is essentially a stream of raw PCM. Opus, Vorbis, AAC, could also be supported for audio.
--
GitHub Notification of comment by guest271314
Please view or discuss this issue at https://github.com/w3c/webrtc-insertable-streams/issues/40#issuecomment-677981573 using your GitHub account
--
Sent via github-notify-ml as configured in https://github.com/w3c/github-notify-ml-config
Received on Friday, 21 August 2020 01:06:39 UTC