Skip to content

Commit b6b750e

Browse files
quickiwikiquickiwiki
committed
VNC-151 Refactor
1 parent 5c4e2b7 commit b6b750e

File tree

2 files changed

+89
-57
lines changed

2 files changed

+89
-57
lines changed

‎core/decoders/kasmvideo.js‎

Lines changed: 68 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ const VIDEO_CODEC_NAMES = {
1717
3: 'av01.0.04M.08'
1818
}
1919

20-
const TARGET_FPS = 60;
20+
const TARGET_FPS = 120;
2121
const FRAME_DURATION_US = Math.round(1_000_000 / TARGET_FPS);
2222
//avc1.4d002a - main
2323
/// avc1.42001E - baseline
@@ -26,36 +26,29 @@ export default class KasmVideoDecoder {
2626
constructor(display) {
2727
this._len = 0;
2828
this._keyFrame = 0;
29-
this._screenId = 0;
29+
this._screenId = null;
3030
this._ctl = null;
31-
this.codec = 0;
31+
//this.codec = 0;
3232
this._display = display;
33-
this._codedWidth = null;
34-
this._codedHeight = null;
3533

36-
this._width = null;
37-
this._height = null;
34+
//this._width = null;
35+
//this._height = null;
3836

3937
this._timestamp = 0;
4038
this._timestampMap = new Map();
41-
this._decoder = new VideoDecoder({
42-
output: (frame) => {
43-
this._handleProcessVideoChunk(frame);
44-
// frame.close();
45-
}, error: (e) => {
46-
Log.Error(`There was an error inside KasmVideoDecoder`, e)
47-
}
48-
});
39+
this._decoders = new Map();
4940
}
5041

5142
// ===== Public Methods =====
5243

44+
5345
decodeRect(x, y, width, height, sock, display, depth, frame_id) {
5446
if (this._ctl === null) {
55-
if (sock.rQwait("KasmVideo compression-control", 1)) {
47+
if (sock.rQwait("KasmVideo screen and compression-control", 2)) {
5648
return false;
5749
}
5850

51+
this._screenId = sock.rQshift8();
5952
this._ctl = sock.rQshift8();
6053

6154
// Figure out filter
@@ -67,41 +60,43 @@ export default class KasmVideoDecoder {
6760
if (this._ctl === 0x00) {
6861
ret = this._skipRect(x, y, width, height, sock, display, depth, frame_id);
6962
} else if ((this._ctl === 0x01) || (this._ctl === 0x02) || (this._ctl === 0x03)) {
70-
ret = this._processVideoFrameRect(x, y, this._ctl, width, height, sock, display, depth, frame_id);
63+
ret = this._processVideoFrameRect(this._screenId, this._ctl, x, y, width, height, sock, display, depth, frame_id);
7164
} else {
7265
throw new Error("Illegal KasmVideo compression received (ctl: " + this._ctl + ")");
7366
}
7467

7568
if (ret) {
7669
this._ctl = null;
70+
this._screenId = null;
7771
}
7872

7973
return ret;
8074
}
8175

82-
resize(codec, width, height) {
83-
this._updateSize(codec, width, height);
76+
resize(screen, codec, width, height) {
77+
this._updateSize(screen, codec, width, height);
8478
}
8579

8680
// ===== Private Methods =====
8781

88-
_configureDecoder(codec, width, height) {
89-
this._decoder.configure({
90-
codec: VIDEO_CODEC_NAMES[codec],
91-
codedWidth: width,
92-
codedHeight: height,
82+
_configureDecoder(screen) {
83+
Log.Debug('Configuring decoder for screen: ', screen.id, ' codec: ', VIDEO_CODEC_NAMES[screen.codec], ' width: ', screen.width, ' height: ', screen.height);
84+
screen.decoder.configure({
85+
codec: VIDEO_CODEC_NAMES[screen.codec],
86+
codedWidth: screen.width,
87+
codedHeight: screen.height,
9388
optimizeForLatency: true,
9489
})
9590
}
9691

97-
_updateSize(codec, width, height) {
92+
_updateSize(screen, codec, width, height) {
9893
Log.Debug('Updated size: ', {width, height});
9994

100-
this._width = width;
101-
this._height = height;
102-
this.codec = codec;
95+
screen.width = width;
96+
screen.height = height;
97+
screen.codec = codec;
10398

104-
this._configureDecoder(codec, width, height);
99+
this._configureDecoder(screen);
105100
}
106101

107102
_skipRect(x, y, width, height, _sock, display, _depth, frame_id) {
@@ -112,45 +107,77 @@ export default class KasmVideoDecoder {
112107
_handleProcessVideoChunk(frame) {
113108
Log.Debug('Frame ', frame);
114109
const {frame_id, x, y, width, height} = this._timestampMap.get(frame.timestamp);
110+
Log.Debug('frame_id: ', frame_id, 'x: ', x, 'y: ', y, 'coded width: ', frame.codedWidth, 'coded height: ', frame.codedHeight);
115111
this._display.videoFrameRect(frame, frame_id, x, y, width, height);
116112
this._timestampMap.delete(frame.timestamp);
117113
}
118114

119-
_processVideoFrameRect(x, y, codec, width, height, sock, display, depth, frame_id) {
115+
_processVideoFrameRect(screenId, codec, x, y, width, height, sock, display, depth, frame_id) {
120116
let [keyFrame, dataArr] = this._readData(sock);
121-
Log.Debug('key_frame: ', keyFrame);
117+
Log.Debug('Screen: ', screenId, ' key_frame: ', keyFrame);
122118
if (dataArr === null) {
123119
return false;
124120
}
125121

126-
if (width !== this._width && height !== this._height || codec !== this.codec)
127-
this._updateSize(codec, width, height)
122+
let screen;
123+
if (this._decoders.has(screenId)) {
124+
screen = this._decoders.get(screenId);
125+
} else {
126+
screen = {
127+
id: screenId,
128+
width: width,
129+
height: height,
130+
decoder: new VideoDecoder({
131+
output: (frame) => {
132+
this._handleProcessVideoChunk(frame);
133+
// frame.close();
134+
}, error: (e) => {
135+
Log.Error(`There was an error inside KasmVideoDecoder`, e)
136+
}
137+
})
138+
};
139+
Log.Debug('Created new decoder for screen: ', screenId);
140+
this._decoders.set(screenId, screen);
141+
}
142+
143+
if (width !== screen.width && height !== screen.height || codec !== screen.codec)
144+
this._updateSize(screen, codec, width, height)
128145

129146
const vidChunk = new EncodedVideoChunk({
130147
type: keyFrame ? 'key' : 'delta',
131148
data: dataArr,
132149
timestamp: this._timestamp,
133150
});
151+
152+
Log.Debug('Type ', vidChunk.type, ' timestamp: ', vidChunk.timestamp, ' bytelength ', vidChunk.byteLength);
153+
134154
this._timestampMap.set(this._timestamp, {
155+
screenId,
135156
frame_id,
136157
x,
137158
y,
138159
width,
139160
height
140161
});
141162
this._timestamp += FRAME_DURATION_US;
142-
this._decoder.decode(vidChunk);
163+
164+
try {
165+
screen.decoder.decode(vidChunk);
166+
} catch (e) {
167+
Log.Error('Screen: ', screenId,
168+
'Key frame ', keyFrame, ' frame_id: ', frame_id, ' x: ', x, ' y: ', y, ' width: ', width, ' height: ', height, ' codec: ', codec, ' ctl ', this._ctl, ' dataArr: ', dataArr, ' error: ', e);
169+
Log.Error('There was an error inside KasmVideoDecoder: ', e)
170+
}
143171
return true;
144172
}
145173

146174
_readData(sock) {
147175
if (this._len === 0) {
148-
if (sock.rQwait("KasmVideo", 4)) {
176+
if (sock.rQwait("KasmVideo", 5)) {
149177
return [0, null];
150178
}
151179

152180
this._keyFrame = sock.rQshift8();
153-
// this._screenId = sock.rQshift8();
154181
let byte = sock.rQshift8();
155182
this._len = byte & 0x7f;
156183
if (byte & 0x80) {
@@ -167,15 +194,17 @@ export default class KasmVideoDecoder {
167194
return [0, null];
168195
}
169196

170-
let data = sock.rQshiftBytes(this._len);
171-
let keyFrame = this._keyFrame;
197+
const data = sock.rQshiftBytes(this._len);
198+
const keyFrame = this._keyFrame;
172199
this._len = 0;
173200
this._keyFrame = 0;
174201

175202
return [keyFrame, data];
176203
}
177204

178205
dispose() {
179-
this._decoder.close();
206+
for (let screen of this._decoders.values()) {
207+
screen.decoder.close();
208+
}
180209
}
181210
}

‎core/display.js‎

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -937,7 +937,7 @@ export default class Display {
937937
drawImage(img, x, y, w, h, overlay=false) {
938938
try {
939939
let targetCtx = ((this._enableCanvasBuffer && !overlay) ? this._drawCtx : this._targetCtx);
940-
if (img.width != w || img.height != h) {
940+
if (img.width !== w || img.height !== h) {
941941
targetCtx.drawImage(img, x, y, w, h);
942942
} else {
943943
targetCtx.drawImage(img, x, y);
@@ -1382,21 +1382,6 @@ export default class Display {
13821382
}, [a.img]);
13831383
}
13841384
break;
1385-
case 'vid':
1386-
secondaryScreenRects++;
1387-
if (this._screens[screenLocation.screenIndex].channel) {
1388-
this._screens[screenLocation.screenIndex].channel.postMessage({
1389-
eventType: 'rect',
1390-
rect: {
1391-
'type': 'video_frame',
1392-
'img': a.frame,
1393-
'frame_id': a.frame_id,
1394-
'screenLocations': a.screenLocations
1395-
},
1396-
screenLocationIndex: sI
1397-
}, [a.frame]);
1398-
}
1399-
break;
14001385
case 'blit':
14011386
secondaryScreenRects++;
14021387
let buf = a.data.buffer;
@@ -1418,6 +1403,24 @@ export default class Display {
14181403
}, [buf]);
14191404
}
14201405
break;
1406+
case 'video_frame':
1407+
secondaryScreenRects++;
1408+
if (a.frame.format !== null)
1409+
this._screens[screenLocation.screenIndex]?.channel.postMessage({
1410+
eventType: 'rect',
1411+
rect: {
1412+
type: 'video_frame',
1413+
frame: a.frame,
1414+
x: a.x,
1415+
y: a.y,
1416+
width: a.width,
1417+
height: a.height,
1418+
frame_id: a.frame_id,
1419+
screenLocations: a.screenLocations
1420+
},
1421+
screenLocationIndex: sI
1422+
}, [a.frame]);
1423+
break;
14211424
case 'img':
14221425
case '_img':
14231426
secondaryScreenRects++;
@@ -1442,7 +1445,7 @@ export default class Display {
14421445
default:
14431446
secondaryScreenRects++;
14441447
if (a instanceof HTMLImageElement) {
1445-
Log.Warn("Wrong rect type: " + rect.type);
1448+
Log.Warn("Wrong rect type: " + a.type);
14461449
} else {
14471450
if (this._screens[screenLocation.screenIndex].channel) {
14481451
this._screens[screenLocation.screenIndex].channel.postMessage({
@@ -1497,7 +1500,7 @@ export default class Display {
14971500
if (this._asyncFrameQueue[0][2].length > 0) {
14981501
window.requestAnimationFrame( () => { this._pushAsyncFrame(); });
14991502
}
1500-
} else if (this._asyncFrameQueue[0][1] > 0 && this._asyncFrameQueue[0][1] == this._asyncFrameQueue[0][2].length) {
1503+
} else if (this._asyncFrameQueue[0][1] > 0 && this._asyncFrameQueue[0][1] === this._asyncFrameQueue[0][2].length) {
15011504
//how many times has _pushAsyncFrame been called when the frame had all rects but has not been drawn
15021505
this._asyncFrameQueue[0][5] += 1;
15031506
//force the frame to be drawn if it has been here too long

0 commit comments

Comments
 (0)