Zhaopeng's Homepage

My tech notebook

3D character motion control via WebSocket

Posted on In

Several days ago, a friend of mine contacted me and asked me the feasibility of a technical solution for a 3d human character simulation in HTML5 environment. He sent me this article which presents how to create an interactive 3d character with Three.js**. He is expecting to control the character’s real-time motion (whole-body motion) via a hardware such like a joystick.

It’s a very interesting work and it seems quite easy. Thus I have done a little dev work trying to make it work.

Alt Text

In file Index.html, I have defined a websocket server and an according message parser. This file then is wrapped in Electron window so it runs as a desktop software.
Core part of this Index.html is the websocket communication part as below:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
<script  type="text/javascript" >
	var angle1 = 0.0;
	var angle2 = 0.0
	
	const qEvent = new Event('qiu');
	
	/* for debug */    
	function output(s)
	{
		var out = document.getElementById("debug-area");
		out.innerText += s;
	}
	
	output("Start running")
		

	var msg_ready = false;
	var msg_reading = false;  // True: package head 0xAA is received, but 0x7f has not yet been received
	var msg_data_buffer = [];
	var msg_lenth = 0;


	function processMsg(v)
	{		
	if (v[0] == 170) // detect the beginning byte of a message: 0xAA
	{
		// data are sent in little endian, 
		// v.buffer is a byte-array and Int16Array(v.buffer, 8, 1) means that it parses from the 8th byte on to get ONE Int16 number
               
		if ( (v[1] == 0x01) && (v[2] == 0x53) ) // 01 52
		{
			angle1 = new Int16Array(v.buffer, 8, 1)[0];
			angle2 = new Int16Array(v.buffer, 10, 1)[0];
			var temp3 = new Int16Array(v.buffer, 12, 1)[0];
			document.dispatchEvent(qEvent);

		}
		else 
		{
		}
	 }
	}
	

	var ws = require("nodejs-websocket");
	var clients =  new Array();
	output("开始建立连接... ");
	var count = 0;
	var data = new Buffer.alloc(0);
	var server = ws.createServer(function(conn){

		conn.id = count;
		count += 1;
		clients["conn"+count]  = conn;

		conn.on("text", function (str) {
			output("Received " + str + "! " )
			var typeId = str.charAt(0);			
			conn.sendText('Success!');
		})
		conn.on("close", function (code, reason) {
			output("Connection closed!")
			//clients.delete(conn);
		});
		
		conn.on("binary", function (inStream) {
		
			inStream.on("readable", function () {
				var newData = inStream.read();
				
				if (newData)
					data = Buffer.concat([data, newData], data.length + newData.length);
			});
			
			inStream.on("end", function () {

				if(data){
				var t = '', v = new Uint8Array(data);

				for (var i = 0; i < v.length; i++)
				{
					// packet head 0xAA reached, now start reading the data flow
					if  ((!msg_reading ) &&(v[i] == 0xaa)){
						msg_reading = true;
					}
					
					
					if(msg_reading){
						
						if (msg_data_buffer.length == 8) {
							msg_lenth =  msg_data_buffer[5]*16 + msg_data_buffer[4]; // parsing the data length (bytes size)							
						}
						
						// received the end of packet, and the length is correct 
						if ((v[i] == 127 ) && (msg_data_buffer.length == (msg_lenth + 10)))  // 10 extra bytes contained in this package for : length, scope, checksum, msg-id 
						{							
							var msg = new Uint8Array(msg_data_buffer);
							processMsg(msg);
							msg_data_buffer = [];
							msg_reading = false;
							msg_lenth = 0;
						} else if (msg_data_buffer.length == (msg_lenth + 10))
						{
							msg_data_buffer = [];
							msg_reading = false;
							msg_lenth = 0;
							output("Message length error!");
						}
						else{
							msg_data_buffer.push(v[i]);
						}
					}
				}
									
			}else{
				
			};
			data = new Buffer.alloc(0);
			conn.sendText('Binary Received!');
			});
			
			
		});
		conn.on("message", function (code, reason) {
			output("message! " )
		});
		conn.on("error", function (code, reason) {
			output("Error occurs!")
		});
	}).listen(9999)
	output("Server is ready! ");
</script>

In existing file script.js, I have defined function moveOneJoint(). For the moment for test only, this function change only the orientation of the head (namely neck joint). It will be called each time an event ‘qiu’ is dispatched.

1
2
3
4
5
6
7
8
9
10
11
12
13
14

document.addEventListener('qiu', function (e) {
  
  if (neck && waist) {
     moveOneJoint(neck, angle1, angle2);
  }
});

function moveOneJoint(joint, x, y) {

  joint.rotation.y = THREE.Math.degToRad(x);
  joint.rotation.x = THREE.Math.degToRad(y);

}

Entire code has been pushed to github repo:
https://github.com/QiuZhaopeng/3d_character_simulation

I do not have a joystick so I simulate it with several range sliders in another web app (developed using MUI framework with HBuilder). By sliding the sliders, we can send the angle data via websocket to above-mentioned 3d character simulator. Data massage to be sent should be a dataarray like: [0xAA, 0x01,0x53, 0x01, 0x04,0x00,0x00,0x00, 0xMM,0xNN, 0xSS,0xTT, 0xYY,0xZZ, 0x7F] where 0xMM,0xNN and 0xSS,0xTT are angle values in Int16 and 0xYY,0xZZ can be any bytes (designed to be checksum, but I am not checking it in my code).

Below is a demo I’ve recorded. I am controlling the motion of the simulated 3d character’s head using sliders:

In another trial, I run my device simulator app on Android platform and run Electron in full screen. Check out the demo :