diff --git a/exercise/index.js b/exercise/index.js index ea6e584..fa5305b 100755 --- a/exercise/index.js +++ b/exercise/index.js @@ -1,519 +1,586 @@ var Kinect2 = require('kinect2'), - express = require('express'), - app = express(), - server = require('http').createServer(app), - io = require('socket.io').listen(server); - XLSX = require('xlsx'); + express = require('express'), + app = express(), + server = require('http').createServer(app), + io = require('socket.io').listen(server); +XLSX = require('xlsx'); const fs = require('fs'); var kinect = new Kinect2(); var clients = 0; -if(kinect.open()) { - // Server Initiation - server.listen(8000); - console.log('Server listening on port 8000'); - console.log('Point your browser to http://localhost:8000'); - app.use(express.static('public')) - - // Initiation - - // States and sub-state, global variables in server - var systemState = 3, // 3 is the initiation code || 1: recording, 2: display, 0: live, 3: init == live - activityLabeled = false; - - // Data Storage, global variables in server - var bufferTrial= [], // trial is a number of activities, including ground truth (gt) and exercises - bufferBodyFrames =[], gtArray = [], exArray = []; - - // Start Time for the test - var startTime, duration; - var bodyIndex = -1; - - console.log('system init state'); - kinect.openBodyReader(); - - // Connection On: - io.on('connection', function(socket){ - ++clients; - console.log('a user connected'); - // systemState could be 0..3 during connection event, but only 2 needs signal emission - if (systemState == 2) { - socket.emit('disp',bufferBodyFrames,systemState, bodyIndex, activityLabeled); - } - - // State Transition controlled by the client's command - socket.on('command',function(){ - systemState = StateTrans(systemState); - switch (systemState) { // During the transition, prepare the buffer - case 1: // 0->1 or 3->1 Get ready for recording - activityLabeled = false; - bodyIndex = locateBodyTrackedIndex(bufferBodyFrames); - bufferBodyFrames = []; - console.log('System in Recording state'); - startTime = new Date().getTime(); - break; - - case 2: // 1->2, Push the BodyFrames Data to the current trial - duration = ((new Date().getTime() - startTime)/1000).toFixed(2); - kinect.closeBodyReader(); // if closeBodyReader is called twice, the server is down. - bufferBodyFrames.durationsecs = duration; - bufferBodyFrames.duration = duration.toString(); - bufferBodyFrames.bodyIndex = bodyIndex; - bodyIndex = -1; - //console.log(JSON.stringify(bufferBodyFrames)); - bufferTrial.push(bufferBodyFrames); - console.log('system in Result Display state'); // Action - socket.emit('disp',bufferBodyFrames,systemState, bodyIndex, activityLabeled); // activityLabeled should be false because recording is just ended - socket.broadcast.emit('disp',bufferBodyFrames,systemState, bodyIndex, activityLabeled); - break; - - case 0: // 2->0, get the system from Result Disp to Live state. - kinect.openBodyReader();// No Other Specific Actions in this block because it is done by kinect.on() - console.log('system in Live state'); - default: - } - }); - // Speical Buttons: label buttons(3), report button(1), save(1) & curve show(1) - socket.on('dataLabelFromClient',function(num){ // label reference, exercise or discard - testID = bufferTrial.length-1; - if (gtArray[gtArray.length-1] == testID) { gtArray.pop(); } - if (exArray[exArray.length-1] == testID) { exArray.pop(); } - if (num == 1) { gtArray.push(testID); } - else{ if (num == 2) { exArray.push(testID); } } - activityLabeled = true; - socket.emit('serverDataLabeled'); - socket.broadcast.emit('serverDataLabeled'); - }) - - socket.on('analyze',function(){ - console.log('analyze signal received!'); - var chartData = chartAnalyze(bufferTrial,gtArray,exArray); - var barData = barAnalyze(bufferTrial, gtArray, exArray); - socket.emit('report',chartData, barData, gtArray, exArray); +if (kinect.open()) { + // Server Initiation + server.listen(8000); + console.log('Server listening on port 8000'); + console.log('Point your browser to http://localhost:8000'); + app.use(express.static('public')) + + + // States and sub-state, global variables in server + var systemState = 3, // 3 is the initiation code || 1: recording, 2: display, 0: live, 3: init == live + activityLabeled = false; + + // Data Storage, global variables in server + var bufferTrial = [], // trial is a number of activities, including ground truth (gt) and exercises + bufferBodyFrames = [], gtArray = [], exArray = []; + + // Variables used in calculating speed + var startTime, duration; + var bodyIndex = -1; + + console.log('system init state'); + kinect.openBodyReader(); + + // Connection On: + io.on('connection', function (socket) { + ++clients; + console.log('a user connected'); + // systemState could be 0..3 during connection event, but only 2 needs signal emission + if (systemState == 2) { + socket.emit('disp', bufferBodyFrames, systemState, bodyIndex, activityLabeled); + } + + // State Transition controlled by the client's command + socket.on('command', function () { + systemState = StateTrans(systemState); + switch (systemState) { // During the transition, prepare the buffer + case 1: // 0->1 or 3->1 Get ready for recording + activityLabeled = false; + bodyIndex = locateBodyTrackedIndex(bufferBodyFrames); + bufferBodyFrames = []; + console.log('System in Recording state'); + startTime = new Date().getTime(); + break; + + case 2: // 1->2, Push the BodyFrames Data to the current trial + duration = ((new Date().getTime() - startTime)).toFixed(2); + kinect.closeBodyReader(); // if closeBodyReader is called twice, the server is down. + bufferBodyFrames.durationsecs = duration / 1000; + bufferBodyFrames.duration = duration.toString(); + bufferBodyFrames.bodyIndex = bodyIndex; + bodyIndex = -1; + bufferTrial.push(bufferBodyFrames); + console.log('system in Result Display state'); // Action + socket.emit('disp', bufferBodyFrames, systemState, bodyIndex, activityLabeled); // activityLabeled should be false because recording is just ended + socket.broadcast.emit('disp', bufferBodyFrames, systemState, bodyIndex, activityLabeled); + break; + + case 0: // 2->0, get the system from Result Disp to Live state. + kinect.openBodyReader();// No Other Specific Actions in this block because it is done by kinect.on() + console.log('system in Live state'); + default: + } + }); + // Special Buttons: label buttons(3), report button(1), save(1) & curve show(1) + socket.on('dataLabelFromClient', function (num) { // label reference, exercise or discard + testID = bufferTrial.length - 1; + if (gtArray[gtArray.length - 1] == testID) { + gtArray.pop(); + } + if (exArray[exArray.length - 1] == testID) { + exArray.pop(); + } + if (num == 1) { + gtArray.push(testID); + get_ref_info(bufferTrial); + } + else { + if (num == 2) { + exArray.push(testID); + } + } + activityLabeled = true; + socket.emit('serverDataLabeled'); + socket.broadcast.emit('serverDataLabeled'); + }); + + // For showing joint speed with speedometer + socket.on('joint-speed', function (joint_num) { + console.log('inside joint speed socket on'); + var speed = getSpeed(bufferTrial, bufferTrial.length - 1, joint_num); + socket.emit('showing-speed', speed); + }); + + + socket.on('analyze', function () { + console.log('analyze signal received!'); + var chartData = chartAnalyze(bufferTrial, gtArray, exArray); + var barData = barAnalyze(bufferTrial, gtArray, exArray); + socket.emit('report', chartData, barData, gtArray, exArray); + }); + + socket.on('saveRequest', function (filename) { + save2xlsx(bufferTrial, gtArray, exArray, filename); + }); + + socket.on('curveRequest', function (gtInd, exInd, jt, datatype) { + var curveData = curveAnalyze(bufferTrial, gtArray, exArray, gtInd, exInd, jt, datatype); + socket.emit('curveResult', curveData); + }); + + // States + kinect.on('bodyFrame', function (bodyFrame) { + + switch (systemState) { + case 1: //recording: save the data being recorded, give identification to client + socket.emit('rec', bodyFrame, systemState, bodyIndex); + socket.broadcast.emit('rec', bodyFrame, systemState, bodyIndex); + bufferBodyFrames.push(bodyFrame); // save the bodyFrame by pushing it to buffer + break; + + case 2: //display + console.log('system in display state, but system is streaming. Something wrong here.'); + break; + + case 0: //live + bufferBodyFrames = []; + bufferBodyFrames.push(bodyFrame); // clean buffer and push the current bodyFrame to buffer. It is used to find the (1) bodyIndex to track. + socket.emit('live', bodyFrame, systemState) + socket.broadcast.emit('live', bodyFrame, systemState); + break; + + case 3: //3 init + bufferBodyFrames = []; + bufferBodyFrames.push(bodyFrame); + socket.emit('init', bodyFrame, systemState); + socket.broadcast.emit('init', bodyFrame, systemState); + break; + + default: + console.log('System State unknown'); + } + }); + + // disconnect + socket.on('disconnect', function () { + console.log('a user disconnect'); + --clients; + }); }); - socket.on('saveRequest',function(filename){ - save2xlsx(bufferTrial,gtArray,exArray,filename); - }); - - socket.on('curveRequest',function(gtInd,exInd,jt,datatype){ - var curveData = curveAnalyze(bufferTrial,gtArray,exArray,gtInd,exInd,jt,datatype); - socket.emit('curveResult',curveData); - }); - - // States - kinect.on('bodyFrame', function(bodyFrame){ - console.log("new bodyframe received..."); - console.log(JSON.stringify(bodyFrame)) ; - - switch (systemState) { - case 1: //recording: save the data being recorded, give identification to client - socket.emit('rec', bodyFrame, systemState, bodyIndex); - socket.broadcast.emit('rec', bodyFrame, systemState, bodyIndex); - bufferBodyFrames.push(bodyFrame); // save the bodyFrame by pushing it to buffer - break; - - case 2: //display - console.log('system in display state, but system is streaming. Something wrong here.'); - break; - - case 0: //live - bufferBodyFrames = []; - bufferBodyFrames.push(bodyFrame); // clean buffer and push the current bodyFrame to buffer. It is used to find the (1) bodyIndex to track. - socket.emit('live', bodyFrame,systemState) - socket.broadcast.emit('live', bodyFrame,systemState); - break; - - case 3: //3 init - bufferBodyFrames = []; - bufferBodyFrames.push(bodyFrame); - socket.emit('init', bodyFrame, systemState); - socket.broadcast.emit('init', bodyFrame, systemState); - break; - - default: - console.log('System State unknown'); - }//end of switch - }); // end of kinect.on('bodyframe',function) - - // disconnect - socket.on('disconnect',function(){ - console.log('a user disconnect'); - --clients; - }) - }); // end of io.on('connection',function) -}//end of kinect.open() - - -// Functions ---------------------------------------------------------------------- + +// Helper functions + +function get_ref_info(bufferTrial, joint=0){ + // Used for testing for max, min vals for reps + var temp_pts = {'neck_y':[],'squat_y':[]} + var buffer_temp = bufferTrial[0]; + + for (var i =0; i < buffer_temp.length; i++){ + var bodies_temp = buffer_temp[i]['bodies']; + for (var j = 0; j < bodies_temp.length; j++){ + if (bodies_temp[j].hasOwnProperty('joints')){ + var temp_bj = bodies_temp[j]['joints']; + temp_pts['neck_y'].push(temp_bj[2]['depthY']) + temp_pts['squat_y'].push(temp_bj[joint]['depthY']) + } + } + } + + // Save full signal to json + require('fs').writeFile( + './reference.json', + + JSON.stringify(temp_pts['squat_y']), + + function (err) { + if (err) { + console.error('Error in saving file'); + } + } + ); + + // Log max, min, average to console + const average = arr => arr.reduce( ( p, c ) => p + c, 0 ) / arr.length; + + console.log('min squat', Math.min.apply(Math, temp_pts['squat_y'].filter(Boolean))) + console.log('max squat', Math.max.apply(Math,temp_pts['squat_y'].filter(Boolean))) + console.log('mean squat', average(temp_pts['squat_y'].filter(Boolean))) +} // Define the legal move between states -function StateTrans(st){ - return (st+1)%3; +function StateTrans(st) { + return (st + 1) % 3; } -function locateBodyTrackedIndex(bufferBodyFrames){ - var ind = -1; - for (var i=0; i<=5; i++){ - if (bufferBodyFrames[0].bodies[i].tracked){ // tracked in the first frame - ind = i; - break; - } - } - return ind; +function locateBodyTrackedIndex(bufferBodyFrames) { + var ind = -1; + for (var i = 0; i <= 5; i++) { + if (bufferBodyFrames[0].bodies[i].tracked) { // tracked in the first frame + ind = i; + break; + } + } + return ind; } -function typeofTest(bufferTrial,id){ - var type // 0: Hand, 1: Squat - var typeList = ["Hand","Squat"]; - if (getAmplitudeY(bufferTrial,id,0) > 0.15) //Base of spine moves > 0.15m - {type = "Squat";} - else - { type = "Hand";} - return type; +function typeofTest(bufferTrial, id) { + var type // 0: Hand, 1: Squat + if (getAmplitudeY(bufferTrial, id, 0) > 0.15) //Base of spine moves > 0.15m + { + type = "Squat"; + } + else { + type = "Hand"; + } + return type; } -function getRaw(bufferTrial,id,jt,datatype){ - var frames = bufferTrial[id]; - var ind = frames.bodyIndex, time = frames.durationsecs; - var rawdata = [] - for(var i=0; i90) {T_x += -180;} - if (T_y>180) {T_y += -180;} - if (T_z>180) {T_z += -180;} - if (datatype == 3) - {rawdata.push(T_x);} - if (datatype == 4) - {rawdata.push(T_y);} - if (datatype == 5) - {rawdata.push(T_z);} - - } - } - return rawdata; +function getRaw(bufferTrial, id, jt, datatype) { + var frames = bufferTrial[id]; + var ind = frames.bodyIndex, time = frames.durationsecs; + var rawdata = [] + for (var i = 0; i < frames.length; i++) { + if (datatype == 0) { + rawdata.push(frames[i].bodies[ind].joints[jt].cameraX); + } + if (datatype == 1) { + rawdata.push(frames[i].bodies[ind].joints[jt].cameraY); + } + if (datatype == 2) { + rawdata.push(frames[i].bodies[ind].joints[jt].cameraZ); + } + else { + var x = frames[i].bodies[ind].joints[jt].orientationX; + var y = frames[i].bodies[ind].joints[jt].orientationY; + var z = frames[i].bodies[ind].joints[jt].orientationZ; + var w = frames[i].bodies[ind].joints[jt].orientationW; + + var T_x = 180 + 180 / 3.1416 * Math.atan2(2 * y * z + 2 * x * w, 1 - 2 * x * x - 2 * y * y); // leaning forward/backward + var T_y = 180 / 3.1416 * Math.asin(2 * y * w - 2 * x * z); // turning + var T_z = 180 + 180 / 3.1416 * Math.atan2(2 * x * y + 2 * z * w, 1 - 2 * y * y - 2 * z * z); // leaning left + while (T_x > 90) { + T_x += -180; + } + if (T_y > 180) { + T_y += -180; + } + if (T_z > 180) { + T_z += -180; + } + if (datatype == 3) { + rawdata.push(T_x); + } + if (datatype == 4) { + rawdata.push(T_y); + } + if (datatype == 5) { + rawdata.push(T_z); + } + + } + } + return rawdata; } -function getSpeed(bufferTrial,id,jt){ - var frames = bufferTrial[id]; - var ind = frames.bodyIndex, time = frames.durationsecs; - var accumDist = 0, speed = 0; - for(var i=0; i0){ - for(var i=0 ; imaxlength) {maxlength = rawdata.length} - } - } - - if (exInd.length>0){ - for(var i=0 ; imaxlength) {maxlength = rawdata.length} - } - } - var numMarks = 10; - var pads = Math.ceil(maxlength/numMarks); - - for (var i = 0; i 0) { + for (var i = 0; i < gtInd.length; i++) { + curveData.labels.push("Referece_" + gtInd[i].toString()); + var id = gtArray[gtInd[i]]; + var rawdata = getRaw(bufferTrial, id, jt, datatype); + curveData.dataset.push(rawdata); + if (rawdata.length > maxlength) { + maxlength = rawdata.length + } + } + } + + if (exInd.length > 0) { + for (var i = 0; i < exInd.length; i++) { + curveData.labels.push("Exercise_" + exInd[i].toString()); + var id = exArray[exInd[i]]; + var rawdata = getRaw(bufferTrial, id, jt, datatype); + curveData.dataset.push(rawdata); + if (rawdata.length > maxlength) { + maxlength = rawdata.length + } + } + } + var numMarks = 10; + var pads = Math.ceil(maxlength / numMarks); + + for (var i = 0; i < numMarks; i++) { + var timeMark = (i * pads * 0.008342).toFixed(2); + curveData.xticks.push(timeMark.toString()); + for (var j = 1; j < pads; j++) { + curveData.xticks.push(""); + } + } + // "xticks": ["1", "", "", "", "", "6", "", "", "", "", "11"], + + return curveData; } -function barAnalyze(bufferTrial, gtArray, exArray){ - var barData = {}; - barData.leftHandSpeed = []; - barData.leftHandHeightChange = []; - barData.rightHandSpeed = []; - barData.rightHandHeightChange = []; - barData.pelvicSpeed = []; - barData.pelvicHeightChange = []; - barData.SpineMidOrientation = []; - barData.SpineBaseMovement = []; - var threshold = { - Speed: 0.005, - HeightChange: 0.005, - Orientation: 1, - }; - var speed_jt6_gt = 0, height_jt6_gt = 0,speed_jt10_gt = 0, height_jt10_gt = 0, - speed_jt0_gt = 0, height_jt0_gt = 0; - var lean_jt1_gt = 0, ampZ_jt0_gt=0; - for (var i in gtArray){ - speed_jt6_gt = getSpeed(bufferTrial,gtArray[i],6); - height_jt6_gt += getAmplitudeY(bufferTrial,gtArray[i],6); - speed_jt10_gt += getSpeed(bufferTrial,gtArray[i],10); - height_jt10_gt += getAmplitudeY(bufferTrial,gtArray[i],10); - speed_jt0_gt += getSpeed(bufferTrial,gtArray[i],0); - height_jt0_gt += getAmplitudeY(bufferTrial,gtArray[i],0); - lean_jt1_gt += getOrientation(bufferTrial,gtArray[i],1); - ampZ_jt0_gt += getAmplitudeZ(bufferTrial,gtArray[i],0); - } - speed_jt6_gt/=gtArray.length; - height_jt6_gt/=gtArray.length; - speed_jt10_gt/=gtArray.length; - height_jt10_gt/=gtArray.length; - speed_jt0_gt/=gtArray.length; - height_jt0_gt/=gtArray.length; - lean_jt1_gt /=gtArray.length; - ampZ_jt0_gt/=gtArray.length; - - for (var i in exArray){ - var speed_jt6_ex = getSpeed(bufferTrial,exArray[i],6); - if (Math.abs(speed_jt6_gt) < threshold.Speed) { barData.leftHandSpeed.push(0);} - else { barData.leftHandSpeed.push( (speed_jt6_ex-speed_jt6_gt)/speed_jt6_gt*100);} - - var height_jt6_ex = getAmplitudeY(bufferTrial,exArray[i],6); - if (Math.abs(height_jt6_gt) < threshold.HeightChange) {barData.leftHandHeightChange.push(0);} - else {barData.leftHandHeightChange.push( (height_jt6_ex-height_jt6_gt)/height_jt6_gt*100 );} - - var speed_jt10_ex = getSpeed(bufferTrial,exArray[i],10); - if (Math.abs(speed_jt10_gt) < threshold.Speed) { barData.rightHandSpeed.push(0);} - else {barData.rightHandSpeed.push( (speed_jt10_ex-speed_jt10_gt)/speed_jt10_gt*100);} - - var height_jt10_ex = getAmplitudeY(bufferTrial,exArray[i],10); - if (Math.abs(height_jt10_gt) < threshold.HeightChange) {barData.rightHandHeightChange.push(0);} - else {barData.rightHandHeightChange.push( (height_jt10_ex-height_jt10_gt)/height_jt10_gt*100 );} - - var speed_jt0_ex = getSpeed(bufferTrial,exArray[i],0); - if (Math.abs(speed_jt0_gt) < threshold.Speed) { barData.pelvicSpeed.push(0);} - else {barData.pelvicSpeed.push( (speed_jt0_ex-speed_jt0_gt)/speed_jt0_gt*100);} - - var height_jt0_ex = getAmplitudeY(bufferTrial,exArray[i],0); - if (Math.abs(height_jt0_gt) < threshold.HeightChange) {barData.pelvicHeightChange.push(0);} - else {barData.pelvicHeightChange.push( (height_jt0_ex-height_jt0_gt)/height_jt0_gt*100 );} - - var lean_jt1_ex = getOrientation(bufferTrial,exArray[i],1); - if (Math.abs(lean_jt1_ex) < threshold.HeightChange) {barData.SpineMidOrientation.push(0);} - else {barData.SpineMidOrientation.push( (lean_jt1_ex-lean_jt1_gt)/lean_jt1_gt*100 );} - - var ampZ_jt0_ex = getAmplitudeZ(bufferTrial,exArray[i],0); - if (Math.abs(ampZ_jt0_gt) < threshold.HeightChange) {barData.SpineBaseMovement.push(0);} - else {barData.SpineBaseMovement.push( (ampZ_jt0_ex-ampZ_jt0_gt)/ampZ_jt0_gt*100 );} - - } - return barData; -} -/* Reference -Look-up for joint selection -Kinect2.JointType = { - spineBase : 0, - spineMid : 1, - neck : 2, - head : 3, - shoulderLeft : 4, - elbowLeft : 5, - wristLeft : 6, - handLeft : 7, - shoulderRight : 8, - elbowRight : 9, - wristRight : 10, - handRight : 11, - hipLeft : 12, - kneeLeft : 13, - ankleLeft : 14, - footLeft : 15, - hipRight : 16, - kneeRight : 17, - ankleRight : 18, - footRight : 19, - spineShoulder : 20, - handTipLeft : 21, - thumbLeft : 22, - handTipRight : 23, - thumbRight : 24 -}; - - -Structure of bodyFrame -bodyFrame = { -bodies:[ -0: - bodyIndex: 0 - joints: Array(25) [{depthX, depthY... orientationZ},{depthX, depthY... orientationZ} ] - leftHandState: 0 - rightHandState: 0 - tracked: true - trackingID: "7200399405055" -1: - bodyIndex: 1 - tracked: false -... -] -floorClipPlane: { - w: - x: - y: - z: +function barAnalyze(bufferTrial, gtArray, exArray) { + var barData = {}; + barData.leftHandSpeed = []; + barData.leftHandHeightChange = []; + barData.rightHandSpeed = []; + barData.rightHandHeightChange = []; + barData.pelvicSpeed = []; + barData.pelvicHeightChange = []; + barData.SpineMidOrientation = []; + barData.SpineBaseMovement = []; + var threshold = { + Speed: 0.005, + HeightChange: 0.005, + Orientation: 1, + }; + var speed_jt6_gt = 0, height_jt6_gt = 0, speed_jt10_gt = 0, height_jt10_gt = 0, + speed_jt0_gt = 0, height_jt0_gt = 0; + var lean_jt1_gt = 0, ampZ_jt0_gt = 0; + for (var i in gtArray) { + speed_jt6_gt = getSpeed(bufferTrial, gtArray[i], 6); + height_jt6_gt += getAmplitudeY(bufferTrial, gtArray[i], 6); + speed_jt10_gt += getSpeed(bufferTrial, gtArray[i], 10); + height_jt10_gt += getAmplitudeY(bufferTrial, gtArray[i], 10); + speed_jt0_gt += getSpeed(bufferTrial, gtArray[i], 0); + height_jt0_gt += getAmplitudeY(bufferTrial, gtArray[i], 0); + lean_jt1_gt += getOrientation(bufferTrial, gtArray[i], 1); + ampZ_jt0_gt += getAmplitudeZ(bufferTrial, gtArray[i], 0); + } + speed_jt6_gt /= gtArray.length; + height_jt6_gt /= gtArray.length; + speed_jt10_gt /= gtArray.length; + height_jt10_gt /= gtArray.length; + speed_jt0_gt /= gtArray.length; + height_jt0_gt /= gtArray.length; + lean_jt1_gt /= gtArray.length; + ampZ_jt0_gt /= gtArray.length; + + for (var i in exArray) { + var speed_jt6_ex = getSpeed(bufferTrial, exArray[i], 6); + if (Math.abs(speed_jt6_gt) < threshold.Speed) { + barData.leftHandSpeed.push(0); + } + else { + barData.leftHandSpeed.push((speed_jt6_ex - speed_jt6_gt) / speed_jt6_gt * 100); + } + + var height_jt6_ex = getAmplitudeY(bufferTrial, exArray[i], 6); + if (Math.abs(height_jt6_gt) < threshold.HeightChange) { + barData.leftHandHeightChange.push(0); + } + else { + barData.leftHandHeightChange.push((height_jt6_ex - height_jt6_gt) / height_jt6_gt * 100); + } + + var speed_jt10_ex = getSpeed(bufferTrial, exArray[i], 10); + if (Math.abs(speed_jt10_gt) < threshold.Speed) { + barData.rightHandSpeed.push(0); + } + else { + barData.rightHandSpeed.push((speed_jt10_ex - speed_jt10_gt) / speed_jt10_gt * 100); + } + + var height_jt10_ex = getAmplitudeY(bufferTrial, exArray[i], 10); + if (Math.abs(height_jt10_gt) < threshold.HeightChange) { + barData.rightHandHeightChange.push(0); + } + else { + barData.rightHandHeightChange.push((height_jt10_ex - height_jt10_gt) / height_jt10_gt * 100); + } + + var speed_jt0_ex = getSpeed(bufferTrial, exArray[i], 0); + if (Math.abs(speed_jt0_gt) < threshold.Speed) { + barData.pelvicSpeed.push(0); + } + else { + barData.pelvicSpeed.push((speed_jt0_ex - speed_jt0_gt) / speed_jt0_gt * 100); + } + + var height_jt0_ex = getAmplitudeY(bufferTrial, exArray[i], 0); + if (Math.abs(height_jt0_gt) < threshold.HeightChange) { + barData.pelvicHeightChange.push(0); + } + else { + barData.pelvicHeightChange.push((height_jt0_ex - height_jt0_gt) / height_jt0_gt * 100); + } + + var lean_jt1_ex = getOrientation(bufferTrial, exArray[i], 1); + if (Math.abs(lean_jt1_ex) < threshold.HeightChange) { + barData.SpineMidOrientation.push(0); + } + else { + barData.SpineMidOrientation.push((lean_jt1_ex - lean_jt1_gt) / lean_jt1_gt * 100); + } + + var ampZ_jt0_ex = getAmplitudeZ(bufferTrial, exArray[i], 0); + if (Math.abs(ampZ_jt0_gt) < threshold.HeightChange) { + barData.SpineBaseMovement.push(0); + } + else { + barData.SpineBaseMovement.push((ampZ_jt0_ex - ampZ_jt0_gt) / ampZ_jt0_gt * 100); + } + + } + return barData; } -*/ +} \ No newline at end of file diff --git a/exercise/public/index.html b/exercise/public/index.html index 6afeb5b..609416a 100755 --- a/exercise/public/index.html +++ b/exercise/public/index.html @@ -1,50 +1,72 @@ - - - ExerciseCheck Client - - ExerciseCheck Client - - - - - - - - - + + + ExerciseCheck Client + + ExerciseCheck Client + + + + + + + + + + + - -

ExerciseCheck

-
-
-
- -
- - - - - -
-
-
- - - -
-
- -
- - - - - + +

ExerciseCheck

+
+
+
+ +
+ + + +
0
+ + +
+
+
+ + + + + + +
+
+ +
+ +
+ + + + + - + \ No newline at end of file diff --git a/exercise/public/js/README.md b/exercise/public/js/README.md new file mode 100644 index 0000000..024e666 --- /dev/null +++ b/exercise/public/js/README.md @@ -0,0 +1,29 @@ +## Joint indices per Kinect2 API: + +Joint Type | Index +------------ | ------------- +spineBase | 0 +spineMid | 1 +neck | 2 +head | 3 +shoulderLeft | 4 +elbowLeft | 5 +wristLeft | 6 +handLeft | 7 +shoulderRight | 8 +elbowRight | 9 +wristRight | 10 +handRight | 11 +hipLeft | 12 + kneeLeft | 13 + ankleLeft | 14 + footLeft | 15 + hipRight | 16 + kneeRight | 17 + ankleRight | 18 + footRight | 19 + spineShoulder | 20 + handTipLeft | 21 + thumbLeft | 22 + handTipRight | 23 + thumbRight | 24 \ No newline at end of file diff --git a/exercise/public/js/kinect.js b/exercise/public/js/kinect.js index 09548ba..0691bef 100755 --- a/exercise/public/js/kinect.js +++ b/exercise/public/js/kinect.js @@ -2,166 +2,276 @@ var socket = io.connect('/'); var clientActive = false; $(document).ready(function () { - var canvasSKLT = document.getElementById('bodyCanvas'); - var ctx1 = canvasSKLT.getContext('2d'); + var canvasSKLT = document.getElementById('bodyCanvas'); + var ctx1 = canvasSKLT.getContext('2d'); - document.getElementById("display").style.display = 'none'; - var colors = ['#ff0000', '#00ff00', '#0000ff', '#ffff00', '#00ffff', '#ff00ff']; + document.getElementById("display").style.display = 'none'; - // Globals: - var radius=9; - var width = canvasSKLT.width; - var height = canvasSKLT.height; + // Globals: + var radius = 9; + var width = canvasSKLT.width; + var height = canvasSKLT.height; + var IntervalID; + var threshold_flag; + var reps = 0; + var neck_y; + var neck_x; - var IntervalID; + // Use bodyFrame from server to update the canvas 1 on client + socket.on('init', function (bodyFrame, systemState) { + clientActive = true; + updateCanvas(bodyFrame, -1); + document.getElementById("command").value = 'Start'; + document.getElementById("command").style.backgroundColor = ''; + document.getElementById("display").style.display = 'none'; + }); - // Signals + socket.on('rec', function (bodyFrame, systemState, tracingID) { + clientActive = true; - // Use bodyFrame from server to update the canvas 1 on client - socket.on('init', function(bodyFrame,systemState){ - clientActive = true; - liveupdateCanvas1(bodyFrame,-1); - document.getElementById("command").value= 'Start'; - document.getElementById("command").style.backgroundColor = ''; - document.getElementById("display").style.display = 'none'; - }); + // Initializing flag for counting repetitions - socket.on('rec', function(bodyFrame,systemState, tracingID){ - clientActive = true; - liveupdateCanvas1(bodyFrame,tracingID); - document.getElementById("command").value = 'Stop'; - document.getElementById("command").style.backgroundColor = 'red'; - document.getElementById("display").style.display = 'none'; - }); - - socket.on('disp', function(bufferBodyFrames,systemState, tracingID, activityLabeled){ - clientActive = true; // unlock the button - IntervalID = animateCanvas1(bufferBodyFrames,tracingID); - document.getElementById("command").value = 'Live'; - document.getElementById("command").style.backgroundColor = ''; - if (!activityLabeled) - document.getElementById("display").style.display = 'block'; - }); - - socket.on('live', function(bodyFrame,systemState, tracingID){ - clientActive = true; - clearInterval(IntervalID); - liveupdateCanvas1(bodyFrame,-1); - - document.getElementById("command").value= 'Start'; - document.getElementById("command").style.backgroundColor = ''; - document.getElementById("display").style.display = 'none'; - }); + if (!threshold_flag){ - socket.on('serverDataLabeled',function(){ // hid the buttons "Reference","Exercise","Discard" - document.getElementById("display").style.display = 'none'; - }); - - // Functions - - function drawCircle(x, y, r, color){ // Not used in current code - ctx1.beginPath(); - ctx1.strokeStyle=color; - ctx1.arc(x, y,r,0,Math.PI*2); - ctx1.stroke(); - } - - function drawBody(body){ - //drawCircle(50, 50, 10, "green"); - jointType = [7,6,5,4,2,8,9,10,11,10,9,8,2,3,2,1,0,12,13,14,15,14,13,12,0,16,17,18,19] //re visit and draw in a line - jointType.forEach(function(jointType){ - drawJoints(body.joints[jointType].depthX * width, body.joints[jointType].depthY * height); + // Based on exercise (eg squat), know which direction we start counting reps in + threshold_flag = getExerciseInfo(document.getElementById("exercise").value)['direction']; + } + temp_reps = updateCanvas(bodyFrame, tracingID, 'rec', threshold_flag); + if (temp_reps){ + threshold_flag = temp_reps[1]; + document.getElementById("reps").innerHTML = parseInt(document.getElementById("reps").innerHTML) + temp_reps[0]; + } + document.getElementById("command").value = 'Stop'; + document.getElementById("command").style.backgroundColor = 'red'; + document.getElementById("display").style.display = 'none'; + }); + + socket.on('disp', function (bufferBodyFrames, systemState, tracingID, activityLabeled) { + clientActive = true; // unlock the button + reps = 0; + IntervalID = animateCanvas1(bufferBodyFrames, tracingID, 'disp'); + document.getElementById("command").value = 'Live'; + document.getElementById("command").style.backgroundColor = ''; + if (!activityLabeled) + document.getElementById("display").style.display = 'block'; + }); + + socket.on('live', function (bodyFrame) { + clientActive = true; + clearInterval(IntervalID); + updateCanvas(bodyFrame, -1); + + document.getElementById("command").value = 'Start'; + document.getElementById("command").style.backgroundColor = ''; + document.getElementById("display").style.display = 'none'; }); - drawCenterCircle(width/2, height/5, 50, body.joints[2].depthX * width, body.joints[2].depthY * height); - ctx1.beginPath(); - ctx1.moveTo(body.joints[7].depthX * width, body.joints[7].depthY * height); - jointType.forEach(function(jointType){ - ctx1.lineTo(body.joints[jointType].depthX * width, body.joints[jointType].depthY * height); - ctx1.moveTo(body.joints[jointType].depthX * width, body.joints[jointType].depthY * height); + socket.on('serverDataLabeled', function () { // hide the buttons "Reference","Exercise","Discard" + document.getElementById("display").style.display = 'none'; }); - ctx1.lineWidth=10; - ctx1.strokeStyle='blue'; - ctx1.stroke(); - ctx1.closePath(); - } - - function drawJoints(cx,cy){ - ctx1.beginPath(); - ctx1.arc(cx,cy,radius,0,Math.PI*2); //radius is a global variable defined at the beginning - ctx1.closePath(); - ctx1.fillStyle = "yellow"; - ctx1.fill(); - } - // Draw Center Circle in ctx1 (canvasSKLT) - function drawCenterCircle(x, y, r, nx, ny){ - ctx1.beginPath(); - if(nx > x-r && nx < x+r && ny > y-r && ny < y+r) - ctx1.strokeStyle="green"; - else - ctx1.strokeStyle="red"; - - ctx1.arc(x, y,r,0,Math.PI*2); - ctx1.stroke(); - ctx1.closePath(); - ctx1.strokeStyle="black"; - } - - - function liveupdateCanvas1(bodyFrame, tracingID){ - ctx1.clearRect(0, 0, width, height); - //drawCircle(ctx1, 50, 50, 10, "red"); removed if following code line works - //drawCenterCircle(width/2, height/5, 50, body.joints[2].depthX * width, body.joints[2].depthY * height); - if (tracingID == -1){ - bodyFrame.bodies.some(function(body){ - if(body.tracked){ drawBody(body); return(body.tracked);} - }); + + socket.on('showing-speed', function (speed) { + var gt_speed = 0.3; // temp for testing + var ex_speed = Math.round(speed * 100) / 100; + var msg = 'The reference speed is ' + gt_speed.toString() + ' m/s. Your speed is ' + ex_speed.toString() + ' m/s.'; + + if ((gt_speed + 0.1) < ex_speed){ + msg = 'Try slowing down. ' + msg; + } + else if ((gt_speed - 0.1) > ex_speed){ + msg = 'Try increasing your speed. ' + msg; + } + document.getElementById("speed").innerHTML = msg; + }); + + + function drawBody(body) { + jointType = [7, 6, 5, 4, 2, 8, 9, 10, 11, 10, 9, 8, 2, 3, 2, 1, 0, 12, 13, 14, 15, 14, 13, 12, 0, 16, 17, 18, 19] //re visit and draw in a line + jointType.forEach(function (jointType) { + drawJoints(body.joints[jointType].depthX * width, body.joints[jointType].depthY * height); + }); + + + drawCenterCircle(width / 2, height / 5, 50, body.joints[2].depthX * width, body.joints[2].depthY * height, body.joints[2].cameraZ, body.joints[19].depthY, body.joints[0].depthY); + + ctx1.beginPath(); + ctx1.moveTo(body.joints[7].depthX * width, body.joints[7].depthY * height); + jointType.forEach(function (jointType) { + ctx1.lineTo(body.joints[jointType].depthX * width, body.joints[jointType].depthY * height); + ctx1.moveTo(body.joints[jointType].depthX * width, body.joints[jointType].depthY * height); + }); + ctx1.lineWidth = 10; + ctx1.strokeStyle = 'blue'; + ctx1.stroke(); + ctx1.closePath(); + } + + function drawJoints(cx, cy) { + ctx1.beginPath(); + ctx1.arc(cx, cy, radius, 0, Math.PI * 2); //radius is a global variable defined at the beginning + ctx1.closePath(); + ctx1.fillStyle = "yellow"; + ctx1.fill(); } - else { - drawBody(bodyFrame.bodies[tracingID]); + + // Draw center circle - to help user position themselves + function drawCenterCircle(x, y, r, nx, ny,nz, footright, spinebase) { + ctx1.beginPath(); + if (nx > x - r && nx < x + r && ny > y - r && ny < y + r) { + ctx1.strokeStyle = "green"; + if (!neck_y || !neck_x){ + neck_x = nx/width; + neck_y = ny/height; + console.log('neck_y', neck_y) + console.log('footright', footright) + console.log('spinebase', spinebase) + } + } else { + ctx1.strokeStyle = "red"; + } + + ctx1.arc(x, y, r, 0, Math.PI * 2); + ctx1.stroke(); + ctx1.closePath(); + ctx1.strokeStyle = "black"; + } + + // Function parameters: range_scale is how much to scale the distance between spine and foot, + // top_thresh and bottom_thresh are the thresholds the user should reach when the signal + // has been scaled to a range between 0 and 1, for a repetition to count + + function countReps(body, threshold_flag, range_scale=0.7, top_thresh=0.25, bottom_thresh=0.75){ + var reps = 0; + var norm; + + // Get info based on exercise (default is squat) + // This should eventually be a db call + var exInfo = getExerciseInfo(document.getElementById("exercise").value); + var joint = exInfo['joint']; + var coordinate = exInfo['axis']; + + // This is set when user is correctly positioned in circle + if (coordinate == 'depthY') { + norm = neck_y; + } else if (coordinate == 'depthX') { + norm = neck_x; + } + + // Normalize reference points to neck + var ref_norm = exInfo['ref_neck']; + var ref_max = exInfo['ref_max'] - ref_norm; + var ref_min = exInfo['ref_min'] - ref_norm; + + // Range is based on distance between foot and spine + var ref_foot = exInfo['ref_foot']; + var ref_spine = exInfo['ref_spine']; + var range = (ref_foot - ref_norm - ref_spine)*range_scale; + + // Normalize current point by range and current neck value + var current_pt = (body.joints[joint][coordinate] - norm - ref_max)/range; + + if ((threshold_flag == 'down') && (current_pt < top_thresh)){ + reps++; + return [reps,'up']; + } else if ((threshold_flag == 'up') && (current_pt > bottom_thresh)){ + return [reps,'down']; + } else { + return [reps, threshold_flag] + } + } + + // Line to indicate user reaching some standard or requirement + // Input is start of line, ground truth height, exercise height + // Line is yellow if requirement unfulfilled, green if fulfilled + function draw_height_line(starting_x, gt_y, ex_y, comparison='less', direction='right') { + ctx1.beginPath(); + if (comparison == 'less') { + if (ex_y < gt_y) { + ctx1.strokeStyle = "green"; + } else { + ctx1.strokeStyle = "yellow"; + } + } + else { + if (ex_y > gt_y) { + ctx1.strokeStyle = "green"; + } else { + ctx1.strokeStyle = "yellow"; + } + } + + ctx1.moveTo(starting_x, gt_y); + if (direction == 'both') { + ctx1.lineTo(width, gt_y); + ctx1.lineTo(0, gt_y); + } else if (direction == 'left') { + ctx1.lineTo(0, gt_y); + } else { + ctx1.lineTo(width, gt_y); + } + ctx1.stroke(); + ctx1.closePath(); + ctx1.strokeStyle = "black"; + } + + + // Draw every new bodyframe on the canvas + function updateCanvas(bodyFrame, tracingID, mode='', threshold_flag='') { + ctx1.clearRect(0, 0, width, height); + var reps = 0; + if (tracingID == -1) { + bodyFrame.bodies.some(function (body) { + if (body.tracked) { + drawBody(body); + if (mode=='rec'){ + reps = countReps(body, threshold_flag); + } + } + }); + } else { + var body = bodyFrame.bodies[tracingID]; + drawBody(body); + reps = countReps(body, threshold_flag); + } + return reps; + } + + function animateCanvas1(bufferBodyFrames, tracingID){ + var i = 0; + var TimerID = setInterval(function(){ + updateCanvas(bufferBodyFrames[i], tracingID); + i++; + if (i>=bufferBodyFrames.length){i=0;} + },20); + return TimerID; + } + + + // Get info from JSON about a given exercise + function getExerciseInfo(exercise){ + // This info should be patient-specific and eventually stored in the db + exercise_info = { + 'hand_raise': + {'joint': 11, + 'direction': 'down', + 'axis': 'depthY', + 'ref_max': 0.73, + 'ref_min': 1.19, + 'ref_mean': 0.76, + 'ref_neck': 0.27, + }, + 'squat': + {'joint': 0, + 'direction': 'up', + 'axis': 'depthY', + 'ref_max': 0.68, + 'ref_min': 1.13, + 'ref_mean': 0.76, + 'ref_neck': 0.27, + 'ref_foot': 1.43, + 'ref_spine': 0.70 + }, + } + return exercise_info[exercise]; } - } - - function animateCanvas1(bufferBodyFrames,tracingID){ - var i = 0; - var TimerID = setInterval(function(){ - liveupdateCanvas1(bufferBodyFrames[i], tracingID); - i++; - if (i>=bufferBodyFrames.length){i=0;} - },20); - return TimerID; - } -}); - - - -/* Reference -Look-up for joint selection -Kinect2.JointType = { - spineBase : 0, - spineMid : 1, - neck : 2, - head : 3, - shoulderLeft : 4, - elbowLeft : 5, - wristLeft : 6, - handLeft : 7, - shoulderRight : 8, - elbowRight : 9, - wristRight : 10, - handRight : 11, - hipLeft : 12, - kneeLeft : 13, - ankleLeft : 14, - footLeft : 15, - hipRight : 16, - kneeRight : 17, - ankleRight : 18, - footRight : 19, - spineShoulder : 20, - handTipLeft : 21, - thumbLeft : 22, - handTipRight : 23, - thumbRight : 24 -}; -*/ +}); \ No newline at end of file diff --git a/exercise/public/js/navigation.js b/exercise/public/js/navigation.js index cbf8d90..2b9e4f4 100755 --- a/exercise/public/js/navigation.js +++ b/exercise/public/js/navigation.js @@ -1,29 +1,35 @@ $(document).ready(function () { - //state button - $("#command").click(function () { - if (clientActive){ - socket.emit('command'); - clientActive = false; // lock the client until server responds - } - }); + //state button + $("#command").click(function () { + if (clientActive) { + socket.emit('command'); + clientActive = false; // lock the client until server responds + } + }); - $("#gt").click(function () { - socket.emit('dataLabelFromClient', 1); - }); + $("#gt").click(function () { + socket.emit('dataLabelFromClient', 1); + }); - $("#ex").click(function () { - socket.emit('dataLabelFromClient', 2); - }); + $("#ex").click(function () { + socket.emit('dataLabelFromClient', 2); + }); - $("#re").click(function () { - socket.emit('dataLabelFromClient', 3); - }); + $("#re").click(function () { + socket.emit('dataLabelFromClient', 3); + }); - $("#report").click(function () { - console.log('report button pressed!'); - location.href = "report.html"; - location.target = "_blank"; - }); + $("#report").click(function () { + console.log('report button pressed!'); + location.href = "report.html"; + location.target = "_blank"; + }); + + $("#show-speed").click(function () { + console.log('show speed button clicked'); + var e = document.getElementById("joint-speed"); + var joint = e.options[e.selectedIndex].value; + socket.emit('joint-speed', joint); + }); }); -//hide button