Interacting with the Browser Using Gestures
This chapter aims to take everything you’ve learned so far and put it all together to make an ambitious music creation web app that allows multiple people to play instruments, using computer-generated audio, at the same time using just JavaScript, Node.js, and some HTML5 APIs. Oh, and it also needs to use the webcam to allow input using your fingers in the air.
As I’m sure you can imagine this is a massive task. This chapter gets you to a basic version of it and allows you to extend the task however you please. For example, instead of tracking all fingers so that you can play the keyboard properly, it will instead have a cursor that you can move left or right by pointing in that direction, or pointing in the middle to simulate a key press.
Taking the Keyboard Server-Side
We base this project on the keyboard from Chapter 5. Currently, the keyboard is set up so that you can play multiple notes at once (which is a good start) but we need to modify it so that multiple people can play multiple notes. Of course, even if two people are playing the same note, it should only be played once. This is ideal as it means we only need one set of oscillators per person, so when person B starts playing a note it will play using the set of oscillators that person A is using.
Previously we turned off the oscillator within stopSound but this can cause issues with race conditions (that is, things done always happen in the order you expect) and since you cannot play a stopped oscillator, it is better to delete it (in this case, we will set it to null) and create a new one (which we were doing anyway). To make it easier to check existence of an oscillator, I’ve changed the array initialization loops to set the indexes to null instead of creating new oscillators to begin with.
// White Keys
var o = new Array(21);
for (var i = 0; i < 21; i++) {
o[i] = null;
}
// Black Keys
var ob = new Array(15);
for (var i = 0; i < 15; i++) {
ob[i] = null;
}
function stopSound(i, black) {
var osc;
if (black) {
osc = ob[i];
} else {
osc = o[i];
}
if ((black && ob[i] !== null) || (!black && o[i] !== null)) {
osc.noteOff(0);
osc.disconnect();
if (black) {
ob[i] = null;
} else {
o[i] = null;
}
}
}
The playSound function now needs to only play a new sound if there is no oscillator already on the key; otherwise it will cause problems such as not being able to stop it playing or it will only play for a short amount of time. I’ve also cleaned it up a little so that the same conditions are not checked twice without needing to be. Note that I’ve changed ctx to actx, this is so that it is obviously an Audio Context and to avoid confusion with the Canvas Context used for the computer vision.
var gainNode = actx.createGainNode();
gainNode.connect(actx.destination);
function playSound(i, black) {
if ((black && ob[i] === null) || (!black && o[i] === null)) {
var osc = actx.createOscillator();
var freq;
if (black) {
freq = blackNotes[i];
}
else {
freq = whiteNotes[i];
}
osc.type = 3;
osc.frequency.value = freq;
osc.connect(gainNode);
osc.noteOn(0);
if (black) {
ob[i] = osc;
} else {
o[i] = osc;
}
}
}
So now the keyboard is set up a little better and we can write the server code. Socket.IO allows you to define the structure of all communication between the client and server in a very nice way; each message is a task that is usually passed on to the other clients. The keyboard has two tasks: play sounds and stop playing sounds. All we need is a couple of listeners for messages that play or stop sounds.
socket.on('play_sound', function (data) {
data.color = socket.color;
console.log("play sound %j", data);
socket.broadcast.to(room).emit('play_sound', data);
});
socket.on('stop_sound', function (data) {
console.log("stop sound %j", data);
socket.broadcast.to(room).emit('stop_sound', data);
});
You may have noticed that we send a color to everyone in the room; this is so that they can easily see keys that other people have pressed. I’ve not created them uniquely, using a database or anything, but when the socket originally connects it creates a random color. I’ve done this by creating a random number and multiplying it by 16777215, this is essentially the same as doing a bitwise shift on the hex color for white (#ffffff).
socket.color = '#'+ ('000000' + Math.floor(Math.random()*16777215).toString(16)).slice(-6);
And that’s it! The rest of the server-side code is the same as we had in Chapter 7.
server.js
var express = require('express'),
http = require('http'),
uuid = require ("node-uuid"),
app = express(),
server = http.createServer(app),
io = require('socket.io').listen(server);
io.set('log level', 1); // Removes debug logs
app.use(express.static('public'));
server.listen(8080);
app.get('/:id', function (req, res) {
res.sendfile(__dirname + '/index.html'),
});
io.sockets.on('connection', function (socket) {
var room = socket.handshake.headers.referer;
console.log('JOINED', room);
socket.join(room);
socket.id = uuid.v1();
socket.color = '#'+ ('000000' + Math.floor(Math.random()*16777215).toString(16)).slice(-6);
io.sockets.in(room).emit('room_count', io.sockets.manager.rooms['/' + room].length);
// WebRTC signalling
socket.on('received_offer', function(data) {
console.log("received_offer %j", data);
io.sockets.in(room).emit('received_offer', data);
});
socket.on('received_candidate', function(data) {
console.log("received_candidate %j", data);
io.sockets.in(room).emit('received_candidate', data);
});
socket.on('received_answer', function(data) {
console.log("received_answer %j", data);
io.sockets.in(room).emit('received_answer', data);
});
// Chatroom messages
socket.on('message', function (data) {
io.sockets.in(room).emit('message', data);
});
// Music
socket.on('play_sound', function (data) {
data.color = socket.color;
console.log("play sound %j", data);
socket.broadcast.to(room).emit('play_sound', data);
});
socket.on('stop_sound', function (data) {
console.log("stop sound %j", data);
socket.broadcast.to(room).emit('stop_sound', data);
});
// Close socket and let others know
socket.on('close', function () {
console.log("closed %j", room);
io.sockets.in(room).emit('closed', room);
socket.leave(room);
});
});
On the client-side we need to both send the messages, as well as handle received messages. To begin with, let’s set the color. We set a variable called color when the ‘color’ message is received (when you break it down bit by bit, this really is simple!).
socket.on('color', function(data) {
color = data;
})
When the play or stop messages are received, we need to call the corresponding function; on play we also need to set the color of the particular key to be the color that relates to the person that sent it. On stop, the colors should go back to being black or white.
socket.on('play_sound', function(data) {
if (data.black) {
bkeys[data.i].fill({ color: data.color });
} else {
keys[data.i].fill({ color: data.color });
}
playSound(data.i, data.black);
});
socket.on('stop_sound', function(data) {
if (data.black) {
bkeys[data.i].fill({ color: '#000' });
} else {
keys[data.i].fill({ color: '#fff' });
}
stopSound(data.i, data.black);
});
To send the messages, we just emit the i and black variables, as well as the color. These need to be emitted every time a key is pressed—including through mouse, keyboard, and webcam inputs. Note that we don’t just emit on playSound or stopSound because they are called when a message is received and this causes a circular loop. With the emmited messages added, the code for all inputs looks like the following:
var keys = [];
var cursor = [];
for (var i = 0; i < 21; i++) {
keys[i] = keyboard.rect(width/21, height);
keys[i].move(width/21 * i, 0);
keys[i].attr({ fill: '#fff', stroke: '#000', id: "key"+i });
keys[i].mousedown ((function(n) {
return function() {
var key = SVG.get("key"+n);
key.fill({ color: '#f06' });
socket.emit('play_sound', { "i":n,"black":false});
playSound(n, false);
}
})(i));
keys[i].mouseup((function(n) {
return function() {
keys[n].fill({ color: '#fff' });
socket.emit('stop_sound', { "i":n,"black":false});
stopSound(n, false);
}
})(i));
cursor[i] = keyboard.circle(width/21);
cursor[i].attr({ fill: '#ff0000' });
cursor[i].move(width/21 * i, height / 1.5);
cursor[i].hide();
}
var bkeys = [];
var prev = 0;
for (var i = 0; i < 15; i++) {
bkeys[i] = keyboard.rect(width/42, height / 1.7);
bkeys[i].attr({ fill: '#000', stroke: '#000', id: "bkey"+i });
bkeys[i].move(prev + (width/(21*1.3)), 0);
prev = prev + width/21;
if (i == 1 || i == 4 || i == 6 || i == 9 || i == 11) {
prev += width/21;
}
bkeys[i].mousedown ((function(n) {
return function() {
var key = SVG.get("bkey"+n);
key.fill({ color: '#f06' });
socket.emit('play_sound', { "i":n,"black":true});
playSound(n, true);
}
})(i));
bkeys[i].mouseup((function(n) {
return function() {
bkeys[n].fill({ color: '#000' });
socket.emit('stop_sound', { "i":n,"black":true});
stopSound(n, true);
}
})(i));
}
window.addEventListener('keypress', function(e) {
for (var i = 0; i < keyboardPressKeys.length; i++) {
if (e.keyCode == keyboardPressKeys[i]) {
var n = i + octave * 7;
var key = SVG.get("key"+n);
key.fill({ color: '#f06' });
socket.emit('play_sound', { "i":n,"black":false, "color":color});
playSound(n, false);
}
}
for (var i = 0; i < blackKeyPress.length; i++) {
if (e.keyCode == blackKeyPress[i]) {
var n = i + (octave * 5);
var key = SVG.get("bkey"+n);
key.fill({ color: '#f06' });
socket.emit('play_sound', { "i":n,"black":true, "color":color});
playSound(n, true);
}
}
if (e.keyCode == 97 && octave > 0) {
--octave;
}
if (e.keyCode == 108 && octave < 2) {
++octave;
}
});
window.addEventListener('keyup', function(e) {
console.log(e.keyCode);
for (var i = 0; i < keyboardKeys.length; i++) {
if (e.keyCode == keyboardKeys[i]) {
var key = SVG.get("key"+(i+octave*7));
key.fill({ color: '#fff' });
socket.emit('stop_sound', { "i":i+octave*7,"black":false, "color":color });
stopSound(i+octave*7, false);
}
}
for (var i = 0; i < blackKeys.length; i++) {
if (e.keyCode == blackKeys[i]) {
var n = i + octave * 5;
var key = SVG.get("bkey"+n);
key.fill({ color: '#000' });
socket.emit('stop_sound', { "i":n,"black":true, "color":color });
stopSound(n, true);
}
}
});
So now, every time a key is pressed on the keyboard it plays to everybody in the room. This is a good example of how you can go from something simple such as a chatroom to something far more advanced with minimal actual differences to how it works. Figure 11-1 shows an example of two notes being played, with the keys filled with the colour that is unique to the user.
Figure 11-1. The keyboard playing with keys of the user’s own color
Controlling the Music with a Webcam
Adding motion control to the keyboard is just a matter of implementing the steps from the previous chapter. We start by capturing two frames of the webcam and analyzing it to find motion between the two frames. Remember, we are not using any artificial intelligence to do this so you may need to fiddle with the numbers to make it accurate for your lighting conditions and distance from the screen (as well as movement behind you).
To find the motion, we iterate through the pixels of the second frame comparing them to the previous frame. If they are different, then change the color to black otherwise make it white. For this example we only need motion on the X axis so, every time motion is detected, we add the percentage of the width that corresponds to the X position of the movement. For example, motionX += 100*(x/w);. Once everything has been checked, we can divide it by the count to find the average X position. Due to simple math this is not completely accurate, but it is enough to find an indication of motion. Depending on the orientation of the canvas, you may need to invert the numbers to mirror motion correctly: motionX = 100 - motionX;.
Now, we can check to see whether motionX is within the bounds of the particular motion we are looking for. In this we want to move a cursor left and right, pressing the key if there is motion in the middle third. So if motionX is less than 100/3, then move left. If it is between 100/3 and (100/3)*2, then press the key or if it is above (100/3)*2, then move right.
This example only works for the white keys, but you could quite easily extend it to include black keys by finding motion on the Y-axis as well.
function motionDetection(data, oldData) {
var motionX = 0;
var count = 0;
// Iterate through each pixel, changing to 255 if it has not changed
for( var y = 0 ; y < h; y++ ) {
for( var x = 0 ; x < w; x++ ) {
var indexOld = (y * w + x) * 4,
oldr = oldData[indexOld],
oldg = oldData[indexOld+1],
oldb = oldData[indexOld+2],
olda = oldData[indexOld+3];
var indexNew = (y * w + x) * 4,
r = data[indexNew],
g = data[indexNew+1],
b = data[indexNew+2],
a = data[indexNew+3];
if (oldr > r - 30 || oldg > g - 30 || oldb > b - 30)
{
data[indexNew] = 255;
data[indexNew+1] = 255;
data[indexNew+2] = 255;
data[indexNew+3] = 255;
}
else
{
data[indexNew] = 0;
data[indexNew+1] = 0;
data[indexNew+2] = 0;
data[indexNew+3] = 255;
motionX += 100*(x/w); // motionX = The percentage of W than X is at
count++;
}
}
}
motionX = motionX / count;
motionX = 100 - motionX;
if (motionX > 0) {
if (motionX > 100/3 && motionX < (100/3)*2) {
if (o[i] === null)
playSound(currentCursor, false);
console.log('2'),
}
if (motionX > (100/3)*2) {
if (currentCursor + 1 < keys.length) {
cursor[currentCursor].hide();
stopSound(currentCursor, false);
currentCursor++;
cursor[currentCursor].show();
}
console.log('3'),
}
if (motionX < 100/3) {
if (currentCursor - 1 >=0) {
cursor[currentCursor].hide();
stopSound(currentCursor, false);
currentCursor--;
cursor[currentCursor].show();
}
console.log('1'),
}
}
return data;
}
The cursor is a round red dot that is created at the same time as the keys; instead of moving a cursor along the keys it hides and only shows the correct one.
cursor[i] = keyboard.circle(width/21);
cursor[i].attr({ fill: '#ff0000' });
cursor[i].move(width/21 * i, height / 1.5);
cursor[i].hide();
Now, the project should all be working. You can find the complete code below in Listing 11-1 and in the download that is available on the Apress website at www.apress.com/9781430259442 or on my own website at www.shanehudson.net/javascript-creativity.
Listing 11-1. server.js
var express = require('express'),
http = require('http'),
uuid = require ("node-uuid"),
app = express(),
server = http.createServer(app),
io = require('socket.io').listen(server);
io.set('log level', 1); // Removes debug logs
app.use(express.static('public'));
server.listen(8080);
app.get('/:id', function (req, res) {
res.sendfile(__dirname + '/index.html'),
});
io.sockets.on('connection', function (socket) {
var room = socket.handshake.headers.referer;
console.log('JOINED', room);
socket.join(room);
socket.id = uuid.v1();
socket.color = '#'+ ('000000' + Math.floor(Math.random()*16777215).toString(16)).slice(-6);
io.sockets.in(room).emit('room_count', io.sockets.manager.rooms['/' + room].length);
// WebRTC signalling
socket.on('received_offer', function(data) {
console.log("received_offer %j", data);
io.sockets.in(room).emit('received_offer', data);
});
socket.on('received_candidate', function(data) {
console.log("received_candidate %j", data);
io.sockets.in(room).emit('received_candidate', data);
});
socket.on('received_answer', function(data) {
console.log("received_answer %j", data);
io.sockets.in(room).emit('received_answer', data);
});
// Chatroom messages
socket.on('message', function (data) {
io.sockets.in(room).emit('message', data);
});
// Music
socket.on('play_sound', function (data) {
data.color = socket.color;
console.log("play sound %j", data);
socket.broadcast.to(room).emit('play_sound', data);
});
socket.on('stop_sound', function (data) {
console.log("stop sound %j", data);
socket.broadcast.to(room).emit('stop_sound', data);
});
// Close socket and let others know
socket.on('close', function () {
console.log("closed %j", room);
io.sockets.in(room).emit('closed', room);
socket.leave(room);
});
});
script.js
var socket = io.connect('http://localhost:8080'),
try {
if (! window.AudioContext) {
if (window.webkitAudioContext) {
window.AudioContext = window.webkitAudioContext;
}
}
actx = new AudioContext();
}
catch(e) {
console.log('Web Audio API is not supported in this browser'),
}
window.URL = window.URL || window.webkitURL;
navigator.getUserMedia = navigator.getUserMedia ||
navigator.webkitGetUserMedia ||
navigator.mozGetUserMedia;
if (navigator.getUserMedia === undefined) {
if (console !== undefined) {
console.log("Browser doesn't support getUserMedia");
}
}
var videoElement, canvas, ctx, manip, w, h;
var oldData = null;
window.addEventListener('DOMContentLoaded', setup);
function setup() {
videoElement = document.querySelector("video");
videoElement.width = w = window.innerWidth;
videoElement.height = h = window.innerHeight;
videoElement.autoplay = true;
canvas = document.createElement('canvas'),
canvas.width = w;
canvas.height = h;
ctx = canvas.getContext('2d'),
bcanvas = document.createElement('canvas'),
bcanvas.width = w;
bcanvas.height = h;
bctx = bcanvas.getContext('2d'),
navigator.getUserMedia({video: true}, function (stream) {
videoElement.src = window.URL.createObjectURL(stream);
videoElement.addEventListener('canplay', draw);
}, function() {});
}
var width = window.innerWidth;
var height = window.innerHeight;
var keyboard = SVG('keyboard'),
var keyboardKeys = [83,68,70,71,72,74,75];
var blackKeys = [69,82,89,85,73];
var keyboardPressKeys = [115,100,102,103,104,106,107];
var blackKeyPress = [101, 114, 121, 117, 105];
var octave = 1; // where octave 1 = middle C
var currentCursor = 10;
var color;
var keys = [];
var cursor = [];
for (var i = 0; i < 21; i++) {
keys[i] = keyboard.rect(width/21, height);
keys[i].move(width/21 * i, 0);
keys[i].attr({ fill: '#fff', stroke: '#000', id: "key"+i });
keys[i].mousedown ((function(n) {
return function() {
var key = SVG.get("key"+n);
key.fill({ color: '#f06' });
socket.emit('play_sound', { "i":n,"black":false});
playSound(n, false);
}
})(i));
keys[i].mouseup((function(n) {
return function() {
keys[n].fill({ color: '#fff' });
socket.emit('stop_sound', { "i":n,"black":false});
stopSound(n, false);
}
})(i));
cursor[i] = keyboard.circle(width/21);
cursor[i].attr({ fill: '#ff0000' });
cursor[i].move(width/21 * i, height / 1.5);
cursor[i].hide();
}
var bkeys = [];
var prev = 0;
for (var i = 0; i < 15; i++) {
bkeys[i] = keyboard.rect(width/42, height / 1.7);
bkeys[i].attr({ fill: '#000', stroke: '#000', id: "bkey"+i });
bkeys[i].move(prev + (width/(21*1.3)), 0);
prev = prev + width/21;
if (i == 1 || i == 4 || i == 6 || i == 9 || i == 11) {
prev += width/21;
}
bkeys[i].mousedown ((function(n) {
return function() {
var key = SVG.get("bkey"+n);
key.fill({ color: '#f06' });
socket.emit('play_sound', { "i":n,"black":true});
playSound(n, true);
}
})(i));
bkeys[i].mouseup((function(n) {
return function() {
bkeys[n].fill({ color: '#000' });
socket.emit('stop_sound', { "i":n,"black":true});
stopSound(n, true);
}
})(i));
}
window.addEventListener('keypress', function(e) {
for (var i = 0; i < keyboardPressKeys.length; i++) {
if (e.keyCode == keyboardPressKeys[i]) {
var n = i + octave * 7;
var key = SVG.get("key"+n);
key.fill({ color: '#f06' });
socket.emit('play_sound', { "i":n,"black":false, "color":color});
playSound(n, false);
}
}
for (var i = 0; i < blackKeyPress.length; i++) {
if (e.keyCode == blackKeyPress[i]) {
var n = i + (octave * 5);
var key = SVG.get("bkey"+n);
key.fill({ color: '#f06' });
socket.emit('play_sound', { "i":n,"black":true, "color":color});
playSound(n, true);
}
}
if (e.keyCode == 97 && octave > 0) --octave;
if (e.keyCode == 108 && octave < 2) ++octave;
});
window.addEventListener('keyup', function(e) {
console.log(e.keyCode);
for (var i = 0; i < keyboardKeys.length; i++) {
if (e.keyCode == keyboardKeys[i]) {
var key = SVG.get("key"+(i+octave*7));
key.fill({ color: '#fff' });
socket.emit('stop_sound', { "i":i+octave*7,"black":false, "color":color });
stopSound(i+octave*7, false);
}
}
for (var i = 0; i < blackKeys.length; i++) {
if (e.keyCode == blackKeys[i]) {
var n = i + octave * 5;
var key = SVG.get("bkey"+n);
key.fill({ color: '#000' });
socket.emit('stop_sound', { "i":n,"black":true, "color":color });
stopSound(n, true);
}
}
});
var gainNode = actx.createGainNode();
gainNode.connect(actx.destination);
// White Keys
var o = new Array(21);
for (var i = 0; i < 21; i++) {
o[i] = null;
}
// Black Keys
var ob = new Array(15);
for (var i = 0; i < 15; i++) {
ob[i] = null;
}
var PI_2 = Math.PI*2;
var SAMPLE_RATE = 44100;
var whiteNotes = [130.82, 146.83, 164.81, 174.61, 196, 220, 246.94, 261.63
, 293.66, 329.63, 349.23, 392, 440, 493.88, 523.25, 587.33, 659.26, 698.46, 783.99, 880, 987.77];
var blackNotes = [138.59, 155.56, 185, 207.65, 233.08, 277.18, 311.13, 369.99, 415.3, 466.16, 554.37, 622.25, 739.99, 830.61, 932.33];
function playSound(i, black) {
if ((black && ob[i] === null) || (!black && o[i] === null)) {
var osc = actx.createOscillator();
var freq;
if (black) {
freq = blackNotes[i];
}
else {
freq = whiteNotes[i];
}
osc.type = 3;
osc.frequency.value = freq;
osc.connect(gainNode);
osc.noteOn(0);
if (black) ob[i] = osc;
else o[i] = osc;
}
}
function stopSound(i, black) {
var osc;
if (black) osc = ob[i];
else osc = o[i];
if ((black && ob[i] !== null) || (!black && o[i] !== null)) {
osc.noteOff(0);
osc.disconnect();
if (black) ob[i] = null;
else o[i] = null;
}
}
socket.on('color', function(data) {
color = data;
});
socket.on('play_sound', function(data) {
if (data.black)
bkeys[data.i].fill({ color: data.color });
else
keys[data.i].fill({ color: data.color });
playSound(data.i, data.black);
});
socket.on('stop_sound', function(data) {
if (data.black)
bkeys[data.i].fill({ color: '#000' });
else
keys[data.i].fill({ color: '#fff' });
stopSound(data.i, data.black);
});
function draw() {
if (videoElement.paused || videoElement.ended) {
return;
}
try {
bctx.drawImage(videoElement, 0, 0, w, h);
manip = bctx.getImageData(0, 0, w, h);
var data = manip.data;
if (oldData != null) {
data = motionDetection(data, oldData);
ctx.putImageData(manip, 0, 0);
oldData = null;
}
else {
oldData = manip.data;
}
requestAnimationFrame(draw);
}
catch (e) {
if (e.name == "NS_ERROR_NOT_AVAILABLE") {
setTimeout(draw, 0);
}
else {
throw e;
}
}
}
function motionDetection(data, oldData) {
var motionX = 0;
var count = 0;
// Iterate through each pixel, changing to 255 if it has not changed
for( var y = 0 ; y < h; y++ ) {
for( var x = 0 ; x < w; x++ ) {
var indexOld = (y * w + x) * 4,
oldr = oldData[indexOld],
oldg = oldData[indexOld+1],
oldb = oldData[indexOld+2],
olda = oldData[indexOld+3];
var indexNew = (y * w + x) * 4,
r = data[indexNew],
g = data[indexNew+1],
b = data[indexNew+2],
a = data[indexNew+3];
if (oldr > r - 30 || oldg > g - 30 || oldb > b - 30)
{
data[indexNew] = 255;
data[indexNew+1] = 255;
data[indexNew+2] = 255;
data[indexNew+3] = 255;
}
else
{
data[indexNew] = 0;
data[indexNew+1] = 0;
data[indexNew+2] = 0;
data[indexNew+3] = 255;
motionX += 100*(x/w); // motionX = The percentage of W than X is at
count++;
}
}
}
motionX = motionX / count;
motionX = 100 - motionX;
var key = SVG.get("key"+currentCursor);
if (motionX > 0) {
if (motionX > 100/3 && motionX < (100/3)*2) {
if (o[i] === null) {
key.fill({ color: '#f06' });
playSound(currentCursor, false);
}
console.log('2'),
}
if (motionX > (100/3)*2) {
if (currentCursor + 1 < keys.length) {
cursor[currentCursor].hide();
key.fill({ color: '#fff' });
stopSound(currentCursor, false);
currentCursor++;
cursor[currentCursor].show();
}
console.log('3'),
}
if (motionX < 100/3) {
if (currentCursor - 1 >=0) {
cursor[currentCursor].hide();
key.fill({ color: '#fff' });
stopSound(currentCursor, false);
currentCursor--;
cursor[currentCursor].show();
}
console.log('1'),
}
}
return data;
}
In Figure11-2you can see an example of controlling the cursor (the circle) to press keys on the keyboard using motion detection through a webcam.
Figure 11-2. The keyboard controlled using the cursor through the webcam
Summary
This chapter was mostly about putting it all together, as the previous chapters explained the techniques. I hope it has helped you but also shown you just how easy it is to do some quite impressive things, if you just take it bit by bit. You’ve gone from making a square slide in canvas to moving your hands to make the browser generate sounds.
There are quite a few ways for you to improve the project, such as making the webcam work with black keys (you could perhaps do this by looking for motion on the Y axis as well as the X axis). Or you can just go ahead and make something else—but remember you learn the most by playing around with the code, not just reading a book. I hope you’ve enjoying reading this book. The appendix provides some extra information that is considered out of the scope of the main bulk of the book, as well as recommendations for further reading.