Projekt Leaper

Členové projektu

Vratislav Kalenda, Studia nových médií, e-mail: v.kalenda@gmail.com

Marta Kolárová, Studia nových médií, e-mail: martavkolarova@gmail.com

Motivace vzniku projektu a jeho význam

Původním záměrem bylo pomocí leap motion naprogramovat světelné diody - ovládané pohyby. Následně jsme se rozhodli využít leap motion i pro vytváření a modulování zvuku, tj. vytvořit pohyby ovládaný virtuální hudební nástroj v ideálním případě napojený na knihovnu samplovaných zvuků. A ještě kombinovaný s barevnými světelnými diodami, které informují o hraných tónech. Cílem byla zároveň možnost spoluhry více hráčů a vytváření harmonií.

V první fázi byl zvuk generován přímo počítačem (bylo potřeba nastavit temperované ladění a rozmezí oktáv), v závěrečné fázi se ale podařilo systém propojit s knihovnou zvuků GarageBand pomocí protokolu MIDI – v současné době je k dispozici využití veškerých nástrojů GarageBand.

Vlastní zdrojový kód

Upozornění! Vzhledem k omezení syntaxe wikisofia kopírujte prosím zdrojový kód přímo z editace zdroje stránky (nebo využijte možnosti si jej stáhnout), nikoliv z podoby, do jaké je intepretována při zobrazení na stránce. Přislušný úsek je v souladu se syntaxí české wikipedie umístěn mezi značkou <tt> a </tt>.

var lifx = require('./lifx'); var util = require('util'); var packet = require('./packet'); var leap = require('leapjs'); var midi = require('midi');

var lx = lifx.init();

lx.on('bulbstate', function(b) { //console.log('Bulb state: ' + util.inspect(b)); });

lx.on('bulbonoff', function(b) { //console.log('Bulb on/off: ' + util.inspect(b)); });

Number.prototype.map = function ( in_min , in_max , out_min , out_max ) {

 return ( this - in_min ) * ( out_max - out_min ) / ( in_max - in_min ) + out_min;

}

// Set up a new output. var output = new midi.output();

// Count the available output ports. var count = output.getPortCount();

console.log("Available ports: " + count)

// Get the name of a specified output port. output.getPortName(0);

// Open the first available output port. output.openPort(0);

// Send a MIDI message. output.sendMessage([176,22,1]);

var damperPedal = true;

var toneMap = { 'c' : [ 60,72,84], //'cis' : [277.18,554.36], 'd' : [ 62,74], //'dis' : [ 311.13,622.26], 'e' : [ 64,76], 'f' : [ 65,77], //'fis' : [ 369.00,739.98], 'g' : [67,79], //'gis' : [ 415.00,830.60], 'a' : [69,81], //'ais' : [233.08,466.17], 'h' : [71,83] }

var dateDiff = function(start,end){ return (end.getTime() - start.getTime())/1000; }

var findTone = function(freq){ var diff = 9999999999; var tone = null; for(var toneName in toneMap){ //console.log("looking in " + toneName) for(var i = 0; i < toneMap[toneName].length; i++){ var newDiff = Math.abs(freq- toneMap[toneName][i]); //console.log("diff " + newDiff) if(newDiff < diff){ diff = newDiff; tone = {name : toneName, freq : toneMap[toneName][i]} //console.log("Found " + toneName + i) } } } if(tone != null){ //console.log("Found " + tone.name + " " + tone.freq + "hz") }else{ //console.log("No tone for " + freq) }

return tone;

}

var midiMessages = { "noteOn" : parseInt('10010000', 2), "noteOff" : parseInt('10000000',2), "mode" : parseInt('10110000',2), "damper" : parseInt('01000000',2), "pedalOff" : 127, "pedalOn" : 0 }

var findToneByMidi = function(step){ var diff = 9999999999; var tone = null; var stepIndex = 0; var i = 0; for(i = 0; i< 3; i++){

for(var toneName in toneMap){ //console.log("looking in " + toneName)

if(toneMap[toneName].length > i){

var newDiff = Math.abs(step-stepIndex); //console.log("aaa" + newDiff) //console.log("diff " + newDiff) if(newDiff < diff){ diff = newDiff; tone = {name : toneName, freq : toneMap[toneName][i]} //console.log("Found " + toneName + i) } stepIndex++; }

} }

if(tone != null){ //console.log("Found " + tone.name + " " + tone.freq + "hz") }else{ //console.log("No tone for " + freq) }

return tone; }

var InterpolatedVal = function(){ this.data = []; this.windowSize = 5; this.add = function(val){ this.data.unshift(val); //console.log("Pusging value " + val + " win size " + this.windowSize) if(this.data.length > this.windowSize){ this.data.pop(val) } //console.log("CData lenght " + this.data.length) } this.getValue = function(){ var result = 0.0; for(var i = 0; i < this.data.length; i++){ result += this.data[i]; } //console.log("value " + result / this.data.length) return result / this.data.length }

};

var Hand = function(){ this.grip = new InterpolatedVal(); this.grip.windowSize = 30; this.rotation = new InterpolatedVal(); this.rotation.windowSize = 10; this.bulb = null;

this.note = new InterpolatedVal(); };

var leftHand = new Hand(); var rightHand = new Hand();

var baudio = require('baudio'); var leftTone = 440; var rightTone = 440; var curLeftTone = leftTone; var curRightTone = rightTone; var note = null;

var n = 0; // var b = baudio({rate: 704000},function (t) { // //console.log(t) // //console.log(leftTone) // var x = Math.sin(2* (rightTone+0.01) * Math.PI * t) ;//+ Math.sin(n)); // var next_x = Math.sin(2* (curLeftTone+0.02) * Math.PI * t); // var a = Math.sin(2* (leftTone+0.01) * Math.PI * t) ; // // if(Math.abs(x) < 0.001 && x < next_x) { // // curLeftTone = leftTone // // x = 0; // // } // // if(Math.abs(a) < 0.001) { // // curRightTone = rightTone // // a = 0; // // } // //n += Math.sin(t); // return (x * rightHand.grip.getValue()) + (a * leftHand.grip.getValue()); // }); // b.play();

// var baudio = require('baudio'), tune = require('tune');

// var blackholesun = tune(String( // 'A4 E4 A5 D5 A5 E4 A4 C4 E4 A5 D5 . . . ' + // 'G3 D4 G4 D5 G4 D4 E3 F#3 C#4 F#4 C#5 . . . ' + // 'E3 F3 C4 F4 A#5 F4 C4 F3 E3 D4 E4 B5 . . .' // ).split(' '));

// var b = baudio(function(t) { // // black hole sun + some fx // return blackholesun(t) + Math.sin(2 * Math.PI * t); // }); // b.play();

leftBulb = {}; leftBulb.isOn = false; rightBulb = {}; rightBulb.isOn = false; lx.on('bulb', function(b) { console.log('New bulb found: ' + b.name + " : " + b.addr.toString("hex")); addr = b.addr.toString("hex"); if(addr == "d073d5001d70"){ console.log("Assigning left bulb"); leftBulb.bulb = lx.bulbs[addr]; leftHand.bulb = leftBulb }else if(addr == "d073d50139d0"){ console.log("Assigning right bulb"); rightHand.bulb = rightBulb rightBulb.bulb = lx.bulbs[addr]; }

});

lx.on('gateway', function(g) { console.log('New gateway found: ' + g.ip); });

lx.on('packet', function(p) { // Show informational packets switch (p.packetTypeShortName) { case 'powerState': case 'wifiInfo': case 'wifiFirmwareState': case 'wifiState': case 'accessPoint': case 'bulbLabel': case 'tags': case 'tagLabels': //case 'lightStatus': case 'timeState': case 'resetSwitchState': case 'meshInfo': case 'meshFirmware': case 'versionState': case 'infoState': case 'mcuRailVoltage': console.log(p.packetTypeName + " - " + p.preamble.bulbAddress.toString('hex') + " - " + util.inspect(p.payload)); break; default: break; } });

console.log("Keys:"); console.log("Press 1 to turn the lights on"); console.log("Press 2 to turn the lights off"); console.log("Press 3 to turn the lights a dim red"); console.log("Press 4 to turn the lights a dim purple"); console.log("Press 5 to turn the lights a bright white"); console.log("Press 6 to cycle forwards through colours"); console.log("Press 7 to cycle backwards through colours"); console.log("Press 8 to show debug messages including network traffic"); console.log("Press 9 to hide debug messages including network traffic"); console.log("Press letters a-m to request various status fields");

var stdin = process.openStdin(); process.stdin.setRawMode(true); process.stdin.resume();

var cycledColour = 0;

var exit = false;

stdin.on('data', function (key) { //process.stdout.write('Got key ' + util.inspect(key) + '\n'); switch (key[0]) { case 0x20: console.log("hit") if(note != null){ sendMidi(null,note.freq,127); } case 0x31: // 1 console.log("Lights on"); // lx.lightsOn('Bedroom Lamp'); // Can specify one bulb by name var b = lx.bulbs['d073d5014163']; lx.lightsOn(b); // lx.lightsOn(); break;

case 0x32: // 2 console.log("Lights off"); var b = lx.bulbs['d073d5014163']; lx.lightsOff(b); // lx.lightsOff(); break;

case 0x33: // 3 console.log("Dim red"); // BB8 7D0 lx.lightsColour(0x0000, 0xffff, 1000, 0, 0); break;

case 0x34: // 4 console.log("Dim purple"); //lx.lightsColour(0x0000, 0xffff, 500, 0, 0); lx.lightsColour(0xcc15, 0xffff, 500, 0, 0); break;

case 0x35: // 5 console.log("Bright white"); lx.lightsColour(0x0000, 0x0000, 0x8000, 0x0af0, 0x0513); break;

case 0x36: // 6 cycledColour = (cycledColour+100) & 0xffff; console.log("Colour value is " + cycledColour); lx.lightsColour(cycledColour, 0xffff, 0x0200, 0x0000, 0x0000); break;

case 0x37: // 7 cycledColour = (cycledColour-100) & 0xffff; console.log("Colour value is " + cycledColour); lx.lightsColour(cycledColour, 0xffff, 0x0200, 0x0000, 0x0000); break;

case 0x38: // 8 console.log("Enabling debug"); lifx.setDebug(true); break;

case 0x39: // 9 console.log("Disabling debug"); lifx.setDebug(false); break;

case 0x61: // a console.log("Requesting voltage"); var message = packet.getMcuRailVoltage(); lx.sendToAll(message); break;

case 0x62: // b console.log("Requesting power state"); var message = packet.getPowerState(); lx.sendToAll(message); break;

case 0x63: // c console.log("hit") if(note != null){ sendMidi(midiMessages['noteOn'],note.freq,127); } break;

case 0x64: // d console.log("Requesting wifi firmware state"); var message = packet.getWifiFirmwareState(); lx.sendToAll(message); break;

case 0x65: // e console.log("Requesting wifi state"); var message = packet.getWifiState({interface:2}); lx.sendToAll(message); break;

case 0x66: // f console.log("Requesting bulb label"); var message = packet.getBulbLabel(); lx.sendToAll(message); break;

case 0x67: // g console.log("Requesting tags"); var message = packet.getTags(); lx.sendToAll(message); break;

case 0x68: // h console.log("Requesting tag label for tag 1"); var message = packet.getTagLabels({tags:new Buffer([1,0,0,0,0,0,0,0])}); lx.sendToAll(message); break;

case 0x69: // i console.log("Requesting time"); var message = packet.getTime(); lx.sendToAll(message); break;

case 0x6a: // j console.log("Requesting info"); var message = packet.getInfo(); lx.sendToAll(message); break;

case 0x6b: // k console.log("Requesting reset switch state"); var message = packet.getResetSwitchState(); lx.sendToAll(message); break;

case 0x6c: // l console.log("Requesting mesh info"); var message = packet.getMeshInfo(); lx.sendToAll(message); break;

case 0x6d: // m console.log("Requesting mesh firmware"); var message = packet.getMeshFirmware(); lx.sendToAll(message); break;

case 0x03: // ctrl-c console.log("Closing..."); lx.close(); // Close the port when done. output.closePort(); process.stdin.pause(); exit = true; process.exit(); break;

} });

var lastMidiMessage = new Date();

function sendMidi(c,b1,b2){ var diff = dateDiff(lastMidiMessage,new Date()); if(c == null){ c = midiMessages['noteOn'] } console.log("Diff is " + diff); if(diff > 0.10){ lastMidiMessage = new Date(); console.log("Sending c:"+c + " n:" + b1 + " v:" + b2) output.sendMessage([c,b1,b2]); if(c == midiMessages['noteOn']){ setTimeout(function(){ console.log("Sending OFF c:"+midiMessages['noteOff'] + " n:" + b1 + " v:" + b2) output.sendMessage([midiMessages['noteOff'],b1,b2]); }, 20); }

} }

function vectorToString(vector, digits) {

 if (typeof digits === "undefined") {
   digits = 1;
 }
 return "(" + vector[0].toFixed(digits) + ", "
            + vector[1].toFixed(digits) + ", "
            + vector[2].toFixed(digits) + ")";

}

var lightsOn = false; var lastNote = null; var previousFrame = null; gripInterpol = new InterpolatedVal toneInterpol = new InterpolatedVal

leap.loop(function(frame){

 if (frame.hands.length > 0) {
   for (var i = 0; i < frame.hands.length; i++) {
     var hand = frame.hands[i];
     var handString = "";

handString += "

";

     handString += "Hand ID: " + hand.id + "
";
     handString += "Type: " + hand.type + " hand" + "
";
     handString += "Direction: " + vectorToString(hand.direction, 2) + "
";
     handString += "Palm position: " + vectorToString(hand.palmPosition) + " mm
";
     handString += "Grab strength: " + hand.grabStrength + "
";
     handString += "Pinch strength: " + hand.pinchStrength + "
";
     handString += "Confidence: " + hand.confidence + "
";
     handString += "Arm direction: " + vectorToString(hand.arm.direction()) + "
";
     handString += "Arm center: " + vectorToString(hand.arm.center()) + "
";
     handString += "Arm up vector: " + vectorToString(hand.arm.basis[1]) + "
";
     //console.log(tone)

     //console.log(handString);
     var handModel = null; 
     if(hand.type == "right"){
     	handModel = rightHand;
     }else{
     	handModel =  leftHand;
     	console.log(handModel.grip.getValue());
     }

     handModel.grip.add(hand.grabStrength);
     handModel.note.add(hand.palmPosition[1])
     
     var value = handModel.note.getValue();
     value = Math.max(value,60);
     value = Math.min(value,500);
     value = value.map(60,500,0,14)
     note = findToneByMidi(value);

     if(handModel.grip.getValue() > 0.8 && damperPedal == true){
     	console.log("gripping " + handModel.grip.getValue())
     	
     	//sendMidi(midiMessages["mode"],parseInt('00000001',2),80)
     	sendMidi(midiMessages["mode"],midiMessages["damper"],midiMessages["pedalOff"])
     	damperPedal = false
     }
     else if(damperPedal == false && handModel.grip.getValue() < 0.8){
     	damperPedal = true
     	console.log("letting go")
     	sendMidi(midiMessages["mode"],midiMessages["damper"],midiMessages["pedalOn"])
     	//sendMidi(midiMessages["mode"],parseInt('00000001',2),0)
     }

     if(lastNote != note.freq){
     	lastNote = note.freq;
     	console.log(note.freq)
     }

if(note != null && hand.type == "right"){ // Send a MIDI message. //sendMidi(midiMessages['noteOn'],note.freq,127);

//rightTone = note.freq; //console.log("assigning right tone " + rightTone) }else if(note != null){ //leftTone = note.freq; //sendMidi(midiMessages['noteOn'],note.freq,127);

//console.log("assigning left tone " + leftTone) }

     if (previousFrame && previousFrame.valid) {
       var translation = hand.translation(previousFrame);
       //handString += "Translation: " + vectorToString(translation) + " mm
";

       var rotationAxis = hand.rotationAxis(previousFrame, 2);
       var rotationAngle = hand.rotationAngle(previousFrame);
       handModel.rotation.add(rotationAxis[0]);
       //console.log('rotation ' + handModel.rotation.getValue())
       //handString += "Rotation axis: " + vectorToString(rotationAxis) + "
";
       //console.log(handString)
     }

// Store frame for motion functions previousFrame = frame;

     // if(handModel.grip.getValue() >= 1 && handModel.bulb.isOn == false){
     // 	lx.lightsOn(handModel.bulb.bulb);
     // 	handModel.bulb.isOn = true;
     // }else if(handModel.grip.getValue() <= 0.2  && handModel.bulb.isOn == true){
     // 	lx.lightsOff(handModel.bulb.bulb);
     // 	handModel.bulb.isOn = false;
     // }

   }
  }

});

Popis vývoje a konečné verze vlastního produktu

Fotodokumentace