Stage (14) Week Commencing (31.03.14)
Week (25)
Description
This weeks session was spent getting the Kinect tracking system to work with Arduino. I managed to get the skeleton tracking with simpleOpenNi working at home which in the past did not work but with a bit of tweaking I managed to get it working. The problem was that the simpleOpenNi library had been updated and the code I was using was set up for use with the older library. (Code I was using out of the book -
‘Arduino and Kinect Projects’, Melgar et al, 2012) was now outdated.
//code - Processing
import processing.serial.*;
import SimpleOpenNI.*;
SimpleOpenNI kinect;
Serial arduinoport;
void setup() {
size(1024, 480);
textSize(20);
if (Serial.list().length < 1) {
println("No serial port seen. Quitting.");
exit();
}
if (Serial.list().length > 1) {
print("There are multiple serial ports. Using the first one: ");
println(Serial.list()[0]);
}
arduinoport = new Serial(this, Serial.list()[0], 115200);
//delay(2000); // Wait for arduino to reboot
kinect = new SimpleOpenNI(this);
kinect.enableDepth();
kinect.enableUser();
}
void draw() {
kinect.update();
PImage depth = kinect.depthImage();
image(depth, 0, 0);
IntVector userList = new IntVector();
kinect.getUsers(userList);
while (arduinoport.available() > 0) {
char inByte = (char)arduinoport.read();
print(inByte);
}
boolean stopRobot = true; // Only move robot if someone is there
for (int i=0; i < userList.size(); i++) {
int userId = userList.get(i);
if (kinect.isTrackingSkeleton(userId)) {
loat x = 0;
float y = 0;
float rotation = 0;
// Get the position of the torso
PVector torsopos = new PVector();
float torsoconfidence = kinect.getJointPositionSkeleton(userId, SimpleOpenNI.SKEL_TORSO, torsopos);
if (torsoconfidence > 0.5) {
displayJoint(torsopos);
float userPos = torsopos.x;
userPos = map(userPos, -600, 600, 1000, -1000);
userPos = constrain(userPos, -1000, 1000);
}
PVector rightpos = new PVector();
float rightconfidence = kinect.getJointPositionSkeleton(userId, SimpleOpenNI.SKEL_RIGHT_HAND, rightpos);
PVector leftpos = new PVector();
float leftconfidence = kinect.getJointPositionSkeleton(userId, SimpleOpenNI.SKEL_LEFT_HAND, leftpos);
if (rightconfidence > 0.5 && leftconfidence > 0.5) {
displayJoint(rightpos);
displayJoint(leftpos);
PVector differenceVector = PVector.sub(rightpos, leftpos);
x = constrain(map(differenceVector.x, -600, 600, -1000, 1000), -1000, 1000);
y = constrain(map(differenceVector.z, -200, 200, -1000, 1000), -1000, 1000);
if (differenceVector.y > 0) { // If left hand is higher
x = -x;
y = -y;
}
//print(x); print("\t"); print (y); print("\t");
}
//println(userPos);
String arduinocmd = String.format("<r,%.0f,%.0f,%.0f>", x, y, rotation);//"<r,"+x+","+y+"," + rotation + ">";
fill(50);
rect(725, 175, 250, 250);
fill(255);
ellipse(map(x, -1000, 1000, -100, 100) + 850, map(y, -1000, 1000, -100, 100) + 300, 20, 20);
//println (arduinocmd);
arduinoport.write(arduinocmd);
stopRobot = false;
}
}
if (stopRobot) {
arduinoport.write("<r,0,0,0>");
}
}
void onNewUser(SimpleOpenNI kinect,int userId) {
kinect.startTrackingSkeleton(userId);
}
void onEndCalibration(int userId, boolean successful) {
background(0);
fill(255);
if (successful) {
text(" User Calibrated!", 700, 30);
kinect.startTrackingSkeleton(userId);
}else {
text(" Failed to calibrate user", 700, 30);
//kinect.startPoseDetection("Psi", userId);
}
}
void onStartPose(String pose, int userId) {
background(0);
fill(255);
text("Started pose for user " + userId, 700, 30);
//kinect.stopPoseDetection(userId);
//kinect.requestCalibrationSkeleton(userId, true);
}
void displayJoint(PVector joint) {
PVector converted = new PVector();
kinect.convertRealWorldToProjective(joint, converted);
pushMatrix();
noStroke();
fill(255,0,0);
ellipse(converted.x, converted.y, 20, 20);
popMatrix();
}
...................................................................................................................................................................
Code Arduino//
float temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9;
// Joint Servos
int servo3Pin = 3; //shoulder 1
int servo4Pin = 4; // arm 1
int servo7Pin = 7; // hip 1
int servo8Pin = 8; // leg 1
int servo9Pin = 9; // hip 2
int servo10Pin = 10; //leg 2
int servo11Pin = 11; //rotation
int pulse2 = 1500;
int pulse3 = 2500;
unsigned long previousMillis = 0;
long interval = 20;
pinMode (servo3Pin, OUTPUT);
pinMode (servo4Pin, OUTPUT);
pinMode (servo5Pin, OUTPUT);
pinMode (servo6Pin, OUTPUT);
pinMode (servo7Pin, OUTPUT);
pinMode (servo8Pin, OUTPUT);
pinMode (servo9Pin, OUTPUT);
pinMode (servo10Pin, OUTPUT);
pinMode (servo11Pin, OUTPUT);
if (Serial.available() > 18) {
int servo5Pin = 5; // shoulder 2
int servo6Pin = 6; // arm 2
//initial pulse values
int pulse1 = 500;
int pulse4 = 1500;
int pulse5 = 500;
int pulse6 = 2500;
int pulse7 = 2500;
int pulse8 = 500;
int pulse9 = 1500;
int speedServo = 0;
void setup() {
Serial.begin(9600);
}
void loop() {
char led = Serial.read();
if (led == 'S'){
temp1 = Serial.read();
temp2 = Serial.read();
temp3 = Serial.read();
temp4 = Serial.read();
temp5 = Serial.read();
temp6 = Serial.read();
temp7 = Serial.read();
temp8 = Serial.read();
temp9 = Serial.read();
}
}
//Next, remap the angles from the incoming range (0-255) to the servo range (500-2500).
pulse9 = (int)map(temp1,0,255,2500,500);
pulse2 = (int)map(temp2,0,255,500,2500);
pulse1 = (int)map(temp3,0,255,500,2500);
pulse4 = (int)map(temp4,0,255,2500,500);
pulse3 = (int)map(temp5,0,255,500,2500);
pulse6 = (int)map(temp6,0,255,2500,500);
pulse5 = (int)map(temp7,0,255,2500,500);
pulse8 = (int)map(temp8,0,255,500,2500);
pulse7 = (int)map(temp9,0,255,2500,500);
//And finally, move the servo to the new position.
unsigned long currentMillis = millis();
if(currentMillis - previousMillis > interval)
//rotation
//leftElbow
//left Shoulder
//left Knee
//left Hip
//right Elbow
//right Shoulder
//right Knee
//right Hip
{
previousMillis = currentMillis;
updateServo(servo3Pin, pulse1);
updateServo(servo4Pin, pulse2);
updateServo(servo5Pin, pulse3);
updateServo(servo6Pin, pulse4);
updateServo(servo7Pin, pulse5);
updateServo(servo8Pin, pulse6);
updateServo(servo9Pin, pulse7);
updateServo(servo10Pin,pulse8);
updateServo(servo11Pin, pulse9);
}
}
void updateServo (int pin, int pulse){
digitalWrite(pin, HIGH);
delayMicroseconds(pulse);
digitalWrite(pin, LOW):}
I took my Kinect unit and robot into Uni this week and managed to get some time with my tutor. we sat down and after a few hours struggling to get the Kinect to output co-ordinates to the Arduino we managed to get it working ...just. We got slight twitches on the servo, after my tutor left I spent a few hours divinding the values of the co-ordinates coming out of processing into Arduino and managed to get the tracking nearly spot on, however when I tried to duplicate this at home on my dining room table it did not function correctly. This is due to the placement of the Kinect relative to hight and distance from the user. This means that every time I want to move the project as a whole I will have to spend time re-calibrating the tracking on the Kinect to get the robot functioning correctly. This could prove to be a major issue when setting up the project in different places. (1.1)
Another issue concerns, when connecting the Kinect unit to the USB port of my laptop the Arduino overrides the computers Keyboard and mouse cutting it out. This is because the new Ardunio board (Leonardo) uses a driver that is the same as the apple keyboard and pointer, and because the Kinect was drawing a lot of power from the USB the computer used the Arduino driver in place of its own keyboard driver. This is an issue that will have to be worked around, I plan on getting a wireless mouse and keyboard to use for when this anomaly happens. I have purchased a usb power hub with 900ma at 5 amps output to each individual port, hopefully this will also help any power issues.
Feelings
I am pretty happy with how the project is running at the moment, I am getting great input from my tutors and I think that when it is finished the project will work and stand well as a installation piece.
Evaluation
This session was great experience, the code I had been working on at home using the kinect - arduino book helped me understand how the process was going to work. However it was my tutor who really explained the full process in depth and how to utilise it in the project - which I found easier to understand than the lengthy explanations in the book.
Analysis
I am pleased with the current situation of project, however I am becoming increasingly aware of how much more work I have to do on other areas of the project such as - Designing a brief, A poster, Conceptual purpose and if I have time I want to write a short study on the research behind the project including testing of some of the final elements against original systems - such as Eliza and my second year animatronic project, in order to show the development of ideas and process.
Conclusion
In conclusion I am very pleased with where I am at currently however I am aware that time is running out and that finding the time to spend on this project will become more difficult as time goes on because of other obligations.
Action Plan
The next stages of the project will have to be planned in order to fit them within the dwindling time frame. Things are need to be addressed in the next few weeks are.
Power Issues - Servo shield V.5 with 6v 2A psi to power all servos
Power Issues - Arduino, Usb Hub 5v, 900ma
Driver issues - Wireless keyboard and mouse
1.1
