//Midterm progress

Serial Communication
We started doing the serial following the serialport lab which causes a few problems. Also, we did it without setting up all the callback functions so it didn’t work for the first time. So after we properly did all the call back functions,we start to use parseInt() for the serial communication sending a string between the p5 code and the arduino because we have 2 separate parameters to pass (happy and rage) . The parseInt enables us to convert multiple byte number strings to numeric values. 

We had a lot of problems reading the data coming through. We once assumed we can also use the serial monitor to see the data, but it turns out tthat we can only use the console log in p5 to check everything?

p5 code:

let serial;
let portButton;
let inData; // for incoming serial data
let outByte = 0; // for outgoing data

function setup() {
createCanvas(400, 400);

const W2 = 0.5 * width;
const H2 = 0.5 * height;

eyes = new Eyes(W2, 0.7 * H2, 0.3 * W2, 80, 30);
nose = new Nose(W2, H2, 15);
mouth = new Mouth(W2, 1.33 * H2, 0.8 * W2);

joy = 0;
rage = 0

serial = new p5.WebSerial();

// check to see if serial is available:
if (!navigator.serial) {
alert("WebSerial is not supported in this browser. Try Chrome or MS Edge.");
}
// if serial is available, add connect/disconnect listeners:
navigator.serial.addEventListener target="_blank">navigator.serial.addEventListener("connect", portConnect);
navigator.serial.addEventListener("disconnect", portDisconnect);
// check for any ports that are available:
serial.getPorts();
// if there's no port chosen, choose one:
serial.on target="_blank">serial.on target="_blank">serial.on target="_blank">serial.on target="_blank">serial.on("noport", makePortButton);
// open whatever port is available:
serial.on("portavailable", openPort);
// handle serial errors:
serial.on("requesterror", portError);
// handle any incoming serial data:
serial.on("data", serialEvent);
serial.on("close", makePortButton);

}

function draw() {

const MIN = 128;
const R = map(rage,-100, 100, MIN, 255);
const B = map(joy, -100, 100, MIN, 255);
background(color(R, MIN, B));

// draw the face
nose.draw();
eyes.draw(map(rage,-100, 100, -1, 1));
mouth.draw(map(joy,-100, 100, -1, 1));

}


function mouseMoved() {
const delta = 10;
let newJoy = min(map(mouseX, 0, width, -100, 100), 100);
let newRage = min(map(mouseY, 0, height, -100, 100),100);

// send to the arduino
if(abs(joy - newJoy) > delta) {
joy = newJoy;
serial.print target="_blank">serial.print("," + joy);
};
if(abs(rage - newRage) > delta) {
rage = newRage;
serial.print("-" + rage);
}

//console.log target="_blank">console.log target="_blank">console.log target="_blank">console.log target="_blank">console.log("input: " + joy + "," + rage");

}



// Serial
// if there's no port selected,
// make a port select button appear:
function makePortButton() {
// create and position a port chooser button:
portButton = createButton("choose port");
portButton.position(10, 10);
// give the port button a mousepressed handler:
portButton.mousePressed(choosePort);
}

// make the port selector window appear:
function choosePort() {
if (portButton) portButton.show();
serial.requestPort();
}

// open the selected port, and make the port
// button invisible:
function openPort() {
serial.open();
console.log("port open")
// hide the port button once a port is chosen:
if (portButton) portButton.hide();
}

// pop up an alert if there's a port error:
function portError(err) {
alert("Serial port error: " + err);
}
// read any incoming data as a string
// (assumes a newline at the end of it):
function serialEvent() {
let inData = serial.readStringUntil("rn");
console.log(inData);
}

// try to connect if a new serial port
// gets added (i.e. plugged in via USB):
function portConnect() {
console.log("port connected");
serial.getPorts();
}

// if a port is disconnected:
function portDisconnect() {
serial.close target="_blank">serial.close();
console.log("port disconnected");
}

function closePort() {
serial.close();
}

arduino code:

#include <Servo.h>
Servo servoEyeL;
Servo servoEyeR;
Servo servoMouth;
const int pinEyeL = 4;
const int pinEyeR = 5;
const int pinMouth = 6;
const int ledRed = 2;
const int ledBlue = 3;
// float counter = 0;
// int angle = 0;
int joy = 0;
int rage = 0;
int newJoy = 0;
int newRage = 0;


void setup() {
servoEyeL.attach(pinEyeL);
servoEyeR.attach(pinEyeR);
servoMouth.attach(pinMouth);
// put your setup code here, to run once:
pinMode(ledRed, OUTPUT);
pinMode(ledBlue, OUTPUT);
Serial.begin(9600);
Serial.setTimeout(10);
}

void loop() {
if (Serial.available() > 0) {
// if there's serial data available data
int inByte = Serial.read();
if (inByte == 44) {
// ','
newJoy = Serial.parseInt();
}

else if (inByte == 45) {
// '-'
newRage = Serial.parseInt();
}
if (joy != newJoy) {
joy = newJoy;
Serial.print("Joy: ");
Serial.println(joy);
servoMouth.write(map(joy, -100, 100, 45, -10));
analogWrite(ledBlue, map(joy, -100, 100, 0, 255));
}

if (rage != newRage) {
rage = newRage;
Serial.print("Rage: ");
Serial.println(rage);
servoEyeL.write(map(rage, -100, 100, 0, 128));
servoEyeR.write(map(rage, -100, 100, 128, 0));
analogWrite(ledRed, map(rage, -100, 100, 0, 255));
}
}
delay(20);
}














Problems & Solutions

Originally there was a lot of lagging when we did the serial communication so we deleted all the console logs in the p5 code to avoid that. 
Also, originally the serial print for rage and joy did not work properly. Before adding delay, there were a lot of random numbers showing in the console log which we assume it was showing the data in the buffer? We tried with using the Serial.flush() function to clear these random numbers but it didn’t work.  But after adding delay and adding 2 new variables representing the eye and nose, the problem was solved after setting up 2 conditional statements representing the joy and rage data. 

(My arduino stopped functioning after we forgot the important statement of one port at a time and unplugged it when we were using IDE to upload the data...... Tried to reset it, did not work.)

Fabricating the robot’s face
We decided to use lazer cut to cut out the acrylic board we purchased. Here is the layout for the hexagonal box drawn in AI.  The circular element on the bottom right is the arm created for the servo lifting up and down for the rubberband mouth.

Neither Lenin nor I had any previous experience working with lazer cut. Yet we found Kai who was extremely kind to help instruct us how to do it with demonstrations. This marks our first time working with lazercut on our own I guess. And it was much easier than I assumed.

We once had some debates on what material we need to use for the eyes. Lenin insisted to use a solid orange, but I thought the transparent one works better with some see-through of the inner elements we are using. The servo itself can be shaped as the eyes as well through the acrylic board. So we decided to stick with that plan. I also prefer the semi see-through effect of the translucent acrylic board to show some of the inner logic and mechanics of our work.
We once encountered some problems with the servo motor angle rotating the mouth. It was way to high after we install the acrylic arm to it. We calibrated it with some adjustments of the code to make it rotate from 255 to 0 to 60 to 0. For the eyes, we adjusted the physical servo. We also added some boards beneath the servo for the mouth to let it move in the most prefered way. We decided to tape the edges so the entire work is more adjustable instead of using the glue gun.
Adding PoseNet

We then started with adding posenet to the p5 code to make it interactive with people present in front of the camera. Just a little context information of poseNet, it is a pre-trained model to recognize the people’s different body parts/joints such as wrists, shoulders, nose, etc. We first tested with mapping the movement of the eyes and mouth of the robot (joy and rage expression) according to the distance between the wrists and the up and down direction of them with relation to the shoulders. So hands up and down determines the happy and sad expressions and the extent the hands are away and closer to each other determines the rage and calm expressions. Then we changed the factors for happy and sad to the movement. The logic goes as follow:

A: if we have an old pose, record the pose, not do anything.
B: going sad when people’s wrists are down.
C: moving: have an array storing the current poses. Remember the last 2 states of the body movements. Record the distance between the previous and the current nose position and divide the difference by body factor so it won’t get too high. We add the difference into the array, and if the distance’s average is more than a certain number, it will executes the reaction for joy.

Here is the code for the p5 with posenet:

We then calibrated the arduino code a bit with attempts to add easing to the movements. Yet, it turns out that the easing we added kind of restrict the servo’s movement? 
The easing code: rage = (newRage - rage) / 2;

Here is the arduino code (we didn’t do much editing to it):

#include <Servo.h>




Servo servoEyeL;

Servo servoEyeR;

Servo servoMouth;




const int pinEyeL = 4;

const int pinEyeR = 5;

const int pinMouth = 6;




const int ledRed = 2;

const int ledBlue = 3;




// float counter = 0;

// int angle = 0;




int joy = 0;

int rage = 0;

int newJoy = 0;

int newRage = 0;




void setup() {




  servoEyeL.attach(pinEyeL);

  servoEyeR.attach(pinEyeR);

  servoMouth.attach(pinMouth);




  // put your setup code here, to run once:




  pinMode(ledRed, OUTPUT);

  pinMode(ledBlue, OUTPUT);




  Serial.begin(9600);

  Serial.setTimeout(10);

/*

  int v = 0;

  servoEyeL.write(v);

  servoEyeR.write(128 - v);

  servoMouth.write(0);

*/

}




void loop() {




  if (Serial.available() > 0) {  // if there's serial data available data




    int inByte = Serial.read();




    if (inByte == 44) {  // ','

      newJoy = Serial.parseInt();

    } else if (inByte == 45) {  // '-'

      newRage = Serial.parseInt();

    }

   

  }




  if (joy != newJoy) {

    joy = newJoy;

    servoMouth.write(map(joy, -100, 100, 45, 0));

    analogWrite(ledBlue, map(joy, -100, 100, 0, 255));

  } else {

    joy = newJoy;

  }




  if (rage != newRage) {

    rage = newRage;

    servoEyeL.write(map(rage, -100, 100, 0, 128));

    servoEyeR.write(map(rage, -100, 100, 128, 0));

    analogWrite(ledRed, map(rage, -100, 100, 0, 255));

  } else {

    rage = newRage;

  }




  delay(20);

}

Some Reflections
This is what I both like and feel uncertain about our project: to me, the relation between human and machine is reversed in this project. While it should machines doing service to the human, now it is us constantly trying to entertain it with our labor devoted. Is this beneficial or detremental?