Goal
In this tutorial, we will use the ExcitementState and
PleasureState characteristics to interpret the basic emotion of the
human in front of Pepper.
Prerequisites
Before stepping in this tutorial, you should:
Let’s start a new project
For further details, see: Creating a robot application.
Create the BasicEmotion enumeration to represent the human basic emotion:
/**
* Represent a basic emotion.
*/
enum class BasicEmotion {
UNKNOWN,
NEUTRAL,
CONTENT,
JOYFUL,
SAD,
ANGRY
}
/**
* Represent a basic emotion.
*/
public enum BasicEmotion {
UNKNOWN,
NEUTRAL,
CONTENT,
JOYFUL,
SAD,
ANGRY
}
And a listener to react to a basic emotion change:
/**
* Listener used to notify when the basic emotion changes.
*/
interface OnBasicEmotionChangedListener {
fun onBasicEmotionChanged(basicEmotion: BasicEmotion)
}
/**
* Listener used to notify when the basic emotion changes.
*/
public interface OnBasicEmotionChangedListener {
void onBasicEmotionChanged(BasicEmotion basicEmotion);
}
We need a class to observe the basic emotion of the human in front of Pepper.
Create the BasicEmotionObserver class:
/**
* Observe the basic emotion of the first human seen by the robot.
*/
public class BasicEmotionObserver {
// Store the basic emotion listener.
private var listener: OnBasicEmotionChangedListener? = null
// Store the HumanAwareness service.
private var humanAwareness: HumanAwareness? = null
// Store the observed emotion.
private var observedEmotion: Emotion? = null
// Observe the last excitement state and notify the listener.
private var lastExcitement by Delegates.observable(ExcitementState.UNKNOWN) { _, _, _ ->
notifyListener()
}
// Observe the last pleasure state and notify the listener.
private var lastPleasure by Delegates.observable(PleasureState.UNKNOWN) { _, _, _ ->
notifyListener()
}
// Observe the last basic emotion state and change basic emotion.
private var lastBasicEmotion by Delegates.observable(BasicEmotion.UNKNOWN) { _, _, new ->
listener?.onBasicEmotionChanged(new)
}
/**
* Start the observation.
* @param qiContext the qiContext
*/
fun startObserving(qiContext: QiContext) {
// Get the HumanAwareness service.
humanAwareness = qiContext.humanAwareness
// Retrieve the humans around and update the observed emotion.
val humansAround: List<Human> = humanAwareness.humansAround
updateObservedEmotion(humansAround)
// Update the observed emotion when the humans around change.
humanAwareness.addOnHumansAroundChangedListener { this.updateObservedEmotion(it) }
this.humanAwareness = humanAwareness
}
/**
* Stop the observation.
*/
fun stopObserving() {
// Clear observed emotion.
clearObservedEmotion()
// Remove listener on HumanAwareness.
humanAwareness?.let {
it.removeAllOnHumansAroundChangedListeners()
humanAwareness = null
}
}
private fun updateObservedEmotion(humansAround: List<Human>) {
// Clear observed emotion.
clearObservedEmotion()
if (humansAround.isNotEmpty()) {
// Update observed emotion.
val observedHuman: Human = humansAround[0]
observedEmotion = observedHuman.emotion
// Get and store human excitement and pleasure.
lastExcitement = observedEmotion.excitement
lastPleasure = observedEmotion.pleasure
// Notify the listener when excitement changes.
observedEmotion.addOnExcitementChangedListener { excitementState ->
if (excitementState != lastExcitement) {
lastExcitement = excitementState
}
}
// Notify the listener when pleasure changes.
observedEmotion.addOnPleasureChangedListener { pleasureState ->
if (pleasureState != lastPleasure) {
lastPleasure = pleasureState
}
}
this.observedEmotion = observedEmotion
}
}
private fun clearObservedEmotion() {
// Remove listeners on observed emotion.
observedEmotion?.let {
it.removeAllOnExcitementChangedListeners()
it.removeAllOnPleasureChangedListeners()
observedEmotion = null
}
}
private fun computeBasicEmotion(excitement: ExcitementState, pleasure: PleasureState): BasicEmotion {
if (excitement == ExcitementState.UNKNOWN) {
return BasicEmotion.UNKNOWN
}
return when (pleasure) {
PleasureState.UNKNOWN -> BasicEmotion.UNKNOWN
PleasureState.NEUTRAL -> BasicEmotion.NEUTRAL
PleasureState.POSITIVE -> if (excitement == ExcitementState.CALM) BasicEmotion.CONTENT else BasicEmotion.JOYFUL
PleasureState.NEGATIVE -> if (excitement == ExcitementState.CALM) BasicEmotion.SAD else BasicEmotion.ANGRY
}
}
private fun notifyListener() {
// Compute the basic emotion.
val basicEmotion: BasicEmotion = computeBasicEmotion(lastExcitement, lastPleasure)
// Changing only if the basic emotion changed.
if (basicEmotion != lastBasicEmotion) {
lastBasicEmotion = basicEmotion
}
}
}
/**
* Observe the basic emotion of the first human seen by the robot.
*/
public class BasicEmotionObserver {
// Store the basic emotion listener.
private OnBasicEmotionChangedListener listener;
// Store the HumanAwareness service.
private HumanAwareness humanAwareness;
// Store the observed emotion.
private Emotion observedEmotion;
// Store the last excitement, pleasure and basic emotion.
private ExcitementState lastExcitement;
private PleasureState lastPleasure;
private BasicEmotion lastBasicEmotion;
/**
* Start the observation.
* @param qiContext the qiContext
*/
public void startObserving(QiContext qiContext) {
// Get the HumanAwareness service.
humanAwareness = qiContext.getHumanAwareness();
// Retrieve the humans around and update the observed emotion.
List<Human> humansAround = humanAwareness.getHumansAround();
updateObservedEmotion(humansAround);
// Update the observed emotion when the humans around change.
humanAwareness.addOnHumansAroundChangedListener(this::updateObservedEmotion);
}
/**
* Stop the observation.
*/
public void stopObserving() {
// Clear observed emotion.
clearObservedEmotion();
// Remove listeners on HumanAwareness.
if (humanAwareness != null) {
humanAwareness.removeAllOnHumansAroundChangedListeners();
humanAwareness = null;
}
}
/**
* Set the listener.
* @param listener the listener
*/
public void setListener(OnBasicEmotionChangedListener listener) {
this.listener = listener;
}
private void updateObservedEmotion(List<Human> humansAround) {
// Clear observed emotion.
clearObservedEmotion();
if (!humansAround.isEmpty()) {
// Update observed emotion.
Human observedHuman = humansAround.get(0);
observedEmotion = observedHuman.getEmotion();
// Get and store human excitement and pleasure.
lastExcitement = observedEmotion.getExcitement();
lastPleasure = observedEmotion.getPleasure();
// Notify the listener.
notifyListener();
// Notify the listener when excitement changes.
observedEmotion.addOnExcitementChangedListener(excitementState -> {
if (excitementState != lastExcitement) {
lastExcitement = excitementState;
notifyListener();
}
});
// Notify the listener when pleasure changes.
observedEmotion.addOnPleasureChangedListener(pleasureState -> {
if (pleasureState != lastPleasure) {
lastPleasure = pleasureState;
notifyListener();
}
});
}
}
private void clearObservedEmotion() {
// Remove listeners on observed emotion.
if (observedEmotion != null) {
observedEmotion.removeAllOnExcitementChangedListeners();
observedEmotion.removeAllOnPleasureChangedListeners();
observedEmotion = null;
}
}
private BasicEmotion computeBasicEmotion(ExcitementState excitement, PleasureState pleasure) {
if (excitement == ExcitementState.UNKNOWN || pleasure == PleasureState.UNKNOWN) {
return BasicEmotion.UNKNOWN;
}
switch (pleasure) {
case POSITIVE:
return (excitement == ExcitementState.CALM) ? BasicEmotion.CONTENT : BasicEmotion.JOYFUL;
case NEGATIVE:
return (excitement == ExcitementState.CALM) ? BasicEmotion.SAD : BasicEmotion.ANGRY;
}
return BasicEmotion.NEUTRAL;
}
private void notifyListener() {
// Compute the basic emotion.
BasicEmotion basicEmotion = computeBasicEmotion(lastExcitement, lastPleasure);
// Notify the listener only if the basic emotion changed.
if (basicEmotion != lastBasicEmotion) {
lastBasicEmotion = basicEmotion;
if (listener != null) {
listener.onBasicEmotionChanged(basicEmotion);
}
}
}
}
This class observes the ExcitementState and PleasureState of the first
human seen by using the HumanAwareness service, transforms them into a
BasicEmotion and notifies its listener when this emotion changes.
Here is the transformation matrix we used, based on James Russel’s work:
This transformation is done in the computeBasicEmotion method.
Make your MainActivity implement the OnBasicEmotionChangedListener
interface:
class MainActivity: RobotActivity(), RobotLifecycleCallbacks, OnBasicEmotionChangedListener
public class MainActivity extends RobotActivity implements RobotLifecycleCallbacks, OnBasicEmotionChangedListener
And override the onBasicEmotionChanged method:
class override onBasicEmotionChanged(basicEmotion: BasicEmotion) {
Log.i(TAG, "Basic emotion changed: $basicEmotion")
}
@Override
public void onBasicEmotionChanged(BasicEmotion basicEmotion) {
Log.i(TAG, "Basic emotion changed: " + basicEmotion);
}
Store the BasicEmotionObserver in the MainActivity:
// Store the basic emotion observer.
private var basicEmotionObserver: BasicEmotionObserver? = null
// Store the basic emotion observer.
private BasicEmotionObserver basicEmotionObserver;
Create it in and subscribe to it the onCreate method:
// Create the basic emotion observer and listen to it.
basicEmotionObserver = BasicEmotionObserver()
basicEmotionObserver?.listener = this
// Create the basic emotion observer and listen to it.
basicEmotionObserver = new BasicEmotionObserver();
basicEmotionObserver.setListener(this);
And unsubscribe from it in the onDestroy method:
// Stop listening to basic emotion observer and remove it.
basicEmotionObserver?.listener = null
basicEmotionObserver = null
// Stop listening to basic emotion observer and remove it.
basicEmotionObserver.setListener(null);
basicEmotionObserver = null;
In the onRobotFocusGained callback, start the emotion observation:
// Start the basic emotion observation.
basicEmotionObserver?.startObserving(qiContext)
// Start the basic emotion observation.
basicEmotionObserver.startObserving(qiContext);
And stop it in the onRobotFocusLost callback:
// Stop the basic emotion observation.
basicEmotionObserver?.stopObserving()
// Stop the basic emotion observation.
basicEmotionObserver.stopObserving();
The sources for this tutorial are available on GitHub.
| Step | Action |
|---|---|
Install and run the application. For further details, see: Running an application. |
|
| Choose “Detect human emotions”. | |
Stay in front of Pepper, try to express an emotion with your smile, your voice or by touching Pepper’s sensors. 
|
You should see some log information displaying your emotion when it changes.