Table Tennis Robot

Computer Vision, Algorithm, Arduino


  • Date : 2022.06 - 2022.08
  • Goal : Hitting a flying ping-pong ball with robot arm.
  • Reference : GIST IIT
  • Awards : GIST 6th 창의융합경진대회 Preliminaries 1st, Finals 4th.
  • Help : GIST MSE of Yuna Ji, GIST EECS of Jeonghoon Sung, Yuseung Lee.

Ball Tracking

  • Target : Tracking color of ping-pong ball by using OpenCV
  • Camera : Intel Realsence d435
  • Language : I use c++ because of computation speed and compatibility with linear actuator.
Library Setting
#include <opencv2/opencv.hpp>
#include <opencv2/imgproc.hpp>
using namespace cv;
Convert Video to Image
VideoCapture cap(0); // Camera object
Mat img; // Image object
while (1)
{
    cap >> img; // allocate captured video to image
    imshow("", img); // showing image
    if (waitKey(1) == 'q') 
        break;
}
Finding Contours of Ball
Scalar lower = Scalar(10, 126, 75); // lower boundary of orange ball color in HSV unit
Scalar upper = Scalar(25, 255, 255); // upper boundary of orange ball color in HSV unit
vector<vector<Point>> contours;
vector<Vec4i> hierarchy;

Mat hsvImg;
cvtColor(img, hsvImg, COLOR_BGR2HSV); // RGB to HSV
inRange(hsvImg, lower, upper, hsvImg); //Convert HSV
findContours(hsvImg, contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE); // Finding Contours

imshow("",hsvImg); // Show only Orange Area



Finding Positions of ball
int X,Y;
Moments moment;

moment = moments(contours[0]); // Calculate center of moment
X = static_cast<float>(moment.m10 / moment.m00 + 1e-5); // allocate X
Y = static_cast<float>(moment.m01 / moment.m00 + 1e-5); // allocate Y

Path of Ball Expection

  • Target : Expect path of ball positions. Position data can get by above process.
  • Method : The method of least square.
  • Language : I use c++ because of computation speed and compatibility with linear actuator.
void LinearRegression(int* pX, int* pY, int arraySize)
{
    int i;
    float meanX=0, meanY=0;
    float constant = 0, slope = 0, div = 0;

    for (i = 0; i < arraySize; i++)
    {
        meanX += pX[i];
        meanY += pY[i];
    }
    
    meanX /= arraySize; // mean of x positions
    meanY /= arraySize; // mean of y positions


    
    for (i = 0; i < arraySize; i++)
    {
        div += (X[i] - meanX) * (X[i] - meanX); // variance
        slope += (X[i] - meanX) * (Y[i] - meanY); // variance
    }
    
    slope /= div;
    constant = meanY - meanX * slope;
}

Robot Arm

  • Target : Making robot arm with linear actuator and arduino
  • Hardware : Dynamixel X, OpenCR
  • Language : I use Arduino for Dynamixel, and I use C++ for linear actuator.
Dynamixel
#define NUM 00
#define BR 00

uint8_t moter = NUM;
uint16_t m = 0;
DynamixelWorkbench dxl;

void setup()
{
    Serial.begin(57600);
    const char *log;

    dxl.init("", BR, &log);  // initalization of port rate 
    dxl.ping(moter, &m, &log);  // initalization of moter ping
    dxl.jointMode(moter, 0, 0, &log); // setting moter mode
}

void initalizePosition()
{
  dxl.goalPosition(moter[0], (int32_t)2047);
  delay(50);
  dxl.goalPosition(moter[1], (int32_t)2371);
  delay(50);
  dxl.goalPosition(moter[2], (int32_t)2747);
  delay(50);
  dxl.goalPosition(moter[3], (int32_t)2047);
  delay(50);
  return;
}

void swing()
{
  dxl.goalPosition(moter[2], (int32_t)2047);
  dxl.goalPosition(moter[3], (int32_t)1647);
  delay(1000);
  initalizePosition();
  return; 
}
Initalized State



Swing State



Linear Actuator
#include "stddef.h"
#include "CLinear_actu.h"

int main ()
{
    CLinear_actu linear_ac = CLinear_actu();
    
    int r;
    
    while (1)
    {
        VideoCapture cap(0); 
        Mat img;
        vector<float> Xs,Ys;
        
        GetPosition(img,Xs,Ys); // from Convert Video to Image
        LinearRegression(int* pX, int* pY, int arraySize) // from Path of Ball Expection
        
        r = slope * -150 + constant;
        r /= 4;
        r = r-70;
        
        linear_ac.move(r);
        delay(r*r/10);
        linear_ac.move(-r);
    }
}



Connecting C++ with Arduino

C++ : Sender
#include <iostream>
#include <tchar.h>
#include "SerialClass.h"
#include <string>

int _tmain(int argc, _TCHAR* argv[])    
{
 
    Serial* SP = new Serial("\\\\.\\COM3");    
    int a;
    if (SP->IsConnected())
        cout<<"Connected"<<endl;
 
    char incomingData[256] = "";
 
    while (SP->IsConnected())
    {
        scanf_s("%d", &a);
 
        if (a == 1)
        {
            SP->WriteData("ON", 255);
        }
        if (a == 2)
        {
            SP->WriteData("OFF", 255);
        }
    }
    
    return 0;
}
Arduino : Receiver
void ON();
void OFF();

void setup() {
  Serial.begin(57600);
}

void loop() {
  if (Serial.available())
  {
    text = Serial.readStringUntil('\n');
    if (text.equals("ON"))
        ON();
    if (text.equals("OFF"))
        OFF();
  }
}



Limits

  • Slow performance of linear actuator. I struggled with that problem so we can’t improve performance.
  • Delays occurred due to Arduino’s computational speed limitations.
  • There were many hardware stability issues(broken stick, broken camera).