Object tracker

An object tracker will follow the detected objects during the intermediate frames between model executions and match existing objects to new detections. This reduces the possibility of duplication of recognitions for the same object in different frames, and can provide a smoother experience by updating an object's location more frequently than a model can execute.

To use an object tracker, call its onFrame() method on each new frame and trackResults() method on each model recognition result. The parameters for each method vary based on the tracker used, but the function of each remains the same. Detections from the tracker can be received by calling the getTrackedObjects() method.

onFrame() method processes each new frame and updates the location of any currently tracked objects. trackResults() method adds new objects to the tracker and finds a correlation between the new object and existing tracked objects to determine if they are the same objects.

Based on the tracked feature type, there are two kinds of trackers:

• BasicObjectTracker - Best suited when tracking the features' presence / existence is of greater importance than their instantaneous changes in position. Recommended for small and static objects (ex. traffic signs, cones).

• MultiBoxObjectTracker - Best suited when features' instantaneous position is of greater importance as that is required for alerting a user of a potential collision. Recommended for constantly moving objects (ex. cars, rider, bicycle, motorcycle).

// OpenCV is used for image transformations in sample, but OpenCV is not required for trackers.
using namespace cv;   // Mat, cvtColor, resize, split
using namespace ls;   // Recognition, MultiBoxObjectTracker, BasicObjectTracker, TrackedRecognition

// Keeps track if model is running in background.
std::atomic<bool> isDetectionRunning(false);

// Tracker instances
MultiBoxObjectTracker objectTracker
BasicObjectTracker basicTracker

// Timestamp value used by trackers
long timestamp = 1;

/**
 * Classes to use with MultiBoxObjectTracker
 */
std::set<std::string> multiBoxTrackerClasses{
    "rider",
    "car",
    "bus",
    "truck",
    "bicycle",
    "motorcycle",
    "car-brake-light-on",
};

// Vector of models to run in background
std::vector<LivesenseModel *> livesenseModels;

void processFrameWithTracker(Mat frame) {

    // Extracting the y component from color image
    Mat frameLuminance;
    cvtColor(frame, frameLuminance, cv::COLOR_BGR2YUV);
    Mat yuv[3];                  //destination array
    split(frameLuminance, yuv);  //split source

    // Passing the y luminance frame to tracker for processing
    objectTracker.onFrame(
        frame_width,
        frame_height,
        frame_width,
        0,
        yuv[0].data,
        timestamp);
    // Basic object tracker does not use frame data so only takes the timestamp
    basicTracker.onFrame(timestamp);

    if (isDetectionRunning == false) {
        isDetectionRunning = true;
        // Convert colorspace as model use RGB images
        Mat frameRGB;
        cvtColor(frame, frameRGB, COLOR_BGR2RGB);
        // Run detection in background
        std::thread th(runDetectionInBackground, livesenseModels, &objectTracker,
                       &basicTracker, frameRGB, yuv[0].data, timestamp);
        th.detach();
    }

    // Get vector of tracked objects from the object trackers
    auto trackedObjects = objectTracker.getTrackedObjects();
    auto basicTrackedObject = basicTracker.getTrackedObjects()

    // Increment timestamp
    timestamp = timestamp + 1
}

The model inference is executed in the background with the model results processed by the trackers as shown below.

void runDetectionInBackground(const vector<ls::LivesenseModel *> models,
                              ls::MultiBoxObjectTracker *objectTracker,
                              ls::BasicObjectTracker *basicTracker,
                              const cv::Mat frame,
                              const uint8_t frameLuminance[],
                              const long timestamp) {
    using namespace ls;
    isDetectionRunning = true;

    // Resize the input RGB frame
    unsigned char *resizedInput = NULL;
    unsigned char *origInput = &frame.data[0];
    int origWidth = frame.cols;
    int origHeight = frame.rows;

    // OpenCV resize
    cv::Mat resizedFrame;


    std::vector<Recognition> trackMultiBox;
    std::vector<Recognition> trackBasic;
    for (auto livesenseModel : models) {
        // Each model may have its own input shape. For code to assist in
        // managing the different input shapes, see the example applications
        // provided alongside the SDK.
        int resizedInputWidth = livesenseModel->getInputWidth();
        int resizedInputHeight = livesenseModel->getInputHeight();
        // OpenCV resize
        cv::resize(frame, resizedFrame, cv::Size(resizedInputWidth, resizedInputHeight));
        resizedInput = &resizedFrame.data[0];
        // Get recognitions from each individual model in sequence
        auto recognitions = livesenseModel->recognizeImage(resizedInput,
                                                           resizedInputWidth,
                                                           resizedInputHeight,
                                                           origWidth,
                                                           origHeight,
                                                           origInput);
        // Sort recognitions for each tracker
        for (auto r : recognitions) {
            if (multiBoxTrackerClasses.find(r.getTitle()) != multiBoxTrackerClasses.end()) {
                trackMultiBox.push_back(r);
            } else {
                trackBasic.push_back(r);
            }
        }
    }
    // Pass the combined model recognitions with luminance frame and timestamp to tracker
    objectTracker->trackResults(trackMultiBox, frameLuminance, timestamp);
    basicTracker->trackResults(trackBasic, timestamp);
    // Signal detection finished
    isDetectionRunning = false;
}