I am trying to detect if an emoji is happy or sad using OpenCV and C++ based on the mouth contour. For now all i get is “Indeterminate” and i am not sure what to do and how to approach this problem. How can i find a way to select which contour is the mouth?

string classifyEmojiByMouth(const vector<Point>& contour) {
    if (contour.size() < 5) {
        return "Indeterminate";
    }
    RotatedRect ellipse = fitEllipse(contour);
    float aspectRatio = (float)ellipse.size.width / ellipse.size.height;
    bool isHappy = (ellipse.angle > 90.0 && ellipse.angle < 270.0);

    if (isHappy && aspectRatio > 1.2) {
        return "Happy";
    }
    else if (!isHappy && aspectRatio > 1.2) {
        return "Sad";
    }
    return "Indeterminate";
}

void mainProject() {
    char fname[MAX_PATH];
    while (openFileDlg(fname)) { 
        Mat image = imread(fname, IMREAD_COLOR); 
        if (image.empty()) {
            cout << "Could not open or find the image, try again." << endl;
            continue; 
        }

        // Convert to grayscale and threshold
        Mat gray;
        cvtColor(image, gray, COLOR_BGR2GRAY);
        threshold(gray, gray, 100, 255, THRESH_BINARY);

        // Find contours
        vector<vector<Point>> contours;
        findContours(gray, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
        imshow("contur", image);


        // assume contours[0] is the mouth contour, if it exists
        if (contours.empty()) {
            cout << "No contours found, try a different image." << endl;
            continue;
        }

        string result = classifyEmojiByMouth(contours[0]);
        cout << "The emoji is: " << result << endl;
        imshow("Processed Image", image);
        waitKey(0); 
    }

}