I'm trying to use cv::warpAffine to perform image transformations, but I have an issue.

Note: I already saw questions/46998895 and it works well too, but with no interpolation and I wanted to use affine matrixes.

So I want to shear the image and the result goes in a bigger image to have the entire sheared image like this:

Right now, my idea was to perform the shear on inverted image if the values are negatives rather than calculate the new image size with negative values etc as seen on paperspace blog.

When we use a negative shear, the direction of shear is right to left, while x2 is not further in the negative direction than x1. One way to solve this could be to get the other set of corners (that would satisfy the constraint, can you prove it?). Apply the shear transformation and then change to the other set of corners because of the notation we follow. Well, we could do that, but there's a better method. Here's how how perform a negative shear with shearing factor -alpha.

1. Flip the image and boxes horizontally.
2. Apply the positive shear transformation with shearing factor alpha
3. Flip the image and boxes horizontally again.

The left ones are my results, and the right ones are the expected results :

So as you can see it works well when x and y are both positive values or when one is 0 and the other is negative, its ok. When one is positive and the other is negative, or when they are both negative, it provides always the same result for my shear function, see the 4 last lines in the pic above. Again, the left forms are my results, the right ones are the mickaShear() results (and obviously the correct ones).

My question is: what do I have to do in my shear_matrix to perform the correct transformation with negative values? Is the cv::flip() method described by paperSpace the right way to achieve this transformation?

Here you can find the reproductible code with the comparison between my function and the function in this answer. You will need OpenCV. I work on OpenCV 4.5.1

#include <opencv2/core/mat.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/xfeatures2d.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/core.hpp>
#include <iostream>

cv::Mat myShear(const cv::Mat & src, float sx, float sy) {
    cv::Mat tmp;
    cv::Mat dst;

    std::vector<cv::Point2f> extremePoints;
    extremePoints.emplace_back(cv::Point2f(0, 0));
    extremePoints.emplace_back(cv::Point2f((float)src.cols, 0));
    extremePoints.emplace_back(cv::Point2f((float)src.cols, (float)src.rows));
    extremePoints.emplace_back(cv::Point2f(0, (float)src.rows));
    for(auto & pt : extremePoints){
        pt = cv::Point2f(pt.x + pt.y * abs(sx), pt.y + pt.x * abs(sy));
    }
    cv::Rect offsets = cv::boundingRect(extremePoints);
    cv::Size new_size = offsets.size();
    float mat_values[] = {1.0f, abs(sx), 0.0f, abs(sy), 1.0f, 0.0f};
    cv::Mat shear_matrix = cv::Mat(2, 3, CV_32F, mat_values);
    dst = cv::Mat::zeros(new_size, src.type());

    /*
     *cv::flip(img, tmp, INT_FLIP_CODE) where INT_FLIP_CODE can be: 
     * 0   (Vertically) 
     * 1   (Horizontally)
     * -1  (Both)
     */
    if(sx < 0.0f and sy < 0.0f) {
        cv::flip(img, tmp, -1);
        cv::warpAffine(tmp, dst, shear_matrix, new_size, cv::INTER_LINEAR);
        cv::flip(dst, dst, -1);
    } else if(sx < 0.0f) {
        cv::flip(img, tmp, 1);
        cv::warpAffine(tmp, dst, shear_matrix, new_size, cv::INTER_LINEAR);
        cv::flip(dst, dst, 1);
    } else if(sy < 0.0f) {
        cv::flip(img, tmp, 0);
        cv::warpAffine(tmp, dst, shear_matrix, new_size, cv::INTER_LINEAR);
        cv::flip(dst, dst, 0);
    } else {
        tmp = src.clone();
        cv::warpAffine(tmp, dst, shear_matrix, new_size, cv::INTER_LINEAR);
    }
    return dst;
}

cv::Mat mickaShear(const cv::Mat & input, float Bx, float By)
{
    if (Bx*By == 1)
    {
        std::cerr << "error == 1" << std::endl;
    }
    if (input.type() != CV_8UC3) return {};

    std::vector<cv::Point2f> extremePoints;
    extremePoints.emplace_back(0, 0);
    extremePoints.emplace_back(input.cols, 0);
    extremePoints.emplace_back(input.cols, input.rows);
    extremePoints.emplace_back(0, input.rows);
    for (auto & pt : extremePoints)
    {
        pt = cv::Point2f(pt.x + pt.y*Bx, pt.y + pt.x*By);
    }
    cv::Rect offsets = cv::boundingRect(extremePoints);
    cv::Point2f offset = -offsets.tl();
    cv::Size resultSize = offsets.size();
    cv::Mat shearedImage = cv::Mat::zeros(resultSize, input.type());

    for (int j = 0; j < shearedImage.rows; ++j){
        for (int i = 0; i < shearedImage.cols; ++i){
            cv::Point2f pp((float)i, (float)j);
            pp = pp - offset; // go back to original coordinate system
            cv::Point2f p;
            p.x = int((-pp.y*Bx + pp.x) / (1 - By*Bx));
            p.y = int(pp.y - p.x*By);
            if ((p.x >= 0 && p.x < (float)input.cols) && (p.y >= 0 && p.y < (float)input.rows)){
                shearedImage.at<cv::Vec3b>(j, i) = input.at<cv::Vec3b>(p);
            }
        }
    }
    return shearedImage;
}

int main(int argc, char *argv[]){
    float x = -0.2f; //CHANGE SIGN TO TEST
    float y = 0.5f; //CHANGE SIGN TO TEST

    cv::Mat im = cv::imread("MODIFY BY JPG FILE PATH");
    cv::Mat result = im.clone();
    cv::Mat output = cv::Mat();
    for(auto & aug: augments){
        output = mickaShear(im, x, y);
        result = myShear(im);
    }
    cv::imshow("result", result);
    cv::imshow("output", output);
    auto k = cv::waitKey(0);
    if(k == (int)'q'){
        cv::destroyAllWindows();
        break;
    }
    cv::destroyAllWindows();
    return 0;
}