I try to create multiple nested axes inside a figure:

import matplotlib.pyplot as plt
import matplotlib.patches as pat
import copy


# set figure
fig = plt.figure(figsize=(10, 10))
ax = fig.gca()
ax.set_aspect("equal")
# comment to show axis
ax.axis('off')
#
offset = 0
limites = (-offset, 2 + offset)
ax.set_xlim(limites)
ax.set_ylim(limites)
# draw a square patch (dimension = 2)
carre = pat.Rectangle((0, 0), 2, 2, ec="red")
# 4 nested constructions
N = 4
# axes list
liste_axes = [ax] + [None] * (N - 1)
# squares list
liste_grilles = [carre]
for i in range(1, N):
    liste_grilles.append(copy.deepcopy(carre))
#
for i in range(1, N):
    # transformation from data to figure
    # to create new axes object
    t = liste_axes[i - 1].transData + fig.transFigure.inverted()
    # from (0, 0) to (1, 1) -> a quarter (bottom left) of initial dimension
    b = t.transform([(0, 0), (1, 1)]).flatten()
    xy = b[:2]
    wi, he = b[2] - b[0], b[3] - b[1]
    liste_axes[i] = fig.add_axes([*xy, wi, he])
    # setup of the new axes 
    liste_axes[i].set_aspect("equal")
    liste_axes[i].set_xlim(limites)
    liste_axes[i].set_ylim(limites)
    # comment to show axis
    liste_axes[i].axis('off')

# draw all patches
for h, axe in zip(liste_grilles, liste_axes):
    axe.add_patch(h)

plt.savefig("test.png", dpi=200, bbox_inches='tight')

But the test.png image is not correct: bad left alignment of nested axes as one can see on the image below. I tried to use anchor option in add_axes method but I can't get a correct alignment with it either.

How to have these axes alignment correct?