If you look closely at the gif image, when the mouse moves to the greenish container, it actually leaves the internal picBoxExtended, not the CtrlNoticePictureBox instance. The covered area of the main PictureBox – by the modeless Form – does not receive the mouse inputs unless that Form and its children are transparent to the mouse inputs. As a result, the MouseLeave event does not fire and the zoom Form remains. You also need to implement the MouseLeave event of the internal PictureBox to hide the Form.

Here’s a working example with some additions and modifications.

<DesignerCategory("Code")>
Public Class CtrlNoticePictureBox
    Inherits PictureBox

    Public Enum StartPosition
        Relative
        Center
        Cursor
    End Enum

    Private FrmZoom As Form
    Private PicZoom As PictureBox

    <DefaultValue(StartPosition.Center)>
    Public Property ZoomFormStartPosition As StartPosition = StartPosition.Center

    <DefaultValue(True)>
    Public Property KeepZoomFormWithinScreen As Boolean = True

    Protected Overrides Sub OnMouseEnter(e As EventArgs)
        MyBase.OnMouseEnter(e)
        If Image Is Nothing Then Return
        If FrmZoom Is Nothing Then CreateZoomForm()
        If FrmZoom.Visible Then Return
        Dim loc As Point
        Select Case ZoomFormStartPosition
            Case StartPosition.Relative
                loc = PointToScreen(Point.Empty)
            Case StartPosition.Center
                loc = PointToScreen(Point.Empty)
                loc.X += (Width - FrmZoom.Width)  2
                loc.Y += (Height - FrmZoom.Height)  2
            Case Else
                loc = Cursor.Position
        End Select
        If KeepZoomFormWithinScreen Then
            Dim sc = Screen.FromControl(Me).Bounds ' or .WorkingArea
            If loc.X + FrmZoom.Width > sc.Width Then
                loc.X = sc.Right - FrmZoom.Width
            End If
            If loc.Y + FrmZoom.Height > sc.Height Then
                loc.Y = sc.Bottom - FrmZoom.Height
            End If
            loc.X = Math.Max(0, loc.X)
            loc.Y = Math.Max(0, loc.Y)
        End If
        FrmZoom.Location = loc
        PicZoom.Image = Image
        FrmZoom.Show()
    End Sub

    Protected Overrides Sub OnMouseLeave(e As EventArgs)
        MyBase.OnMouseLeave(e)
        If Not ClientRectangle.Contains(PointToClient(Cursor.Position)) Then
            FrmZoom.Hide()
        End If
    End Sub

    Protected Overrides Sub Dispose(disposing As Boolean)
        If disposing Then
            FrmZoom?.Dispose()
        End If
        MyBase.Dispose(disposing)
    End Sub

    Private Sub CreateZoomForm()
        PicZoom = New PictureBox With {
            .SizeMode = PictureBoxSizeMode.Zoom,
            .Dock = DockStyle.Fill,
        }
        FrmZoom = New Form With {
            .FormBorderStyle = FormBorderStyle.None,
            .ControlBox = False,
            .ShowInTaskbar = False,
            .TopMost = True,
            .StartPosition = FormStartPosition.Manual,
            .Size = New Size(Width * 2, Height * 2)
        }
        FrmZoom.Controls.Add(PicZoom)
        AddHandler FrmZoom.Disposed, Sub(s, e) PicZoom.Dispose()
        AddHandler PicZoom.MouseLeave, Sub(s, e) OnMouseLeave(EventArgs.Empty)
    End Sub
End Class

Notes

• In a derived class, override the methods of the base class and do not subscribe to its events. Other controls can subscribe to events if needed.
• No need to keep creating and destroying the zoom Form. Create it on first access and override the owner’s Dispose method to dispose of it.