macOS uses a rounded rectangle filled with NSColor.selectedContentBackgroundColor to indicate selection. I’ll imitate that but using a custom red color.

The variable isEmphasized becomes false when the control is not key or the application is on the background, so I’ll use it to trigger a refresh.

import AppKit

final class CustomTableRowView: NSTableRowView {
    override func layout() {
        super.layout()
        // the default .none won’t even call drawSelection(in:)
        selectionHighlightStyle = .regular
    }

    override func drawSelection(in dirtyRect: NSRect) {
        guard selectionHighlightStyle != .none else { 
            return 
        }
        let selectionRect = NSInsetRect(bounds, 10, 0)
        let fillColor = isEmphasized ? NSColor.red : NSColor.unemphasizedSelectedContentBackgroundColor
        fillColor.setFill()
        let selectionPath = NSBezierPath.init(roundedRect: selectionRect, xRadius: 5, yRadius: 5)
        selectionPath.fill()
    }

    override var isEmphasized: Bool {
        didSet {
            needsDisplay = true
        }
    }
}

Finally, I tell the NSViewController to use the custom class above:

// MARK: - NSOutlineViewDelegate

func outlineView(_ outlineView: NSOutlineView, rowViewForItem item: Any) -> NSTableRowView? {
    CustomTableRowView()
}