I want to create good Images of cars. I have empty garage background image which I want to use for every car.

My users would upload car images with random background. I want to remove that background and put the car in the background I have with proper shadows and reflections to make it look realistic.

I was able to achieve good results when backgrounds are generated from prompt. But how do I use a fixed background and make the final image look seamless?

Here is my current code

def make_inpaint_condition(init_image, mask_image):
    init_image = np.array(init_image.convert("RGB")).astype(np.float32) / 255.0
    mask_image = np.array(mask_image.convert("L")).astype(np.float32) / 255.0
    assert init_image.shape[0:1] == mask_image.shape[0:1], "image and image_mask must have the same image size"
    init_image[mask_image > 0.5] = -1.0  # set as masked pixel
    init_image = np.expand_dims(init_image, 0).transpose(0, 3, 1, 2)
    init_image = torch.from_numpy(init_image)
    return init_image

def generate_with_controlnet():
    controlnet = ControlNetModel.from_pretrained("lllyasviel/control_v11p_sd15_inpaint", torch_dtype=torch.float16, use_safetensors=True)
    pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained(
        "runwayml/stable-diffusion-inpainting", controlnet=controlnet, torch_dtype=torch.float16, variant="fp16"
    )

    init_image = load_image("car-new-image.png")
    mask_image = load_image("car_mask_filled2.png")
    bg_image = load_image("car_bg.png")

    control_image = make_inpaint_condition(init_image, mask_image)

    prompt="A car’s garage with metallic garage door, soft light, minimalistic, High Definition"

    output = pipe(
        prompt="",
        num_inference_steps=50,
        guidance_scale=7.5,
        eta=0.8,
        image=init_image,
        mask_image=mask_image,
        control_image=control_image,
    ).images[0]
    output.save("output_controlnet.jpg")