good morning
I need help related to a process whose popular purpose is data from one table to another, at the moment I’m using spring boot with native query, but it’s taking more than 30 minutes, java method:
@Modifying
@Query(nativeQuery = true, value = "WITH transformed_source AS ( "
+ " SELECT "
+ " lcp.client, "
+ " lcp.product_id, "
+ " lcp.store_id, "
+ " CASE "
+ " WHEN lcp."source" = :client THEN 'PROPRIA' "
+ " WHEN lcp."source" = 'INFOPRICE - INFOPANEL' THEN 'INFOPANEL' "
+ " WHEN lcp."source" = 'INFOPRICE - MPDV' THEN 'MPDV' "
+ " ELSE lcp."source" "
+ " END AS source "
+ " FROM "
+ " last_competitor_prices lcp "
+ " WHERE "
+ " lcp.client = :client "
+ " AND lcp."source" IS NOT NULL "
+ " AND lcp.date BETWEEN :startingDate AND :endingDate "
+ "), "
+ " agg_source AS ( "
+ " SELECT "
+ " product_id, "
+ " store_id, "
+ " string_agg(DISTINCT "source", '-' ORDER BY "source") AS "source" "
+ " FROM "
+ " transformed_source "
+ " GROUP BY "
+ " product_id, "
+ " store_id "
+ " ), "
+ " agg_competitor_prices AS ( "
+ " SELECT "
+ " lcp.store_id, "
+ " lcp.product_id, "
+ " CAST(AVG(lcp.price_retail) OVER (PARTITION BY lcp.store_id, lcp.product_id) AS DECIMAL(17, 2)) AS avg_competitor_price_product, "
+ " CAST(MIN(lcp.price_retail) OVER (PARTITION BY lcp.store_id, lcp.product_id) AS DECIMAL(17, 2)) AS min_competitor_price_product, "
+ " CAST(MAX(lcp.price_retail) OVER (PARTITION BY lcp.store_id, lcp.product_id) AS DECIMAL(17, 2)) AS max_competitor_price_product, "
+ " CAST(AVG(lcp.price_retail) OVER (PARTITION BY lcp.store_id, lcp.product_family_id, lcp.competitor_store_id) AS DECIMAL(17, 2)) AS avg_competitor_price_family, "
+ " CAST(MIN(lcp.price_retail) OVER (PARTITION BY lcp.store_id, lcp.product_family_id, lcp.competitor_store_id) AS DECIMAL(17, 2)) AS min_competitor_price_family, "
+ " CAST(MAX(lcp.price_retail) OVER (PARTITION BY lcp.store_id, lcp.product_family_id, lcp.competitor_store_id) AS DECIMAL(17, 2)) AS max_competitor_price_family, "
+ " MAX(lcp.date) OVER (PARTITION BY lcp.client, lcp.store_id, lcp.product_id) AS max_date_competitor_price_product, "
+ " ags."source" "
+ " FROM "
+ " last_competitor_prices lcp "
+ " LEFT JOIN "
+ " agg_source ags ON lcp.product_id = ags.product_id AND lcp.store_id = ags.store_id "
+ " WHERE "
+ " lcp.client = :client "
+ " AND lcp.date BETWEEN :startingDate AND :endingDate "
+ " ), "
+ " final_competitor_prices AS ( "
+ " SELECT "
+ " store_id, "
+ " product_id, "
+ " avg_competitor_price_product, "
+ " min_competitor_price_product, "
+ " max_competitor_price_product, "
+ " max_date_competitor_price_product, "
+ " source, "
+ " AVG(avg_competitor_price_family) AS avg_competitor_price_family, "
+ " AVG(min_competitor_price_family) AS min_competitor_price_family, "
+ " AVG(max_competitor_price_family) AS max_competitor_price_family "
+ " FROM "
+ " agg_competitor_prices "
+ " GROUP BY "
+ " store_id, "
+ " product_id, "
+ " avg_competitor_price_product, "
+ " min_competitor_price_product, "
+ " max_competitor_price_product, "
+ " max_date_competitor_price_product, "
+ " source "
+ " ) "
+ "UPDATE "
+ " products_to_be_priced ptb "
+ "SET "
+ " acp_avg_competitor_price_product = fcp.avg_competitor_price_product, "
+ " acp_min_competitor_price_product = fcp.min_competitor_price_product, "
+ " acp_max_competitor_price_product = fcp.max_competitor_price_product, "
+ " acp_avg_competitor_price_family = fcp.avg_competitor_price_family, "
+ " acp_min_competitor_price_family = fcp.min_competitor_price_family, "
+ " acp_max_competitor_price_family = fcp.max_competitor_price_family, "
+ " acp_max_date_competitor_price_product = fcp.max_date_competitor_price_product, "
+ " acp_days_competitor_price = :competitorDays, "
+ " source = fcp.source "
+ "FROM "
+ " final_competitor_prices fcp "
+ "WHERE "
+ " ptb.client = :client "
+ " AND ptb.product_id = fcp.product_id "
+ " AND ptb.store_id = fcp.store_id")
void populateProductToBePricedByLastCompetitorPrice(@Param("client") String client, @Param("startingDate") LocalDate startingDate,
@Param("endingDate") LocalDate endingDate, @Param("competitorDays") int competitorDays);
eu ja criei alguns indices na tabelas last_competitor_prices:
create index if not exists idx_lcp_date_client_product_store
on public.last_competitor_prices (date, client, product_id, store_id);
create index if not exists last_competitor_prices_client_store_id_product_id_idx
on public.last_competitor_prices (client, store_id, product_id);
create index if not exists idx_lcp_client_store_product_family_date
on public.last_competitor_prices (client, store_id, product_family_id, date);
create index if not exists idx_lcp_client_date
on public.last_competitor_prices (client, date);
create index if not exists idx_lcp_product_store
on public.last_competitor_prices (product_id, store_id);
create index if not exists idx_lcp_product_family
on public.last_competitor_prices (product_family_id);
create index if not exists idx_lcp_competitor_store
on public.last_competitor_prices (competitor_store_id);
My question is, is there any way to improve either the query or the way it was done, to make this processing faster?
It could be any change, even perhaps migrating to spring batch, I would like some guidance if possible.
Thanks