I am trying to add regression lines with a polynomial regression model to my xyplot which has multiple y-values for each x-value which is creating multiple ablines (I think).

amp.plot.1 <- structure(list(TP = c(-2, -1, 0, 1, 2, 4, 6, 8, 9, -2, -1, 0, 
1, 2, 5, 7, 8, 9, 4, 6), SID_2 = c("UAB003_1", "UAB003_1", "UAB003_1", 
"UAB003_1", "UAB003_1", "UAB003_1", "UAB003_1", "UAB003_1", "UAB003_1", 
"UAB003_2", "UAB003_2", "UAB003_2", "UAB003_2", "UAB003_2", "UAB003_2", 
"UAB003_2", "UAB003_2", "UAB003_2", "UAB005", "UAB005"), UID = c("UAB003_W1D3", 
"UAB003_W1D6", "UAB003_W2D1", "UAB003_W2D2", "UAB003_W2D3", "UAB003_W2D5", 
"UAB003_W2D7", "UAB003_W3D2", "UAB003_W3D3", "UAB003_W8D3", "UAB003_W8D5", 
"UAB003_W9D3", "UAB003_W9D4", "UAB003_W9D5", "UAB003_W10D1", 
"UAB003_W10D3", "UAB003_W10D6", "UAB003_W10D7", "UAB005_W2D4", 
"UAB005_W2D6"), Species = structure(c(1L, 1L, 1L, 1L, 1L, 1L, 
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L), levels = c("Lactobacillus_iners", 
"Lactobacillus_crispatus", "Lactobacillus_jensenii", "Lactobacillus_gasseri", 
"Gardnerella_vaginalis", "Sneathia_sanguinegens", "Atopobium_vaginae", 
"BVAB1", "Prevotella_timonensis", "Prevotella_amnii", "Prevotella_bivia", 
"g_Megasphaera"), class = "factor"), Count = c(5.54644502967252, 
4.88834003849887, 4.31944507354567, 5.47828041152078, 5.59699483640575, 
5.66255777037516, 5.80752511653788, 5.67881819020191, 5.59099252829171, 
3.98942397054398, 4.07929361555486, 4.23489117686723, 3.50636505344502, 
4.47583162261933, 5.23018702562597, 5.14641307286776, 5.18195301770781, 
3.72092821777661, 5.76233696844788, 5.90702924988876)), row.names = c("1", 
"2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", 
"14", "15", "16", "17", "18", "19", "20"), class = "data.frame")

xyplot(amp.plot.1$Count ~ amp.plot.1$TP | amp.plot.1$Species, 
            aspect = 1:1,
            groups = amp.plot.1$SID_2,
            panel = function(x, y, ...) {
              panel.xyplot(x, y, ...)
              fm = lm(y ~ poly(x, 3))
              panel.lines(x, fitted(fm), col.line = "red")
              panel.text(3, 1.5, labels = sprintf("y = %.3f x³ + %.3f x² + %.3f x + %.3fnR² = %.2f", 
                                                  coef(fm)[4], coef(fm)[3], coef(fm)[2], coef(fm)[1], summary(fm)$r.squared), 
                                                  cex = 0.7)},
            scales = "free",
            pch = 1, 
            cex = .5,
            xlim = c(-3, 9),
            ylim = c(0.5, NA),
            ylab = list(label = "log10[estimated absolute abundance + 1]", fontsize = 15),
            xlab = list(label = "Day of MET Treatment", fontsize = 15),
            par.strip.text = list(cex = 0.8),
            layout = c(4, 3)
) 

When I make the same plot with a linear regression, it works perfectly. And when I do the equivalent with ggplot, the regression line is correct.

Can someone please help me figure out the correct way to fix the regression line in the xyplot so there is only one per facet?
Thank you!