I need help removing duplicate items from the feed and mapping the first object. For that I am using xsl:for-each-group to group the object by id and using it.
It works fine with small XML But when I use a large feed that contains 2 Million items, it does not perform xsl transform, the file is empty.
Is there any other way I can achieve the same without using the grouping?
I am using saxon ee 10.5 version
xsl template
<?xml version="1.0"?>
<xsl:stylesheet
xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="3.0">
<xsl:output method="text" encoding="UTF-8" indent="no" omit-xml-declaration="yes"/>
<xsl:mode streamable="yes"/>
<xsl:strip-space elements="*"/>
<xsl:variable name="dq" select="'"'"/>
<xsl:variable name="qcq" select="'","'"/>
<xsl:variable name="lf" select="'
'"/>
<!-- Static defaults -->
<xsl:variable name="data_source" select="'HHTestMedia'"/>
<xsl:template match="items">
<xsl:text>title,city,state,postal_code,country,company_name</xsl:text>
<xsl:value-of select="'
'"/>
<!--<xsl:for-each select="job ! copy-of(.)">-->
<xsl:for-each-group select="item ! copy-of(.)" group-by="id">
<xsl:variable name="currentGroup" select="current-group()[1]"/>
<xsl:variable name="p_title" select="substring($currentGroup/replace(title,'"',''),0,128)"/>
<xsl:variable name="p_city" select="substring-before($currentGroup/location,',')"/>
<xsl:variable name="p_state_code" select="substring(normalize-space(substring-after($currentGroup/location,',')),1,2)"/>
<xsl:variable name="p_postal_code" select="$currentGroup/postcode"/>
<xsl:variable name="p_country_code" select="$currentGroup/country"/>
<xsl:variable name="p_company_name" select="replace(substring($currentGroup/company,1,64),'"','')"/>
<!-- The following line should not need to be updated if you copied this XSL from a different setup. -->
<xsl:value-of disable-output-escaping="yes" select="concat($dq,$p_title,$qcq,$p_city,$qcq,$p_state_code,$qcq,$p_postal_code,$qcq,$p_country_code,$qcq,$p_company_name,$dq,$lf)"/>
<!--</xsl:for-each>-->
</xsl:for-each-group>
</xsl:template>
</xsl:stylesheet>
sample XML
<?xml version="1.0" encoding="UTF-8" ?>
<source>
<items>
<item>
<id><![CDATA[160449417]]></id>
<title><![CDATA[Campaign1]]></title>
<location><![CDATA[Hudson, FL 34667]]></location>
<postcode><![CDATA[34667]]></postcode>
<country><![CDATA[US]]></country>
<company><![CDATA[Halloween]]></company>
</item>
<item>
<id><![CDATA[160449417]]></id>
<title><![CDATA[Campaign1]]></title>
<location><![CDATA[Hudson, FL 34667]]></location>
<postcode><![CDATA[34667]]></postcode>
<country><![CDATA[US]]></country>
<company><![CDATA[Halloween]]></company>
</item>
</items>
</source>
Thanks in advance.