One of the DB load job in Azure data factory failed with the message error:
Sql error number: 1921. Error Message: Invalid partition scheme 'ps_MonthlyEWSOverallUsage_DmsLoadDate' specified.
==== Executing prc_ADFCopyData_MoveData
Call prc_ADFCopyData_DeletePartition
==== Executing prc_ADFCopyData_DeletePartition
prc_ADFCopyData_GetPartitionColumn returned LoadDate
declare @DeltaInterval smalldatetime = 'May 1 2024 12:00AM'
if exists (select top 1 1 from [dbo].[MonthlyEWSOverallUsage] (nolock) where [LoadDate] = @DeltaInterval)
begin
set @hasData = 1
end
select @hasData
returned 1
Call prc_ADFCopyData_CreateStageTable on MonthlyEWSOverallUsage_DELETE_20240501
And we have checked the SQL query statement of it:
begin
declare @partitionColumn sysname
print N'==== Executing prc_ADFCopyData_DeletePartition'
exec prc_ADFCopyData_GetPartitionColumn @TableName = @TableName, @PartitionColumn = @partitionColumn out
if @partitionColumn is null
throw 151000, 'Partition column LoadDate is not present or not a supported data type.', 1
print 'prc_ADFCopyData_GetPartitionColumn returned ' + @partitionColumn
--Determine if the current partition has data
declare @hasData bit = 0
declare @deltaStr nvarchar(20) = convert(varchar(20), @DeltaInterval)
declare @cmd nvarchar(max) = ''
set @cmd = '
declare @DeltaInterval smalldatetime = ''' + @deltaStr + '''
if exists (select top 1 1 from [dbo].' + quotename(@TableName) + ' (nolock) where [' + @partitionColumn + '] = @DeltaInterval)
begin
set @hasData = 1
end
select @hasData
'
exec sp_executesql @cmd, N'@hasData bit output', @hasData output
print @cmd
print N'returned ' + Cast(@hasData AS nvarchar(3))
--No need to empty partition, it has no data
if @hasData = 1
begin
declare @stageTableName sysname = dbo.fn_ADFCopyData_GetTempTableName(@TableName, @PartitionType, @DeltaInterval, 'DELETE')
print N'Call prc_ADFCopyData_CreateStageTable on ' + @stageTableName
exec prc_ADFCopyData_CreateStageTable @TableName = @TableName, @FileGroupName = '[PRIMARY]', @DeltaInterval = @DeltaInterval, @PartitionType = @PartitionType, @PartitionSchemeName = @PartitionSchemeName, @PartitionColumnName = @partitionColumn, @TableNameSuffix = 'DELETE'
print N'Call prc_ADFCopyData_SwitchPartitions '
exec prc_ADFCopyData_SwitchPartitions @TableName = @TableName, @PartitionType = @PartitionType, @DeltaInterval = @DeltaInterval, @TableSuffix = 'DELETE', @IsReverse = 1
if object_id(quotename(@stageTableName)) is not null
begin
set @cmd = ''
set @cmd = concat('drop table [dbo].', quotename(@stageTableName))
print @cmd
exec (@cmd)
end
end
end
We haven’t done any changes to this CreateStageTable job. And even we got these errors, but we still see the monthly data in the databases.
Anyone knows why it throw such exception?