I am doing batch insertion in the below code, extracting data from a database and inserting to another source, the data is getting inserted completely except the last row. The 2nd last row is getting duplicated instead.
The error log shows the actual row count data too.
try {
// Connection to source database
$sourceDB = new PDO("mysql:host=$hostSource;dbname=$dbSource", $userSource, $passSource);
$sourceDB->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
// Connection to target database
$targetDB = new PDO("mysql:host=$hostDest;dbname=$dbDest", $userDest, $passDest);
$targetDB->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
// Step 1: Extract data from source database
$query = "SELECT * FROM chapter_contributor_info";
$sourceData = $sourceDB->query($query)->fetchAll(PDO::FETCH_ASSOC);
// Step 2: Transform data (lowercase email, etc.)
foreach ($sourceData as &$row) {
$row['email'] = strtolower($row['email']);
}
// Step 3: Clear existing data in the target table
$targetDB->exec("DELETE FROM chapter_contributor_info");
// Step 4: Insert data in batches
$batchSize = 50; // Adjust batch size as needed
$dataCount = count($sourceData);
$targetDB->beginTransaction(); // Start transaction for performance
$insertedCount = 0; // Counter for successfully inserted rows
$failedInserts = []; // Array to store failed insertions
for ($i = 0; $i < $dataCount; $i += $batchSize) {
// Avoiding repetition of the last row in the batch
$batchData = array_slice($sourceData, $i, $batchSize);
// Ensure no repetition in the last row
if ($i + $batchSize > $dataCount) {
$batchData = array_unique($batchData, SORT_REGULAR);
}
// Build the query for batch insertion
$values = [];
$params = [];
foreach ($batchData as $row) {
$values[] = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
$params = array_merge($params, [
$row['bookcode'], $row['chapter_doi'], $row['chapter'], $row['chapter_title'],
$row['firstname'], $row['lastname'], $row['contributor_type'], $row['email'],
$row['affiliation'], $row['country'], $row['bookxml_year']
]);
}
// Prepare the batched insert query
$sql = "INSERT INTO chapter_contributor_info
(bookcode, chapter_doi, chapter, chapter_title, firstname, lastname, contributor_type, email, affiliation, country, bookxml_year)
VALUES " . implode(", ", $values);
try {
$stmt = $targetDB->prepare($sql);
$stmt->execute($params);
$insertedCount += $stmt->rowCount(); // Count inserted rows
} catch (PDOException $e) {
// Log error message and the problematic data
$failedInserts[] = [
'data' => $batchData,
'error' => $e->getMessage()
];
}
}
$targetDB->commit(); // Commit the transaction
echo "Data successfully extracted, transformed, and inserted into chapter_contributor_info in batches. Total inserted rows: $insertedCount.";
// Log failed insertions
if (!empty($failedInserts)) {
file_put_contents('failed_inserts.log', print_r($failedInserts, true), FILE_APPEND);
echo "Some rows failed to insert. Check failed_inserts.log for details.";
}
} catch (PDOException $e) {
// Roll back the transaction in case of error
if ($targetDB->inTransaction()) {
$targetDB->rollBack();
}
echo "Error: " . $e->getMessage();
}
// Close connections
$sourceDB = null;
$targetDB = null;
8