I’ve spent about a week trying to figure out why this loop isn’t working and I can’t figure it out.

I have a list of csv files with different parameters which track particles over time. What I’m trying to do is compare the last entry of the previous file with the first n entries in the next file. The program compares different parameters within a certain threshold and if there is a match the id column of each file gets changed so they both match. Then both files are merged to one. If a match isn’t found, the csv then gets merged with the others without making changes to it.

What I’m having issue with is each time I change the threshold parameters, the number of lines written to the file changes. Which shouldn’t be happening. It should stay the same regardless of parameters entered.

I’ve rewritten this program about fifteen times, fifteen different ways. I tried print statements but I still get the same problem.

Here is the main loop:

while (iteration_index < data_iterations):



#Open files and adjust frame rates plus sort and group data
if do_once:
    file_1 = pd.read_csv(sorted_files[0]).copy(deep = True)
    file_1.drop(file_1.index[0:3], inplace = True)
    file_1_sorted = file_1.sort_values(by = ["TRACK_ID","EDGE_TIME"], ascending = [True,True])
    delta = float(file_1_sorted.iloc[0,8])
    file_1_grouped = file_1_sorted.groupby("TRACK_ID", sort = False)

    file_1.to_csv(temp_master_location, mode = 'x', index = False)
    do_once = False
    #continue

else:
    file_1 = pd.read_csv(temp_master_location).copy(deep = True)
    file_1_sorted = file_1.sort_values(by = ["TRACK_ID","EDGE_TIME"], ascending = [True,True])
    file_1_grouped = file_1_sorted.groupby("TRACK_ID", sort = False)

    #Set interval to adjust track times by
    adjustment_time_interval = (iteration_index + time_interval) - (delta + overlap_time)

file_2 = pd.read_csv(sorted_files[file_2_dir]).copy(deep = True)
file_2.drop(file_2.index[0:3], inplace = True)
file_2.iloc[:,8] = pd.to_numeric(file_2.iloc[:,8]) + adjustment_time_interval
file_2_sorted = file_2.sort_values(by = ["TRACK_ID","EDGE_TIME"], ascending = [True,True])
file_2_grouped = file_2_sorted.groupby("TRACK_ID", sort = False)

# Creat a list of indecies to sort through
track_id_1 = file_1.iloc[:,1]
track_id_1.drop_duplicates(inplace = True)
track_id_2 = file_2.iloc[:,1]
track_id_2.drop_duplicates(inplace = True)

#iterate through individual tracks groups from first file
for index_1 in track_id_1:
    group_1 = file_1_grouped.get_group(index_1)
    group_1_length = (len(group_1) - 1)
    last_row = group_1.iloc[group_1_length,:]
    compare_x_1 = float(last_row["EDGE_X_LOCATION"])
    compare_y_1 = float(last_row["EDGE_Y_LOCATION"])
    compare_time_1 = float(last_row["EDGE_TIME"])
    compare_speed_1 = float(last_row["SPEED"])
    lower_time_bound = compare_time_1 - time_window
    upper_time_bound = compare_time_1 + time_window
    lower_x_position = compare_x_1 - position_window
    upper_x_position = compare_x_1 + position_window
    lower_y_position = compare_y_1 - position_window
    upper_y_position = compare_y_1 + position_window
    lower_speed_bound = compare_speed_1 - speed_window
    upper_speed_bound = compare_speed_1 + speed_window

    if iteration_index == 0:
        group_1.iloc[:,1] = track_number
    else:
        track_number = int(group_1.iloc[0,1])

    #And for the last track in each group in 
    #first file compare first n tracks in second file 
    # renumber and save to file if a match is found
    for index_2 in track_id_2:
        group_2 = file_2_grouped.get_group(index_2)
        
        if overlap_frames > len(group_2):
            while  overlap_index and overlap_index > len(group_2):
                overlap_index -= 1  
        
        for row in range(overlap_index):
            first_row = group_2.iloc[row,:]
            compare_x_2 = float(first_row["EDGE_X_LOCATION"])
            compare_y_2 = float(first_row["EDGE_Y_LOCATION"])
            compare_time_2 = float(first_row["EDGE_TIME"])
            compare_speed_2 = float(first_row["SPEED"])

            if ((merge_on == 'n') and (lower_x_position <= compare_x_2 <= upper_x_position) and (lower_y_position <= compare_y_2 <= upper_y_position) and (lower_time_bound <= compare_time_2 <= upper_time_bound) and (lower_speed_bound <= compare_speed_2 <= upper_speed_bound)):
                group_2.iloc[:,1] = track_number
                group_2.to_csv(temp_master_location, mode = 'a', header = None, index = False)
                file_2_sorted = file_2_sorted[file_2_sorted.TRACK_ID != index_2]
                merge_tracker += 1
                break
            if((merge_on == 'y') and (compare_x_1 == compare_x_2) and (compare_y_1 == compare_y_2)):
                group_2.iloc[:,1] = track_number
                group_2.to_csv(temp_master_location, mode = 'a', header = None, index = False)
                file_2_sorted = file_2_sorted[file_2_sorted.TRACK_ID != index_2]
                merge_tracker += 1
                break
            else:
                continue

#Renumber tracks not found in previous file
#And write to master file
index_list = list(file_2_sorted["TRACK_ID"].unique())
file_2_renumber = file_2_sorted.groupby("TRACK_ID")
track_number += 1
for indecies in index_list:
    file_2_output = file_2_renumber.get_group(indecies)
    file_2_output.iloc[:,1] = track_number
    file_2_output.iloc[(overlap_index-1):].to_csv(temp_master_location, mode = 'a', header = None, index = False)
    track_number += 1

#increment counters and reset track number
iteration_index +=1
file_2_dir += 1
time_interval += 1
overlap_index = overlap_frames

I’ve tried writing the files in the for loop towards the bottom of the while loop and I’ve tried writing the files outside of the for loop, but I still get the same results.

Any help would greatly be appreciated.