This program is a minimized version of a larger project that processes text files to generate CSV output with word frequencies. The full version works well for small to medium-sized files but encounters issues with very large texts like “The Lord of the Rings”.
The program must run on Unix-like systems and uses libraries exclusively of the C standard.
- It reads words from an input file, with a maximum word length of 30 characters.
- Words are sent through a pipe from a child process to the parent process.
- The parent process reads from the pipe and writes to an output file, formatting 100 words per line.
- The program uses basic file I/O, pipes, and process forking to demonstrate the core functionality.
-
I’ve simplified the original code to focus on the core functionality of reading, piping, and writing words.
-
I’ve implemented basic error checking for file operations and pipe creation.
-
I’ve used a fixed buffer size for reading words to avoid potential buffer overflow issues.
The program should successfully process files of any size, reading all words from the input file and writing them to the output file in the specified format (100 words per line). It should handle very large input files without memory issues or data loss.
While this minimized version may work for moderately sized files, I’m concerned about its performance and reliability with extremely large inputs. The original, more complex version struggles with very large files, and I’m trying to isolate the cause.
-
Are there any obvious issues in this code that could lead to problems with very large files?
-
How can I improve the pipe handling or memory management to better handle large amounts of data?
-
Are there alternative approaches or optimizations that would be more suitable for processing extremely large text files?
Below, I’ve included both the minimized code and the original, more extensive code. The minimized version attempts to emulate the core functionality where I believe the issue lies. For those who want to examine a more comprehensive implementation, please refer to the extended code below.
I’m not certain if I’ve successfully isolated the problem in this minimized version, so any insights on either the minimized or full code would be greatly appreciated.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/wait.h>
#define MAX_WORD_LENGTH 30
#define WORDS_PER_LINE 30
// Function to read words from file and write to pipe
void read_words(FILE *file, int pipe_fd) {
char word[MAX_WORD_LENGTH + 1];
int count = 0;
while (fscanf(file, "%30s", word) == 1) {
write(pipe_fd, word, strlen(word) + 1);
count++;
if (count % WORDS_PER_LINE == 0) {
char newline = 'n';
write(pipe_fd, &newline, 1);
}
}
close(pipe_fd);
}
// Function to read words from pipe and write to output file
void write_words(int pipe_fd, FILE *output) {
char word[MAX_WORD_LENGTH + 1];
int bytes_read;
int count = 0;
while ((bytes_read = read(pipe_fd, word, MAX_WORD_LENGTH + 1)) > 0) {
if (word[0] == 'n') {
fprintf(output, "n");
count = 0;
} else {
fprintf(output, "%s ", word);
count++;
if (count % WORDS_PER_LINE == 0) {
fprintf(output, "n");
}
}
}
close(pipe_fd);
}
int main() {
// Open input file
FILE *input_file = fopen("input.txt", "r");
if (!input_file) {
perror("Failed to open input file");
exit(1);
}
// Open output file
FILE *output_file = fopen("output.txt", "w");
if (!output_file) {
perror("Failed to open output file");
fclose(input_file);
exit(1);
}
// Create pipe
int pipefd[2];
if (pipe(pipefd) == -1) {
perror("pipe");
exit(1);
}
// Fork process
pid_t pid = fork();
if (pid == -1) {
perror("fork");
exit(1);
} else if (pid == 0) { // Child process
close(pipefd[0]);
read_words(input_file, pipefd[1]);
fclose(input_file);
exit(0);
} else { // Parent process
close(pipefd[1]);
write_words(pipefd[0], output_file);
fclose(output_file);
wait(NULL);
}
return 0;
}
For a more detailed look at the original implementation, please refer to the following extended code:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#include <wchar.h>
#include <wctype.h>
#include <locale.h>
// Define a structure for an object containing a string, its frequency, and coordinates in the matrix
typedef struct {
wchar_t String[31]; // The word or string, limited to 30 characters plus null terminator
float frequency; // The frequency of occurrence of this word
int occurrence; // The number of times this word appears
int x; // The x-coordinate in the matrix
int y; // The y-coordinate in the matrix
} WordObject;
// Define a structure for a matrix containing a list of WordObjects and its dimensions
typedef struct {
WordObject** list; // A dynamic 2D array of WordObjects
int* x; // An array to store the width of each row
int y; // The number of rows in the matrix
} Matrix;
// Define a structure for the alphabet containing sub-alphabets and a word
struct Alphabet {
struct Alphabet* subAlphabet[46]; // Array of pointers to sub-alphabets (46 for extended character set)
WordObject* word; // Pointer to a WordObject
};
// Define the Dictionary type as a pointer to struct Alphabet
typedef struct Alphabet Dictionary;
// Function declarations
Dictionary* deallocation(Dictionary* dict);
int createTable(FILE* file, Matrix* matrix, Dictionary* dict, int pipe1[], int pipe2[], wchar_t firstWord[]);
int asciiIndex(wchar_t character);
bool compareStrings(wchar_t* string1, wchar_t* string2);
bool operativeSearch(wchar_t string[], int index, Dictionary* dict, int i, int p, Matrix* m);
void readWordFromInputFile(FILE* file, int pipe1[], long int position);
void print(FILE* sheet, int pipe2[]);
long int firstWord(FILE* file, Matrix* matrix);
// Main function
int main(void) {
// Set the locale to support Unicode
setlocale(LC_ALL, "en_US.UTF-8");
// Open input and output files
FILE* inputFile = fopen("input.txt", "r");
FILE* outputFile = fopen("output.csv", "w");
// Check if file opening was successful
if (inputFile == NULL || outputFile == NULL) {
exit(EXIT_FAILURE); // Exit with error if file opening failed
}
// Allocate memory for the dictionary
Dictionary* dictionary = (Dictionary*)calloc(1, sizeof(Dictionary));
dictionary->word = (WordObject*)calloc(1, sizeof(WordObject));
// Initialize the matrix
Matrix matrix;
matrix.list = (WordObject**)calloc(1, sizeof(WordObject*));
matrix.list[0] = (WordObject*)calloc(2, sizeof(WordObject));
matrix.x = (int*)calloc(1, sizeof(int));
matrix.x[0] = 1;
// Find the position of the first word in the input file
long int position = firstWord(inputFile, &matrix);
if (position == -1) {
exit(EXIT_FAILURE); // Exit with error if no valid word was found
}
// Create two pipes for inter-process communication
int pipe1[2], pipe2[2];
if (pipe(pipe1) == -1 || pipe(pipe2) == -1) {
exit(EXIT_FAILURE); // Exit with error if pipe creation failed
}
// Variable to store the maximum number of bytes in a matrix row
int MAX;
// Declare process IDs for the three child processes
pid_t pid_Process1, pid_Process2, pid_Process3;
// Create the first child process
pid_Process1 = fork();
if (pid_Process1 == -1) {
// If fork fails, print error message and exit
perror("Fork error");
return 1;
} else if (pid_Process1 == 0) {
// Child process 1
close(pipe1[0]); // Close read end of pipe1
// Read words from input file and write to pipe1
readWordFromInputFile(inputFile, pipe1, position);
exit(0); // Exit child process
} else {
// Parent process
close(pipe1[1]); // Close write end of pipe1
waitpid(pid_Process1, NULL, 0); // Wait for child process 1 to finish
}
// Create the second child process
pid_Process2 = fork();
if (pid_Process2 == -1) {
perror("Fork error");
return 1;
} else if (pid_Process2 == 0) {
// Child process 2
close(pipe2[0]); // Close read end of pipe2
wchar_t firstWordArray[31];
wcscpy(firstWordArray, matrix.list[0][0].String);
// Create table and write to pipe2
createTable(inputFile, &matrix, dictionary, pipe1, pipe2, firstWordArray);
exit(0);
} else {
// Parent process
close(pipe1[0]); // Close read end of pipe1
close(pipe2[1]); // Close write end of pipe2
waitpid(pid_Process2, NULL, 0); // Wait for child process 2 to finish
}
// Create the third child process
pid_Process3 = fork();
if (pid_Process3 == -1) {
perror("Fork error");
return 1;
} else if (pid_Process3 == 0) {
// Child process 3
close(pipe2[1]); // Close write end of pipe2
// Read from pipe2 and print to output file
print(outputFile, pipe2);
exit(0);
}
// Close all pipe ends in the parent process
close(pipe1[0]);
close(pipe1[1]);
close(pipe2[0]);
close(pipe2[1]);
// Wait for all child processes to finish
waitpid(pid_Process3, NULL, 0);
wait(NULL);
wait(NULL);
wait(NULL);
// Close input and output files
fclose(outputFile);
fclose(inputFile);
// Free allocated memory for the dictionary
for (int i = 0; i < 46; i++) {
dictionary->subAlphabet[i] = deallocation(dictionary->subAlphabet[i]);
}
free(dictionary->word);
free(dictionary);
return 0; // Exit successfully
}
// Function to deallocate memory for the Dictionary structure
Dictionary* deallocation(Dictionary* dict) {
// If the dictionary is NULL, return NULL
if (dict == NULL) {
return NULL;
}
// Recursively deallocate memory for all sub-alphabets
for (int i = 0; i < 46; i++) {
dict->subAlphabet[i] = deallocation(dict->subAlphabet[i]);
}
// Free the memory allocated for the word
free(dict->word);
// Free the memory allocated for the dictionary itself
free(dict);
return NULL;
}
// Function to get the ASCII index of a character
int asciiIndex(wchar_t character) {
// Convert the character to lowercase
wchar_t c = towlower(character);
if (iswalpha(c)) {
// Handle non-accented letters
return c - L'a';
} else {
// Handle accented characters and special symbols
switch (c) {
case L'à': return 26;
case L'è': return 27;
case L'é': return 28;
case L'ì': return 29;
case L'ò': return 30;
case L'ù': return 31;
case L'!': return 32;
case L'?': return 33;
case L'.': return 34;
case L''': return 35;
}
}
// Handle numbers
if (c >= L'0' && c <= L'9') {
return 36 + (c - L'0');
}
// Return -1 for unrecognized characters
return -1;
}
// Function to compare two strings
bool compareStrings(wchar_t* String1, wchar_t* String2) {
// If both strings are empty, they are equal
if (*String1 == L'' && *String2 == L'') {
return true;
}
// If the characters are the same or the same when converted to lowercase
if (*String1 == *String2 || (towlower((wchar_t)*String1) == towlower((wchar_t)*String2))) {
// Recursively compare the next characters
return compareStrings(String1 + 1, String2 + 1);
}
// If the characters are different, the strings are not equal
return false;
}
// Function to find the first word in the input file
long int firstWord(FILE* file, Matrix* matrix) {
wchar_t character;
const wchar_t* accentedCharacters = L"àèéìòóùÀÈÉÌÒÓÙ";
int index = 0;
long int pos = 0;
while ((character = fgetwc(file)) != WEOF) {
wchar_t next = fgetwc(file);
ungetwc(next, file);
// Skip initial newlines or spaces
if (character == L'n' || character == L' ') {
continue;
} else if (iswalnum(character) || (wcschr(accentedCharacters, character) != NULL)) {
// Add alphanumeric or accented characters to the matrix
matrix->list[0][0].String[index] = character;
index++;
// Check if this is the end of the word
if (next == L' ' || (next == L'n' || next == L'r') || next == L'!' || next == L'?' || next == L'.') {
matrix->list[0][0].String[index] = L'';
index = 0;
return ftell(file);
} else if (next == L''') {
// Handle apostrophes
matrix->list[0][0].String[index] = L''';
matrix->list[0][0].String[index + 1] = L'';
index = 0;
return ftell(file);
}
continue;
} else if (character == L'!' || character == L'?' || character == L'.') {
// Handle punctuation marks
index = 0;
matrix->list[0][0].String[index++] = character;
if (next == ''') {
matrix->list[0][0].String[index++] = ''';
fgetwc(file);
}
matrix->list[0][0].String[index] = L'';
index = 0;
return ftell(file);
}
}
// Return -1 if no valid word was found
return -1;
}
// Function for operative search of a string in the matrix
bool operativeSearch(wchar_t string[], int index, Dictionary* dict, int i, int p, Matrix* m) {
Dictionary* currentDict = dict;
int ASCII;
// If we've reached the end of the string
if (string[index] == '') {
// Check if the string matches the word in the current dictionary node
if (compareStrings(currentDict->word->String, m->list[i][0].String)) {
int n = 1;
int y = currentDict->word->y;
int x = currentDict->word->x;
int found = 0;
// Increment the occurrence count
m->list[y][0].occurrence++;
// Update frequencies for existing words
while (n <= m->x[y] && m->list[y][n].x == n) {
if (compareStrings(m->list[y][n].String, m->list[i][p].String)) {
m->list[y][n].occurrence++;
m->list[y][n].frequency = (float)m->list[y][n].occurrence / (float)m->list[y][0].occurrence;
found = 1;
}
m->list[y][n].frequency = (float)m->list[y][n].occurrence / (float)m->list[y][0].occurrence;
n++;
}
// If the word wasn't found, add it
if (!found) {
if (n > m->x[y]) {
m->x[y] = n;
}
m->list[y] = realloc(m->list[y], (n + 1) * sizeof(WordObject));
wcscpy(m->list[y][n].String, m->list[i][p].String);
m->list[y][n].occurrence = 1;
m->list[y][n].frequency = (float)m->list[y][n].occurrence / (float)m->list[y][0].occurrence;
m->list[y][n].x = n;
}
return true;
}
// If no match was found, add the new word to the dictionary
wcscpy(currentDict->word->String, string);
currentDict->word->x = p;
currentDict->word->y = i;
m->list[i][1].x = 1;
m->list[i][0].x = 0;
m->list[i][1].occurrence = 1;
m->list[i][0].occurrence = 1;
m->list[i][1].frequency = (float)m->list[i][1].occurrence / (float)m->list[i][0].occurrence;
return false;
}
// Get the ASCII index for the current character
ASCII = asciiIndex(string[index]);
// If the sub-alphabet for this character doesn't exist, create it
if (currentDict->subAlphabet[ASCII] == NULL) {
currentDict->subAlphabet[ASCII] = (Dictionary*)calloc(1, sizeof(Dictionary));
if (currentDict->subAlphabet[ASCII] == NULL) {
exit(1);
}
currentDict->subAlphabet[ASCII]->word = (WordObject*)calloc(1, sizeof(WordObject));
if (currentDict->subAlphabet[ASCII]->word == NULL) {
exit(1);
}
}
// Move to the next sub-alphabet
currentDict = currentDict->subAlphabet[ASCII];
// Recursive call for the next character
return operativeSearch(string, index + 1, currentDict, i, p, m);
}
// Function to create the table from the input
int createTable(FILE* file, Matrix* matrix, Dictionary* dict, int pipe1[], int pipe2[], wchar_t firstWord[]) {
wchar_t string1[31]; // Buffer to store each word (max 30 characters + null terminator)
matrix->y = 1; // Initialize the number of rows to 1
matrix->list = NULL; // Initialize the list to NULL
// Allocate memory for the first row of the matrix
matrix->list = (WordObject**)calloc(1, matrix->y * sizeof(WordObject*));
matrix->list[matrix->y - 1] = (WordObject*)calloc(1, 2 * sizeof(WordObject));
// Copy the first word into the matrix
wcscpy(matrix->list[0][0].String, firstWord);
// Read all words from pipe1
while (read(pipe1[0], &string1, sizeof(string1)) > 0) {
if (string1[0] == '') { // If an empty string is read, break the loop
break;
}
// Add the word to the matrix
wcscpy(matrix->list[matrix->y - 1][1].String, string1);
matrix->list[matrix->y - 1][1].x = 1;
matrix->list[matrix->y - 1][0].x = 0;
// Perform operative search and update the matrix
int control = operativeSearch(matrix->list[matrix->y - 1][0].String, 0, dict, matrix->y - 1, 1, matrix);
if (control == 0) {
// If a new word is found, expand the matrix
matrix->y++;
matrix->list = realloc(matrix->list, matrix->y * sizeof(WordObject*));
matrix->list[matrix->y - 1] = (WordObject*)calloc(2, sizeof(WordObject));
matrix->x = realloc(matrix->x, matrix->y * sizeof(int));
matrix->x[matrix->y - 1] = 1;
}
// Copy the current word to be the first word of the next row
wcscpy(matrix->list[matrix->y - 1][0].String, string1);
string1[0] = ''; // Reset the string
}
// Handle the last word
wcscpy(matrix->list[matrix->y - 1][1].String, matrix->list[0][0].String);
int control = operativeSearch(matrix->list[matrix->y - 1][0].String, 0, dict, matrix->y - 1, 1, matrix);
if (control == 0) {
matrix->y++;
}
close(pipe1[0]); // Close pipe1
// Write the number of rows to pipe2
write(pipe2[1], &(matrix->y), sizeof(int));
// Write each row of the matrix to pipe2
for (int i = 0; i < matrix->y-1; i++) {
// Calculate the length of the row
size_t len = wcslen(matrix->list[i][0].String);
for (int j = 1; j <= matrix->x[i]; j++) {
len += wcslen(matrix->list[i][j].String) + 7; // +7 for ",%.4f,"
}
len++; // For the newline character
// Allocate memory for the row
wchar_t* row = calloc(len + 1, sizeof(wchar_t));
// Construct the row string
swprintf(row, len + 1, L"%ls", matrix->list[i][0].String);
for (int j = 1; j <= matrix->x[i]; j++) {
wchar_t temp[50];
swprintf(temp, 50, L",%ls,%.4f", matrix->list[i][j].String, matrix->list[i][j].frequency);
len += 50;
row = realloc(row, len * sizeof(wchar_t));
wcscat(row, temp);
}
wcscat(row, L"n");
// Write the length of the row and the row itself to pipe2
write(pipe2[1], &len, sizeof(size_t));
write(pipe2[1], row, len * sizeof(wchar_t));
free(row);
}
// Write end signal (length 0) to pipe2
size_t end_signal = 0;
write(pipe2[1], &end_signal, sizeof(size_t));
close(pipe2[1]);
return 0;
}
// Function to print the table to the output file
void print(FILE* outputFile, int pipe2[]) {
close(pipe2[1]); // Close the write end of pipe2
int numRows;
ssize_t bytesRead = read(pipe2[0], &numRows, sizeof(int));
if (bytesRead != sizeof(int)) {
if (bytesRead == 0) {
fprintf(stderr, "Pipe closed unexpectedly while reading the number of rowsn");
} else {
perror("Error reading the number of rows");
}
exit(EXIT_FAILURE);
}
fprintf(stderr, "Number of rows read: %dn", numRows);
for (int i = 0; i < numRows; i++) {
size_t len;
bytesRead = read(pipe2[0], &len, sizeof(size_t));
if (bytesRead != sizeof(size_t)) {
if (bytesRead == 0) {
fprintf(stderr, "Pipe closed unexpectedly while reading the length of row %dn", i);
} else {
perror("Error reading the row length");
}
exit(EXIT_FAILURE);
}
fprintf(stderr, "Length of row %d: %zun", i, len);
if (len == 0) {
fprintf(stderr, "Received end of data signaln");
break;
}
wchar_t* row = calloc(len + 1, sizeof(wchar_t));
if (row == NULL) {
perror("Error allocating memory for the row");
exit(EXIT_FAILURE);
}
bytesRead = read(pipe2[0], row, len * sizeof(wchar_t));
if (bytesRead != len * sizeof(wchar_t)) {
if (bytesRead == 0) {
fprintf(stderr, "Pipe closed unexpectedly while reading row %dn", i);
} else {
perror("Error reading the row");
}
free(row);
exit(EXIT_FAILURE);
}
row[len] = L'';
fprintf(stderr, "Row %d read: %lsn", i, row);
fwprintf(outputFile, L"%ls", row);
free(row);
}
close(pipe2[0]);
fprintf(stderr, "Printing completedn");
}
// Function to read words from the input file
void readWordFromInputFile(FILE* file, int pipe1[], long int position) {
wchar_t string1[31];
int index = 0;
wchar_t character;
const wchar_t* accentedCharacters = L"àèéìòóùÀÈÉÌÒÓÙ";
// Position the file pointer
for(int i = 0; i == position; i++){
fgetwc(file);
}
while ((character = fgetwc(file)) != WEOF) {
wchar_t next = fgetwc(file);
ungetwc(next, file);
// Handle end of file
if ((character == L' ' || (character == L'n' || character == L'r')) && next == WEOF) {
if (index > 0) {
string1[index] = L'';
write(pipe1[1], string1, sizeof(string1));
}
break;
}
// Handle punctuation
if (character == L'!' || character == L'?' || character == L'.') {
if (index > 0) {
string1[index] = L'';
write(pipe1[1], string1, sizeof(string1));
index = 0;
}
string1[index] = character;
if (next == L''') {
index++;
string1[index] = L''';
fgetwc(file);
}
index++;
if (next == WEOF) {
string1[index] = L'';
write(pipe1[1], string1, sizeof(string1));
break;
}
} else if (index > 0 && character == ''') {
string1[index] = L''';
index++;
string1[index] = L'';
write(pipe1[1], string1, sizeof(string1));
index = 0;
string1[index] = L'';
} else if (iswalnum(character) || (wcschr(accentedCharacters, character) != NULL)) {
if (index > 0 && (string1[index - 1] == '!' || string1[index - 1] == '?' || string1[index - 1] == '.' || string1[index - 1] == ''')) {
string1[index] = L'';
write(pipe1[1], string1, sizeof(string1));
index = 0;
string1[index] = L'';
}
string1[index++] = character;
if (next == ''') {
string1[index++] = L''';
string1[index] = L'';
write(pipe1[1], string1, sizeof(string1));
fgetwc(file);
index = 0;
}
if (next == L' ' || (next == L'n' || next == L'r') || next == WEOF ||
next == L'!' || next == L'?' || next == L'.') {
string1[index] = L'';
write(pipe1[1], string1, sizeof(string1));
index = 0;
}
} else if (character == L' ' || (character == L'n' || character == L'r')) {
if (index > 0) {
string1[index] = L'';
write(pipe1[1], string1, sizeof(string1));
index = 0;
string1[index] = L'';
}
} else if (!(iswalnum(character) || (wcschr(accentedCharacters, character) != NULL) ||
character == L''' || character == L'!' ||
character == L'?' || character == L'.') &&
(next == L' ' || (next == L'n' || next == L'r'))) {
continue;
}
if (next == WEOF) {
string1[index] = L'';
write(pipe1[1], string1, sizeof(string1));
break;
}
}
close(pipe1[1]);
}