I am writing a program that uses the Linux perf api to measure the number of instructions run by a program, and I want to receive a notification once the child process reaches a given number of instructions. Perf provides a way to do this via overflow notifications, and according to its official manpage overflow notifications can be captured in two ways:

Overflow handling
       Events can be set to notify when a threshold is crossed,
       indicating an overflow.  Overflow conditions can be captured by
       monitoring the event file descriptor with poll(2), select(2), or
       epoll(7).  Alternatively, the overflow events can be captured via
       sa signal handler, by enabling I/O signaling on the file
       descriptor; see the discussion of the F_SETOWN and F_SETSIG
       operations in fcntl(2).

Enabling I/O signaling on the perf file descriptor and setting a signal handler is not an option for me due to the specifics of my program, so I would like to use epoll on the file descriptor. However, epoll_wait keeps returning EPOLLHUP (even when an overflow notification shouldn’t be sent) and never returns the EPOLLIN that it should. I’ve read pretty much all information available on the internet about this use case of perf_event_open, and still I’ve been unable to ascertain why this happens.

Here is the simplified version of the source code I’m working with:

#include <bits/stdc++.h>
#include <linux/perf_event.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <sys/ioctl.h>
#include <sys/syscall.h>
#include <sys/epoll.h>
#include <fcntl.h>
#include <unistd.h>

using namespace std;

char* stringToCStr(const string &str) {
    char* cStr = new char[str.size() + 1];
    copy(str.begin(), str.end(), cStr);
    cStr[str.size()] = '';
    
    return cStr;
}

static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags) {
    return syscall(SYS_perf_event_open, hw_event, pid, cpu, group_fd, flags);
}

long long get_instructions_used(int perf_fd) {
    long long int instructionsUsed;
    int size = read(perf_fd, &instructionsUsed, sizeof(long long));
    
    if (size != sizeof(instructionsUsed)) {
        cout << "read failed";
        exit(0);
    }
    if (instructionsUsed < 0) {
        cout << "read negative instructions count";
        exit(0);
    }
    
    return static_cast<uint64_t>(instructionsUsed);
}

int main() {
    pthread_barrier_t *barrier_ = (pthread_barrier_t*) mmap(nullptr, sizeof(pthread_barrier_t), PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_SHARED, 0, 0);
    pthread_barrierattr_t attr;
    pthread_barrierattr_init(&attr);
    pthread_barrierattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
    pthread_barrier_init(barrier_, &attr, 2);
    pthread_barrierattr_destroy(&attr);

    int pid = fork();
    
    if(pid == 0) {
        pthread_barrier_wait(barrier_);
        munmap(barrier_, sizeof(pthread_barrier_t));

        auto programName = stringToCStr("test");
        char** programArgv = new char*[2];
        programArgv[0] = programName;
        programArgv[1] = nullptr;

        execv(programName, programArgv);
    };
    
    struct perf_event_attr attrs {};
    memset(&attrs, 0, sizeof(attrs));
    attrs.type = PERF_TYPE_HARDWARE;
    attrs.size = sizeof(attrs);
    attrs.config = PERF_COUNT_HW_INSTRUCTIONS;
    attrs.exclude_user = 0;
    attrs.exclude_kernel = 1;
    attrs.exclude_hv = 1;
    attrs.disabled = 1;
    attrs.enable_on_exec = 1;
    attrs.inherit = 1;
    attrs.sample_period = 500000000LL;
    attrs.wakeup_events = 1;
    int perf_fd = perf_event_open(&attrs, pid, -1, -1, PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_CLOEXEC);
    
    pthread_barrier_wait(barrier_);
    pthread_barrier_destroy(barrier_);
    munmap(barrier_, sizeof(pthread_barrier_t));

    // Uncomment to enable capturing events via signal handler
    /*int my_pid = getpid();
    fcntl(perf_fd, F_SETOWN, my_pid);
    int old_flags = fcntl(perf_fd, F_GETFL, 0);
    fcntl(perf_fd, F_SETFL, old_flags | O_ASYNC);*/

    int epoll_fd = epoll_create1(EPOLL_CLOEXEC);
    struct epoll_event event;
    event.events = EPOLLIN;
    event.data.u64 = 1;
    epoll_ctl(epoll_fd, EPOLL_CTL_ADD, perf_fd, &event);
    
    while(true) {
        struct epoll_event events[1];
        epoll_wait(epoll_fd, events, 1, -1);
        cout << "event: " << events[0].events << ", u64: " << event.data.u64 << ", instruction count: " << get_instructions_used(perf_fd) << "n";
        
        if(events[0].events == 1)
            break;
    }
    
    return 0;
}

The test executable the child runs that is given in the programName variable in the can be any simple program that uses a lot of instructions, for example a compiled executable of the following program:

#include "bits/stdc++.h"

using namespace std;

int main() {
    int n = 1e8, primeCount = 0;
    
    vector<bool> sieve(n + 1, true);
    for(int i = 2; i <= n; i++) {
        if(sieve[i]) {
            for(int j = i * 2; j <= n; j += i)
                sieve[j] = false;
            
            primeCount++;
        }
    }
    
    cout << primeCount << "n";
    
    return 0;
}

This code keeps printing out event: 16, u64: 1, instruction count: <instruction count>, which means that, as explained above, epoll keeps receiving EPOLLHUP from perf_fd. I have considered that this behavior happens because my perf event starts out disabled and only enables when the child process runs execv (as indicated by enable_on_exec), meaning that for a short period of time epoll might be polling the perf_fd while it’s disabled, but that still doesn’t explain why the file descriptor keeps returning EPOLLHUP even after the execv is run, and the behavior doesn’t dissapear if I add a short usleep before the epoll_ctl, making it run after the perf event is enabled.

Also, curiously, if you uncomment the commented lines above the epoll_create1 which make overflow notifications happen using the SIGIO signal (the other method mentioned in the overflow handling section), the signal is sent at the correct time and the process is terminated, however as I said I can’t actually use that in my final code. This does mean however that the problem somehow is with epoll, as the overflow notifications are propperly sent out on the file descriptor, epoll just doesn’t see them and instead spams epoll_wait with EPOLLHUP.