I have a utility to read all files from a directory and it’s sub-directories. I am using goroutines to improve performance. But I am getting deadlock during runtime, I know what the issue is, but don’t know what’s the alternative or the fix. Can someone fix the deadlock?

Deadlock is happening because all child goroutines are waiting to send into channel, while the parent which is receiving from the channel is blocked because of limiter channel. I don’t want to get rid of limiter channel completely, because then my program starts consuming huge memory.

This is the code,

package diskreader

import (
    "bufio"
    "fmt"
    "os"
    "sync"
)

type MDR2 struct {
    Wr       *bufio.Writer
    wg       *sync.WaitGroup
    doneChan chan struct{}
    pathChan chan *pathInfo
}

func NewMDR2(bwr *bufio.Writer) *MDR2 {
    mdr := &MDR2{Wr: bwr, wg: &sync.WaitGroup{}, doneChan: make(chan struct{}), pathChan: make(chan *pathInfo)}
    go mdr.startReaders()
    return mdr
}

func (mdr *MDR2) Read(baseDir string) {
    mdr.wg.Add(1)
    go mdr.read(baseDir)
    mdr.wg.Wait()
}

func (mdr *MDR2) read(baseDir string) error {
    // fmt.Println("Reading: ", baseDir)
    defer mdr.wg.Done()

    baseFileInfo, err := os.Stat(baseDir)
    if err != nil {
        return err
    }

    if !baseFileInfo.IsDir() {
        err = mdr.readFile(baseDir)
        if err != nil {
            return err
        }
        return nil
    }

    base, err := os.ReadDir(baseDir)
    if err != nil {
        return err
    }

    for _, entry := range base {
        nextLevelPath := baseDir + "/" + entry.Name()
        pi := pathInfo{isDir: entry.IsDir(), path: nextLevelPath}
        mdr.wg.Add(1)
        mdr.pathChan <- &pi
    }

    return nil
}

func (mdr *MDR2) readFile(path string) error {
    file, err := os.ReadFile(path)
    if err != nil {
        return err
    }

    msg := "path: " + path + " size: " + fmt.Sprint(len(file)) + "n"
    if mdr.Wr.Available() < len(msg) {
        err = mdr.Wr.Flush()
        if err != nil {
            return err
        }
    }
    mdr.Wr.WriteString(msg)
    return nil
}

func (mdr *MDR2) startReaders() {
    limiter := make(chan struct{}, 10)
    for pi := range mdr.pathChan {
        limiter <- struct{}{}
        go func(pi *pathInfo) {
            defer func() { <-limiter }()
            mdr.read(pi.path)
        }(pi)
    }
}