I need to allocate a max uint32 worth of memory to an array with go but I am getting an error: fatal error: runtime: out of memory. I want to index into the array using a uint32 to manage all possibilities for a given computation that returns a 32 bit index. That I know of, an array of this size should be 4GB in size, give or take a small factor for how big the array entries are. I already checked ulimit, and it is not the source of error. My machine has 16 GB of ram, so there should be enough for this large allocation. This is the start of a project that will later scale to hundreds of gigabytes in memory.

I’m also encoding the result with protobuf, and the below is what I’m working on now.

syntax = "proto3";
package tables;
option go_package = "./tables";

message ExprInput {
    repeated uint32 input = 2;
}


message ExprTable {
    repeated ExprInput ReverseValue = 3;
}

Following the trace, this is the offending line:

const MX_u32 = (^uint32(0))

func precompute_for(name string, tbl *tables.ExprTable) {
    if tbl.ReverseValue == nil {
        tbl.ReverseValue = make([]*tables.ExprInput, MX_u32) // ERROR
    }
//...
}

func precompute(ptables * map[string]tables.ExprTable) {
    //try to read proto files in
    //else, compute them
    names := [...]string{"n1", "n2"};
    for name := range names {
        tbl := tables.ExprTable{};
        _, err := os.Stat(names[name])
        if err != nil {
            precompute_for(names[name], &tbl) 
            buf, _ := proto.Marshal(&tbl)
            os.WriteFile(names[name],buf, os.ModePerm)
        } else {
            dat, _ := os.ReadFile(names[name])
            proto.Unmarshal(dat, &tbl);
        }
        //(*ptables)[names[name]]=tbl;
    }
}

func main() {
    var  ptables map[string]tables.ExprTable
    ptables = make(map[string]tables.ExprTable)
    precompute(&ptables)
}