I am trying to implement multiprocessing with gramformer which is an open source model to correct grammatical errors. I have tried several approaches but i keep getting errors cant be pickled. i am also using spacy which seems to have no problem with multiprocessing.
my main goal is to try to implement this in a class. i have tried creating seperate functions but even that does not work
class GramformerWrapper:
def __init__(self):
# Initialize Gramformer
self.gf = Gramformer(models=1, use_gpu=False)
def process_entries(self,entry):
paragraph=spacy_wrapper.nlp(entry)
corrected_entry=entry=''
for sentence in paragraph.sents:
corrected_sentence = list(self.gf.correct(sentence.text))[0]
print(corrected_sentence)
if not sentence.text.endswith('.') and corrected_sentence.endswith('.'):
corrected_sentence = corrected_sentence[:-1]
corrected_entry=corrected_entry+' '+corrected_sentence
return corrected_entry
def correct_sentences(self, lst):
length_of_lst=len(lst)
part=int(length_of_lst/8)+1
first_index=0
last_index=part
processes=[]
for _ in range(8):
lst_part=lst[first_index:last_index]
p=multiprocessing.Process(target=self.process_entries,args=lst_part)
p.start()
processes.append(p)
first_index+=part
last_index+=part
for process in processes:
process.join()
return corrected_entries
class SpacyWrapper:
def __init__(self):
self.nlp=spacy.load('en_core_web_sm')
gramformer_wrapper = GramformerWrapper()
spacy_wrapper=SpacyWrapper()