I am trying to implement text summarisation in my iOS app.
In order to achieve this, I thought to use one of the text summarisation models on hugging face, and convert that to a coreml model.
I first thought to straight convert the hugging face transformer model to coreml:
import coremltools as ct
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large-cnn")
model = AutoModelForSeq2SeqLM.from_pretrained("facebook/bart-large-cnn")
mlmodel = ct.convert(model)
mlmodel.save("textsum")
However that yielded this error:
Traceback (most recent call last):
File "/Users/anousheh/Documents/tech/projects/apps/NotesReviewApp/ai_true/textSum.py", line 7, in <module>
mlmodel = ct.convert(model)
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/coremltools/converters/_converters_entry.py", line 534, in convert
exact_source = _determine_source(model, source,
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/coremltools/converters/_converters_entry.py", line 972, in _determine_source
raise ValueError(msg)
ValueError: Unable to determine the type of the model, i.e. the source framework. Please provide the value of argument "source", from one of ["tensorflow", "pytorch", "milinternal"]. Note that model conversion requires the source package that generates the model. Please make sure you have the appropriate version of source package installed. E.g., if you're converting model originally trained with TensorFlow 1.14, make sure you have `tensorflow==1.14` installed.
I then went on to specify the model:
import coremltools as ct
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
import torch
# Load tokenizer and model
tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large-cnn")
model = AutoModelForSeq2SeqLM.from_pretrained("facebook/bart-large-cnn")
# Convert to TorchScript using scripting
model.eval() # Set the model to evaluation mode
scripted_model = torch.jit.script(model)
# Convert the scripted model to Core ML
mlmodel = ct.convert(scripted_model, source='pytorch')
mlmodel.save("TextSum")
However now this is still yielding errors:
/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/transformers/modeling_utils.py:4565: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead
warnings.warn(
Traceback (most recent call last):
File "/Users/anousheh/Documents/tech/projects/apps/NotesReviewApp/ai_true/textSum2.py", line 11, in <module>
scripted_model = torch.jit.script(model)
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/_script.py", line 1338, in script
return torch.jit._recursive.create_script_module(
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/_recursive.py", line 558, in create_script_module
return create_script_module_impl(nn_module, concrete_type, stubs_fn)
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/_recursive.py", line 631, in create_script_module_impl
script_module = torch.jit.RecursiveScriptModule._construct(cpp_module, init_fn)
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/_script.py", line 647, in _construct
init_fn(script_module)
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/_recursive.py", line 607, in init_fn
scripted = create_script_module_impl(
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/_recursive.py", line 631, in create_script_module_impl
script_module = torch.jit.RecursiveScriptModule._construct(cpp_module, init_fn)
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/_script.py", line 647, in _construct
init_fn(script_module)
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/_recursive.py", line 607, in init_fn
scripted = create_script_module_impl(
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/_recursive.py", line 571, in create_script_module_impl
method_stubs = stubs_fn(nn_module)
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/_recursive.py", line 898, in infer_methods_to_compile
stubs.append(make_stub_from_method(nn_module, method))
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/_recursive.py", line 87, in make_stub_from_method
return make_stub(func, method_name)
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/_recursive.py", line 71, in make_stub
ast = get_jit_def(func, name, self_name="RecursiveScriptModule")
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 372, in get_jit_def
return build_def(
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 433, in build_def
return Def(Ident(r, def_name), decl, build_stmts(ctx, body))
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 195, in build_stmts
stmts = [build_stmt(ctx, s) for s in stmts]
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 195, in <listcomp>
stmts = [build_stmt(ctx, s) for s in stmts]
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 406, in __call__
return method(ctx, node)
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 782, in build_If
build_stmts(ctx, stmt.body),
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 195, in build_stmts
stmts = [build_stmt(ctx, s) for s in stmts]
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 195, in <listcomp>
stmts = [build_stmt(ctx, s) for s in stmts]
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 406, in __call__
return method(ctx, node)
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 721, in build_Return
return Return(r, None if stmt.value is None else build_expr(ctx, stmt.value))
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 406, in __call__
return method(ctx, node)
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 890, in build_Call
args = [build_expr(ctx, py_arg) for py_arg in expr.args]
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 890, in <listcomp>
args = [build_expr(ctx, py_arg) for py_arg in expr.args]
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 406, in __call__
return method(ctx, node)
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 1235, in build_GeneratorExp
return ExprBuilder.build_ListComp(ctx, stmt)
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/torch/jit/frontend.py", line 1224, in build_ListComp
raise NotSupportedError(r, "Comprehension ifs are not supported yet")
torch.jit.frontend.NotSupportedError: Comprehension ifs are not supported yet:
File "/Users/anousheh/miniforge3/envs/cv/lib/python3.10/site-packages/transformers/models/bart/modeling_bart.py", line 1237
if not return_dict:
return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
return BaseModelOutput(
last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
If anyone knows how I can rectify this so that I can convert this Transformer model to coreml, it would be greatly appreciated.