I trying to test feature store offline on AWS Sagemaker but getting TypeError: ‘JavaPackage’ object is not callable
My spark version 3.5.1
Java version 8
bashrc set
export SPARK_HOME=/opt/spark-3.5.1-bin-hadoop3
export PATH=$SPARK_HOME/bin:$PATH
@feature_processor(
inputs=[CSVDataSource(FAKE_S3_URI)],
output=FAKE_FG_ARN,
target_stores=["OfflineStore"],
)
def transform(raw_s3_data_as_df):
current_time = int(time.time())
transform_df = (
raw_s3_data_as_df.withColumnRenamed('user','User')
.withColumnRenamed('password','PW')
.withColumnRenamed('name','Name')
.withColumn('ingest_time', lit(int(time.time())))
)
transform_df.show(10)
transform_df.printSchema()
return transform_df
transform()
TypeError Traceback (most recent call last)
Cell In[15], line 51
47 transform_df.printSchema()
49 return transform_df
---> 51 transform()
File /opt/conda/lib/python3.10/site-packages/sagemaker/feature_store/feature_processor/_udf_wrapper.py:62, in UDFWrapper.wrap.<locals>.wrapper()
55 udf_args, udf_kwargs = self._prepare_udf_args(
56 udf=udf,
57 fp_config=fp_config,
58 )
60 output = udf(*udf_args, **udf_kwargs)
---> 62 self.udf_output_receiver.ingest_udf_output(output, fp_config)
File /opt/conda/lib/python3.10/site-packages/sagemaker/feature_store/feature_processor/_udf_output_receiver.py:78, in SparkOutputReceiver.ingest_udf_output(self, output, fp_config)
70 return
72 logger.info(
73 "Ingesting transformed data to %s with target_stores: %s",
74 fp_config.output,
75 fp_config.target_stores,
76 )
---> 78 feature_store_manager = self.feature_store_manager_factory.feature_store_manager
79 try:
80 feature_store_manager.ingest_data(
81 input_data_frame=output,
82 feature_group_arn=fp_config.output,
83 target_stores=fp_config.target_stores,
84 )
File /opt/conda/lib/python3.10/site-packages/sagemaker/feature_store/feature_processor/_spark_factory.py:202, in FeatureStoreManagerFactory.feature_store_manager(self)
198 @property
199 @lru_cache()
200 def feature_store_manager(self) -> fsm.FeatureStoreManager:
201 """Instansiate a new FeatureStoreManager."""
--> 202 return fsm.FeatureStoreManager()
File /opt/conda/lib/python3.10/site-packages/feature_store_pyspark/FeatureStoreManager.py:32, in FeatureStoreManager.__init__(self, assume_role_arn)
30 def __init__(self, assume_role_arn: string = None):
31 super(FeatureStoreManager, self).__init__()
---> 32 self._java_obj = self._new_java_obj(FeatureStoreManager._wrapped_class, assume_role_arn)
File /opt/conda/lib/python3.10/site-packages/feature_store_pyspark/wrapper.py:45, in SageMakerFeatureStoreJavaWrapper._new_java_obj(self, java_class, *args)
42 for arg in args:
43 java_args.append(self._py2j(arg))
---> 45 return JavaWrapper._new_java_obj(java_class, *java_args)
File /opt/conda/lib/python3.10/site-packages/pyspark/ml/wrapper.py:86, in JavaWrapper._new_java_obj(java_class, *args)
84 java_obj = getattr(java_obj, name)
85 java_args = [_py2java(sc, arg) for arg in args]
---> 86 return java_obj(*java_args)
TypeError: 'JavaPackage' object is not callable
Tried to change to another JupyterLab on a different instance and reinstall everything, but still got that error again