I am very green to Azure functions. But I have managed to deploy a function and have it run Successfully when the output is to be created to the current Directory. No issues up to here everything is fine.
The next thing I like to do is save the output to a specific folder. This causes an Error. I will list error shortly.
What I like to know is once I get this error even if I go back to the code that I have verified works now produces the same error. Yes I made sure I saved file before I ran the func start -verbose command
Ok here is the working code:
#Works NO issues
import azure.functions as func
import datetime
import logging
import subprocess
import os
app = func.FunctionApp()
@app.timer_trigger(schedule="0 * * * * *", arg_name="myTimer", run_on_startup=True, use_monitor=False)
def ScrapyTimerTrigger(myTimer: func.TimerRequest) -> None:
if myTimer.past_due:
logging.info('The timer is past due!')
logging.info('Python timer trigger function executed.')
week = 'week' # Set your desired week
year = '2023' # Set your desired year
game = 'hall-of-fame-weekend' # Set your desired game
local_file_name = f'{year}_{week}_{game}.json'
#local_file_path = os.path.join(os.getcwd(), local_file_name)
scrapy_command = [
'scrapy', 'crawl', 'NFLWeatherData',
'-a', f'Week={week}', '-a', f'Year={year}', '-a', f'Game={game}',
'-o', local_file_name # local_file_path
]
# Update this path to the correct directory
process = subprocess.Popen(scrapy_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=r"N:githubSportsDataNFLWeatherFunctionLocalNFLWeather")
stdout, stderr = process.communicate()
if process.returncode != 0:
logging.error(stderr)
else:
logging.info(stdout)
That will produce file with no issue
Here is the modified code to write to an another location
import azure.functions as func
import datetime
import logging
import subprocess
import os
app = func.FunctionApp()
@app.timer_trigger(schedule="0 * * * * *", arg_name="myTimer", run_on_startup=True,
use_monitor=False)
def ScrapyTimerTrigger(myTimer: func.TimerRequest) -> None:
if myTimer.past_due:
logging.info('The timer is past due!')
logging.info('Python timer trigger function executed.')
week = 'week' # Set your desired week
year = '2023' # Set your desired year
game = 'hall-of-fame-weekend' # Set your desired game
local_file_name = f'{year}_{week}_{game}.json'
local_file_path = os.path.join(r"N:\github\SportsData\NFLWeather\Data_Gathering", year, local_file_name)
# Create the directory if it does not exist
directory_path = os.path.join(r"N:\github\SportsData\NFLWeather\Data_Gathering", year)
os.makedirs(directory_path, exist_ok=True)
scrapy_command = [
'scrapy', 'crawl', 'NFLWeatherData',
'-a', f'Week={week}', '-a', f'Year={year}', '-a', f'Game={game}',
'-o', local_file_path
]
# Update this path to the correct directory
process = subprocess.Popen(scrapy_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=r"N:githubSportsDataNFLWeatherFunctionLocalNFLWeather")
stdout, stderr = process.communicate()
if process.returncode != 0:
logging.error(stderr)
else:
logging.info(stdout)
Here is the error code that this script produces
[2024-06-24T00:41:45.459Z] Received WorkerInitRequest, python version 3.11.9 | packaged by Anaconda, Inc. | (main, Apr 19 2024, 16:40:41) [MSC v.1916 64 bit (AMD64)], worker version 4.28.1, request ID 583ffc8a-a82c-49d4-b586-a0ded194964c. App Settings state: PYTHON_THREADPOOL_THREAD_COUNT: 1000 | PYTHON_ENABLE_WORKER_EXTENSIONS: False. To enable debug level logging, please refer to https://aka.ms/python-enable-debug-logging
[2024-06-24T00:41:45.767Z] Received WorkerMetadataRequest, request ID 583ffc8a-a82c-49d4-b586-a0ded194964c, function_path: N:githubSportsDataNFLWeatherFunctionLocalNFLWeatherNFLWeatherFunctionAppfunction_app.py
[2024-06-24T00:41:45.797Z] Worker failed to index functions
[2024-06-24T00:41:45.799Z] Result: Failure
Exception: SyntaxError: (unicode error) 'unicodeescape' codec can't decode bytes in position 1269-1270: malformed N character escape (function_app.py, line 43)
Stack: File "M:MicrosoftAzure Functions Core Toolsworkerspython3.11WINDOWSX64azure_functions_workerdispatcher.py", line 413, in _handle__functions_metadata_request
self.load_function_metadata(
File "M:MicrosoftAzure Functions Core Toolsworkerspython3.11WINDOWSX64azure_functions_workerdispatcher.py", line 393, in load_function_metadata
self.index_functions(function_path, function_app_directory))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:MicrosoftAzure Functions Core Toolsworkerspython3.11WINDOWSX64azure_functions_workerdispatcher.py", line 765, in index_functions
indexed_functions = loader.index_function_app(function_path)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:MicrosoftAzure Functions Core Toolsworkerspython3.11WINDOWSX64azure_functions_workerutilswrappers.py", line 44, in call
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "M:MicrosoftAzure Functions Core Toolsworkerspython3.11WINDOWSX64azure_functions_workerloader.py", line 238, in index_function_app
imported_module = importlib.import_module(module_name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:Miniconda3envsAzureLibimportlib__init__.py", line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<frozen importlib._bootstrap>", line 1204, in _gcd_import
File "<frozen importlib._bootstrap>", line 1176, in _find_and_load
File "<frozen importlib._bootstrap>", line 1147, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 690, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 936, in exec_module
File "<frozen importlib._bootstrap_external>", line 1074, in get_code
File "<frozen importlib._bootstrap_external>", line 1004, in source_to_code
File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
Here is what I dont understand an I like to know why or what do I have to do so that the working file works again. Is there some cache or something that has to be cleared in order for the working file to work again. This will allow me to make changes to path location and put something that works