i just learn spark, installing most recent pyspark
my flow is i start with
<code>import findspark
findspark.init()
from pyspark import SparkConf, SparkContext, SQLContext
from pyspark.sql import SparkSession
import pyspark.sql.types as pysparkT
</code>
<code>import findspark
findspark.init()
from pyspark import SparkConf, SparkContext, SQLContext
from pyspark.sql import SparkSession
import pyspark.sql.types as pysparkT
</code>
import findspark
findspark.init()
from pyspark import SparkConf, SparkContext, SQLContext
from pyspark.sql import SparkSession
import pyspark.sql.types as pysparkT
then i try to test my sql connection
<code>import pyodbc
conn_str = (
r"Driver={SQL Server};"
r"Server=DESKTOPSQLEXPRESS;"
r"Database=AdventureWorks2016;" #taken from microsoft database refrence
r"Trusted_Connection=yes;"
)
conn = pyodbc.connect(conn_str)
cursor = conn.cursor()
cursor.execute("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE'")
rows = cursor.fetchall()
for row in rows:
print(row)
</code>
<code>import pyodbc
conn_str = (
r"Driver={SQL Server};"
r"Server=DESKTOPSQLEXPRESS;"
r"Database=AdventureWorks2016;" #taken from microsoft database refrence
r"Trusted_Connection=yes;"
)
conn = pyodbc.connect(conn_str)
cursor = conn.cursor()
cursor.execute("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE'")
rows = cursor.fetchall()
for row in rows:
print(row)
</code>
import pyodbc
conn_str = (
r"Driver={SQL Server};"
r"Server=DESKTOPSQLEXPRESS;"
r"Database=AdventureWorks2016;" #taken from microsoft database refrence
r"Trusted_Connection=yes;"
)
conn = pyodbc.connect(conn_str)
cursor = conn.cursor()
cursor.execute("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE'")
rows = cursor.fetchall()
for row in rows:
print(row)
which able to show the list of the schema of the database
now here is the problem, below is my full code
<code>master = 'local[*]'
appname = 'testing jdbc sqlserver'
database = 'AdventureWorks2016'
username = r'DESKTOP---'
serverName = r'DESKTOPSQLEXPRESS'
url = f"jdbc:sqlserver://{serverName}:1433;DatabaseName={database};integratedSecurity=true;"
driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
conf = SparkConf()
.setAppName(appname)
.setMaster(master)
.set("spark.driver.extraClassPath", "C:path-to-sqljdbcmssql-jdbc-12.8.1.jre8")
.set("spark.executor.memory", "4g")
.set("spark.driver.memory", "4g")
.set("spark.executor.cores", "4")
spark = SparkSession.builder.config(conf=conf).getOrCreate()
query_table = """(
SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE') custom_table"""
jcbdf = spark.read.format("jdbc")
.option("url", url)
.option("query", query_table)
.option("user", username)
.option("driver", driver)
.load()
jcbdf.show()
</code>
<code>master = 'local[*]'
appname = 'testing jdbc sqlserver'
database = 'AdventureWorks2016'
username = r'DESKTOP---'
serverName = r'DESKTOPSQLEXPRESS'
url = f"jdbc:sqlserver://{serverName}:1433;DatabaseName={database};integratedSecurity=true;"
driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
conf = SparkConf()
.setAppName(appname)
.setMaster(master)
.set("spark.driver.extraClassPath", "C:path-to-sqljdbcmssql-jdbc-12.8.1.jre8")
.set("spark.executor.memory", "4g")
.set("spark.driver.memory", "4g")
.set("spark.executor.cores", "4")
spark = SparkSession.builder.config(conf=conf).getOrCreate()
query_table = """(
SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE') custom_table"""
jcbdf = spark.read.format("jdbc")
.option("url", url)
.option("query", query_table)
.option("user", username)
.option("driver", driver)
.load()
jcbdf.show()
</code>
master = 'local[*]'
appname = 'testing jdbc sqlserver'
database = 'AdventureWorks2016'
username = r'DESKTOP---'
serverName = r'DESKTOPSQLEXPRESS'
url = f"jdbc:sqlserver://{serverName}:1433;DatabaseName={database};integratedSecurity=true;"
driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
conf = SparkConf()
.setAppName(appname)
.setMaster(master)
.set("spark.driver.extraClassPath", "C:path-to-sqljdbcmssql-jdbc-12.8.1.jre8")
.set("spark.executor.memory", "4g")
.set("spark.driver.memory", "4g")
.set("spark.executor.cores", "4")
spark = SparkSession.builder.config(conf=conf).getOrCreate()
query_table = """(
SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE') custom_table"""
jcbdf = spark.read.format("jdbc")
.option("url", url)
.option("query", query_table)
.option("user", username)
.option("driver", driver)
.load()
jcbdf.show()
encountering an error
<code>at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.ClientServerConnection.waitForCommands (ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.lang.Thread.run(Thread.java:750)
</code>
<code>at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.ClientServerConnection.waitForCommands (ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.lang.Thread.run(Thread.java:750)
</code>
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.ClientServerConnection.waitForCommands (ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.lang.Thread.run(Thread.java:750)
full error can be looked at here
i try whatever i could find but seems doesnt solve the issue. anyone know what i did wrong?
4