I’m following this tutorial https://learn.microsoft.com/ja-jp/previous-versions/dotnet/spark/tutorials/get-started?tabs=windows to try to test
.Net Spark from Visual Studio.
But at the very begining, got error “System.Net.Sockets.SocketException: The connection was refused by target computer. 127.0.0.1:5567”.
Tried different port number but the simillar error message again and again.
OS: Windows11.
What’s wrong in here?
using Microsoft.Spark.Sql;
using System;
using Newtonsoft.Json.Linq;
using static Microsoft.Spark.Sql.Functions;
namespace MySparkApp
{
class Program
{
static void Main(string[] args)
{
//Environment.SetEnvironmentVariable("DOTNETBACKEND_PORT", "56511");
// Create Spark session
SparkSession spark =
SparkSession
.Builder()
.AppName("word_count_sample")
.GetOrCreate();
// Create initial DataFrame
string filePath = args[0];
DataFrame dataFrame = spark.Read().Text(filePath);
//Count words
DataFrame words =
dataFrame
.Select(Split(Col("value"), " ").Alias("words"))
.Select(Explode(Col("words")).Alias("word"))
.GroupBy("word")
.Count()
.OrderBy(Col("count").Desc());
// Display results
words.Show();
// Stop Spark session
spark.Stop();
}
}
}