I’m building an application like AnyDesk, or TeamViwer but specialized to operate in a specific work environment. There is a central server responsible of handling every request of a client, and the whole network has a star topology. I’m using Kivy 2.3.0 for the client sided GUI functionality and Pillow 10.4.0 for the image handling.
User ‘A’ requests user ‘B’ to start streaming his screen which gets sent to the server and the server searches the active users for a socket object (there is actually a tunnel system which a user can activate or deactivate, and if a tunnel is made, it makes it easier for the server to redirect messages to the requested client). This was purely to give you a background on the actual problem.
Once the user ‘B’ sends the image which was captured by PIL.ImageGrab.Image gets converted into a bytes format and sent through the server to the requested client, user ‘A’ is unable to reconvert the byte data into a Kivy image which is then set to an Image.texture on the GUI.
My problem is that when I try to convert the bytes into a pillow Image there is an SDL2: Unable to load image error.
server_networking.py:
`import socket
import json
import threading
import users
class ServerNetwork:
def init(self):
self.users = users.Users()
with open("../protocols.json") as protocols_file:
self.protocols = json.load(protocols_file)
with open("server_data.json") as server_data_file:
server_data = json.load(server_data_file)
self.PORT = server_data["port"]
self.HEADER = server_data["header"]
self.FORMAT = server_data["format"]
if server_data["ip"] == True:
self.IP = socket.gethostbyname(socket.gethostname())
elif type(server_data["ip"]) is str:
self.IP = server_data["ip"]
else:
self.IP = None
def main(self):
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.IP, self.PORT))
self.server_socket.listen()
threading.Thread(target=self.input_check).start()
while True:
conn, addr = self.server_socket.accept()
threading.Thread(target=self.handle_client, args=(conn, addr)).start()
def handle_client(self, client, address):
self.client_connected = True
while self.client_connected:
msg_length = client.recv(self.HEADER).decode(self.FORMAT)
if msg_length:
msg_length = int(msg_length)
msg = client.recv(msg_length).decode(self.FORMAT)
self.protocol_check(msg, client)
def protocol_check(self, message: str, user):
# 'data' for most of the user related protocols is the name of the user
protocol, data = message.split(self.protocols["PROTOCOL_MESSAGE_SPLITTER"])
if protocol == self.protocols["DISCONNECT"]:
self.client_connected = False
elif protocol == self.protocols["ADD_USER"]:
if self.users.add_user(data, False):
self.users.login(data, user)
self.send(self.protocols["ADD_USER"], True, user)
else:
self.send(self.protocols["ADD_USER"], False, user)
elif protocol == self.protocols["DELETE_USER"]:
self.users.delete_user(data)
elif protocol == self.protocols["LOG_IN"]:
self.users.login(data, user)
elif protocol == self.protocols["LOG_OUT"]:
self.users.logoff(data)
elif protocol == self.protocols["MAKE_RESTRICTED"]:
made_restricted = self.users.make_restricted(data)
if made_restricted:
self.send(self.protocols["MAKE_RESTRICTED"], True, user)
elif not made_restricted:
self.send(self.protocols["MAKE_RESTRICTED"], False, user)
elif protocol == self.protocols["MAKE_UNRESTRICTED"]:
made_unrestricted = self.users.make_unrestricteded(data)
if made_unrestricted:
self.send(self.protocols["MAKE_UNRESTRICTED"], True, user)
elif not made_unrestricted:
self.send(self.protocols["MAKE_UNRESTRICTED"], False, user)
elif protocol == self.protocols["MAKE_TUNNEL"]:
requester_name, requestee_name = data.split(self.protocols["MAKE_TUNNEL_INPUT_SEPARATOR"])
result = self.users.make_tunnel(requester_name, requestee_name)
if result == "User restricted":
self.send(self.protocols["CHOOSE_TUNNEL_CREATION"], requester_name, self.users.get_socket_of_user(requestee_name))
else:
self.send(self.protocols["MAKE_TUNNEL"], result, user)
elif protocol == self.protocols["REMOVE_TUNNEL"]:
requester_name, requestee_name = data.split(self.protocols["REMOVE_TUNNEL_INPUT_SEPARATOR"])
self.users.remove_tunnel(requester_name, requestee_name)
elif protocol == self.protocols["TUNNEL_STREAM"]:
username, tunnel_data = data.split(self.protocols["TUNNEL_STREAM_USERNAME_DATA_SEPARATOR"])
self.send(self.protocols["TUNNEL_STREAM"], tunnel_data, self.users.users[username]["tunneling_socket_object"])
elif protocol == self.protocols["CHANGE_USERNAME"]:
current_username, new_username = data.split(self.protocols["CHANGE_USERNAME_SEPARATOR"])
result = self.users.change_username(current_username, new_username)
self.send(self.protocols["CHANGE_USERNAME"], result, user)
# these two subsequent function are both continuation of make_tunnel initially
# based on the requestee inputs the functions depend
elif protocol == self.protocols["ACCEPTED_TUNNEL_CREATION"]:
original_requester_name, original_requestee_name = data.split(self.protocols["ORIGINAL_NAMES_SEPARATOR"])
self.users.make_forced_tunnel(original_requester_name, original_requestee_name)
self.send(self.protocols["MAKE_TUNNEL"], "True", self.users.get_socket_of_user(original_requester_name))
elif protocol == self.protocols["DECLINED_TUNNEL_CREATION"]:
original_requester = data
self.send(self.protocols["MAKE_TUNNEL"], "User declined your request", self.users.get_socket_of_user(original_requester))
def send(self, protocol, data, client_connection_object):
msg = f"{protocol}{self.protocols["PROTOCOL_MESSAGE_SPLITTER"]}{data}"
message = msg.encode(self.FORMAT)
msg_length = len(msg)
send_length = str(msg_length).encode(self.FORMAT)
send_length += b" " * (self.HEADER - len(send_length))
client_connection_object.send(send_length)
client_connection_object.send(message)
`
client- main.py
`import json
from base64 import b64encode
from io import BytesIO
from PIL import ImageGrab
from threading import Thread
import user_graphics
import user_network
class Main:
def init(self):
with open("data.json") as data_file:
data = json.load(data_file)
self.username = data["username"]
self.restriction_mode = data["restriction_mode"]
self.network = user_network.ControllerNetwork(self)
self.graphics = user_graphics.MasDController(self)
self.network.setup()
self.change_gui_data(self.username, self.restriction_mode)
Thread(target=self.connect).start()
# Globals
self.can_stream_data = False
# kivy.run() is a thread by itself so no need to make it a separate one
self.graphics.run()
######## NETWORKING #########
def local_save_user_data(self):
with open("data.json", "w") as user_data_file:
data = {
"username": self.username,
"restriction_mode": self.restriction_mode
}
json.dump(data, user_data_file, indent=4)
def protocol_check(self, protocol, data):
if protocol == self.network.protocols["ADD_USER"]:
if eval(data) == False:
self.inform("Account creation failed")
else:
self.network.logged_in = True
self.inform("Account creation successful!")
self.local_save_user_data()
self.change_gui_data(self.username, self.restriction_mode)
if protocol == self.network.protocols["MAKE_RESTRICTED"]:
self.restriction_mode = True
if eval(data) == True:
self.inform("Restricted mode set to on")
self.change_gui_data(self.username, self.restriction_mode)
else:
self.inform("Could not change the restriction mode")
self.local_save_user_data()
if protocol == self.network.protocols["MAKE_UNRESTRICTED"]:
self.restriction_mode = False
if eval(data) == True:
self.inform("Restricted mode set to off")
self.change_gui_data(self.username, self.restriction_mode)
else:
self.inform("Could not change the restriction mode")
self.local_save_user_data()
if protocol == self.network.protocols["CHANGE_USERNAME"]:
if eval(data) == False:
self.inform("Username change attempt failed")
else:
self.network.logged_in = True
self.inform("Username change attempt successful!")
self.local_save_user_data()
self.change_gui_data(self.username, self.restriction_mode)
if protocol == self.network.protocols["CHOOSE_TUNNEL_CREATION"]:
requester_name = data
self.graphics.notify("Mas D Controller", f"User {requester_name} is trying to connect")
self.graphics.open_choose_tunnel_dialog(requester_name)
if protocol == self.network.protocols["MAKE_TUNNEL"]:
if data == "True":
self.command_to_start_screen_stream()
else:
self.inform(data) # 'data' contains the request result, Example- User offline, User doesnt exist
# received a tunneled message from another user
if protocol == self.network.protocols["TUNNEL_STREAM"]:
# print("data in tunnel_stream: " + str(data))
tunneled_protocol, tunneled_data = data.split(self.network.protocols["INNER_TUNNEL_SEPARATOR"])
if tunneled_protocol == self.network.protocols["START_SCREEN_STREAM"]:
Thread(target=self.start_streaming_screen).start()
if tunneled_protocol == self.network.protocols["STOP_SCREEN_STREAM"]:
self.stop_streaming_screen()
if tunneled_protocol == self.network.protocols["SCREEN_DATA"]:
# self.show_on_screen(tunneled_data)
print(f"size of recved data: {len(tunneled_data)}")
def start_streaming_screen(self):
self.can_stream_data = True
while self.can_stream_data:
pil_image = ImageGrab.grab()
# Save the image to a BytesIO object
byte_data = BytesIO()
pil_image.save(byte_data, format='PNG')
byte_data.seek(0)
self.network.tunnel_stream_to_user(self.network.protocols["SCREEN_DATA"], byte_data.read())
byte_data.seek(0)
print(f"size of sent data: {len(byte_data.read())}")
break
def stop_streaming_screen(self):
self.can_stream_data = False
def command_to_start_screen_stream(self):
self.network.tunnel_stream_to_user(self.network.protocols["START_SCREEN_STREAM"], ":)")
def command_to_stop_screen_stream(self):
self.network.tunnel_stream_to_user(self.network.protocols["STOP_SCREEN_STREAM"], ":)")
def connect(self):
connection_status = {}
while True:
result = self.network.connect()
if result == False:
connection_status["connected"] = False
self.change_connection_status_gui(connection_status)
continue
else:
connection_status["connected"] = True
connection_status["ip"] = self.network.SERVER_IP
connection_status["port"] = self.network.SERVER_PORT
Thread(target=self.network.receive_data).start()
if self.username is not None:
self.network.login(self.username)
break
self.set_switch_mode()
self.change_connection_status_gui(connection_status)
######## GUI #########
def show_on_screen(self, pickled_image_data):
self.graphics.set_image(pickled_image_data)
def change_connection_status_gui(self, connection_status: dict):
if connection_status["connected"]:
self.graphics.main_screen.ids.connection_status_label.text = f"Connected to [b]{connection_status['ip']}[/b] on port [b]{connection_status['port']}[/b]"
else:
self.graphics.main_screen.ids.connection_status_label.text = "Not Connected!"
def inform(self, msg):
self.graphics.open_information_dialog(msg)
def accept_tunnel_creation(self, requester_name):
self.network.send(self.network.protocols["ACCEPTED_TUNNEL_CREATION"],
f"{requester_name}{self.network.protocols['ORIGINAL_NAMES_SEPARATOR']}{self.username}")
def decline_tunnel_creation(self, requester_name):
self.network.send(self.network.protocols["DECLINED_TUNNEL_CREATION"], requester_name)
def save_all_settings(self):
self.graphics.screen_manager.current = "remote_desktop"
def save_username_setting(self):
inputs = [self.graphics.settings_screen.ids.company_name_text_input.text,
self.graphics.settings_screen.ids.location_text_input.text,
self.graphics.settings_screen.ids.type_text_input.text,
self.graphics.settings_screen.ids.number_text_input.text]
username = ""
for input in inputs:
if input == "":
self.inform("Please fill all the inputs")
return
elif " " in input:
self.inform("No spaces allowed")
return
elif "t" in input:
self.inform("No tabs allowed")
return
else:
username += input
username = username.upper()
if self.username is None or self.username == "":
self.network.make_user(username)
else:
self.network.change_username(self.username, username)
self.username = username
def save_restriction_mode_setting(self):
restricted_mode: bool = self.graphics.settings_screen.ids.restriction_mode_switch.active
self.restriction_mode = restricted_mode
if restricted_mode:
self.network.make_restricted(self.username)
elif not restricted_mode:
self.network.make_unrestricted(self.username)
def change_gui_data(self, username: str, restricted_mode):
mode = "None"
if restricted_mode:
mode = "restricted"
elif not restricted_mode:
mode = "unrestricted"
self.graphics.main_screen.ids.username_label.text = f"You name [b]{str(username)}[/b] - Mode [b]{mode}[/b]"
def set_switch_mode(self):
self.graphics.settings_screen.ids.restriction_mode_switch.active = self.restriction_mode
def make_tunnel_request(self, connector_username_text):
if connector_username_text != "":
if " " not in connector_username_text:
if "t" not in connector_username_text:
if connector_username_text != self.username:
self.network.make_tunnel(self.username, connector_username_text)
else:
self.inform("Cannot enter your own username")
else:
self.inform("No tabs allowed")
else:
self.inform("No spaces allowed")
else:
self.inform("Enter a username")`
cient- user_network.py
import socket
import json
from threading import Thread
class ControllerNetwork:
def __init__(self, main):
self.main = main
self.logged_in = False
self.connected = False
def setup(self):
with open("../protocols.json") as file:
self.protocols = json.load(file)
with open("user_network_data.json") as controller_data_file:
controller_data = json.load(controller_data_file)
self.SERVER_PORT = controller_data["server_port"]
self.HEADER = controller_data["header"]
self.FORMAT = controller_data["format"]
if controller_data["server_ip"]:
self.SERVER_IP = socket.gethostbyname(socket.gethostname())
elif type(controller_data["server_ip"]) is str:
self.SERVER_IP = controller_data["server_ip"]
elif type(controller_data["server_ip"]) is not str:
self.SERVER_IP = None
def connect(self):
try:
self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.client.connect((self.SERVER_IP, self.SERVER_PORT))
self.connected = True
# self.test_code()
except ConnectionRefusedError:
return False
def receive_data(self):
while self.connected:
try:
msg_length = self.client.recv(self.HEADER).decode(self.FORMAT)
if msg_length:
msg_length = int(msg_length)
msg = self.client.recv(msg_length).decode(self.FORMAT)
protocol, data = msg.split(self.protocols["PROTOCOL_MESSAGE_SPLITTER"])
self.main.protocol_check(protocol, data)
except ConnectionResetError:
self.main.connect()
exit()
def send(self, protocol, data):
try:
msg = protocol + self.protocols["PROTOCOL_MESSAGE_SPLITTER"] + data
message = msg.encode(self.FORMAT)
msg_length = len(msg)
send_length = str(msg_length).encode(self.FORMAT)
send_length += b" " * (self.HEADER - len(send_length))
self.client.send(send_length)
self.client.send(message)
except Exception as e:
self.main.inform("Cannot send messages, not connected to a server!")
def make_user(self, username):
self.send(self.protocols["ADD_USER"], username)
def change_username(self, current_username, new_username):
self.send(self.protocols["CHANGE_USERNAME"], f"{current_username}{self.protocols['CHANGE_USERNAME_SEPARATOR']}{new_username}")
def delete_user(self, username):
self.send(self.protocols["DELETE_USER"], username)
def login(self, username):
self.send(self.protocols["LOG_IN"], username)
self.logged_in = True
def logout(self, username):
if self.logged_in:
self.send(self.protocols["LOG_OUT"], username)
# the 'connector' is the person whom 'username' connects to (aka requestee in the server code)
def make_tunnel(self, username, connector_name):
self.send(self.protocols["MAKE_TUNNEL"], f"{username}{self.protocols['MAKE_TUNNEL_INPUT_SEPARATOR']}{connector_name}")
def remove_tunnel(self, username, connector_name):
self.send(self.protocols["REMOVE_TUNNEL"], f"{username}{self.protocols['REMOVE_TUNNEL_INPUT_SEPARATOR']}{connector_name}")
def tunnel_stream_to_user(self, tunnel_protocol, tunnel_data):
data = f"{tunnel_protocol}{self.protocols['INNER_TUNNEL_SEPARATOR']}{tunnel_data}"
self.tunnel_stream(data)
def tunnel_stream(self, data):
# username, seperator, data{tunnel_protocol/_tunnel_data}
self.send(self.protocols["TUNNEL_STREAM"], f"{self.main.username}{self.protocols['TUNNEL_STREAM_USERNAME_DATA_SEPARATOR']}{data}")
def make_restricted(self, username):
self.send(self.protocols["MAKE_RESTRICTED"], username)
def make_unrestricted(self, username):
self.send(self.protocols["MAKE_UNRESTRICTED"], username)
def disconnect(self):
self.connected = False
# revision needed
self.send(self.protocols["DISCONNECT"], " ")
just a snippet of – user_graphics.py (the part responsible for setting the actual image)
@mainthread
def set_image(self, image_data):
#
# # Create a CoreImage from the BytesIO object
# core_image = Image(data, ext='png')
#
# # Update the texture on the main thread
# self.remote_desktop_screen.ids.remote_desktop_image.texture = core_image.texture
Also, there is the error of trying to type cast the msg_length into an int as it should be but instead it is in a byte format, which im suspecting is part of the original image being sent.
I tried to print the len() of when the file was originally sent by a client and what the len() of the supposedly received file is, but the two length do not match. I am suspecting this is something related to the sockets and the file size as if I were to send a normal message such as “hello world” with the protocol
“SCREEN_DATA” I get the exact same result as expected.
1