I am working on adding subtitles to Windows, Android, IOS, and Mac Catalyst. I have it working for windows and android. For ios it works great. Except when I switch to full screen mode it does not work. That was expected. What I am trying to do now is get it working with full screen mode active. From what I understand is that avplayer uses a hidden method to show a full screen version of the player some how. I want to display the subtitles on top of that somehow. I am not sure how to go about that.
Here is the method as is that is used to display the text box on top of avplayer. This works as long as you don’t switch to full screen mode.
using CommunityToolkit.Maui.Core;
using CommunityToolkit.Maui.Core.Views;
using CommunityToolkit.Maui.Primitives;
using CoreFoundation;
using CoreGraphics;
using CoreMedia;
using Foundation;
using UIKit;
namespace CommunityToolkit.Maui.Extensions;
/// <summary>
/// A class that provides subtitle support for a video player.
/// </summary>
public partial class SubtitleExtensions : UIViewController
{
readonly HttpClient httpClient;
readonly UIViewController playerViewController;
readonly PlatformMediaElement? player;
readonly UILabel subtitleLabel;
List<SubtitleCue> cues;
NSObject? playerObserver;
/// <summary>
/// The SubtitleExtensions class provides a way to display subtitles on a video player.
/// </summary>
/// <param name="player"></param>
/// <param name="playerViewController"></param>
public SubtitleExtensions(PlatformMediaElement? player, UIViewController? playerViewController)
{
ArgumentNullException.ThrowIfNull(player);
ArgumentNullException.ThrowIfNull(playerViewController?.View?.Bounds);
this.playerViewController = playerViewController;
this.player = player;
cues = [];
httpClient = new HttpClient();
subtitleLabel = new UILabel
{
Frame = CalculateSubtitleFrame(playerViewController),
TextColor = UIColor.White,
TextAlignment = UITextAlignment.Center,
Font = UIFont.SystemFontOfSize(16),
Text = "",
Lines = 0,
AutoresizingMask = UIViewAutoresizing.FlexibleWidth | UIViewAutoresizing.FlexibleTopMargin | UIViewAutoresizing.FlexibleHeight | UIViewAutoresizing.FlexibleBottomMargin
};
}
/// <summary>
/// Loads the subtitles from the provided URL.
/// </summary>
/// <param name="mediaElement"></param>
public async Task LoadSubtitles(IMediaElement mediaElement)
{
string? vttContent;
try
{
vttContent = await httpClient.GetStringAsync(mediaElement.SubtitleUrl);
}
catch (Exception ex)
{
System.Diagnostics.Trace.TraceError(ex.Message);
return;
}
cues = mediaElement.SubtitleUrl switch
{
var url when url.EndsWith("srt") => SrtParser.ParseSrtContent(vttContent),
var url when url.EndsWith("vtt") => VttParser.ParseVttContent(vttContent),
_ => throw new NotSupportedException("Unsupported subtitle format"),
};
}
/// <summary>
/// Starts the subtitle display.
/// </summary>
public void StartSubtitleDisplay()
{
DispatchQueue.MainQueue.DispatchAsync(() => playerViewController.View?.AddSubview(subtitleLabel));
playerObserver = player?.AddPeriodicTimeObserver(CMTime.FromSeconds(1, 1), null, (time) =>
{
TimeSpan currentPlaybackTime = TimeSpan.FromSeconds(time.Seconds);
ArgumentNullException.ThrowIfNull(subtitleLabel);
subtitleLabel.Frame = CalculateSubtitleFrame(playerViewController);
DispatchQueue.MainQueue.DispatchAsync(() => UpdateSubtitle(currentPlaybackTime));
});
}
/// <summary>
/// Stops the subtitle display.
/// </summary>
public void StopSubtitleDisplay()
{
ArgumentNullException.ThrowIfNull(player);
if (playerObserver is not null)
{
player.RemoveTimeObserver(playerObserver);
playerObserver.Dispose();
playerObserver = null;
subtitleLabel.RemoveFromSuperview();
}
}
void UpdateSubtitle(TimeSpan currentPlaybackTime)
{
ArgumentNullException.ThrowIfNull(subtitleLabel);
ArgumentNullException.ThrowIfNull(playerViewController.View);
foreach (var cue in cues)
{
if (currentPlaybackTime >= cue.StartTime && currentPlaybackTime <= cue.EndTime)
{
subtitleLabel.Text = cue.Text;
subtitleLabel.BackgroundColor = UIColor.FromRGBA(0, 0, 0, 128);
break;
}
else
{
subtitleLabel.Text = "";
subtitleLabel.BackgroundColor = UIColor.FromRGBA(0, 0, 0, 0);
}
}
}
static CGRect CalculateSubtitleFrame(UIViewController uIViewController)
{
ArgumentNullException.ThrowIfNull(uIViewController?.View?.Bounds);
return new CGRect(0, uIViewController.View.Bounds.Height - 60, uIViewController.View.Bounds.Width, 50);
}
}
Handler:
using System.Diagnostics.CodeAnalysis;
using CommunityToolkit.Maui.Core.Views;
using CommunityToolkit.Maui.Views;
using Microsoft.Maui.Handlers;
namespace CommunityToolkit.Maui.Core.Handlers;
public partial class MediaElementHandler : ViewHandler<MediaElement, MauiMediaElement>, IDisposable
{
/// <inheritdoc/>
/// <exception cref="NullReferenceException">Thrown if <see cref="MauiContext"/> is <see langword="null"/>.</exception>
protected override MauiMediaElement CreatePlatformView()
{
if (MauiContext is null)
{
throw new InvalidOperationException($"{nameof(MauiContext)} cannot be null");
}
mediaManager ??= new(MauiContext,
VirtualView,
Dispatcher.GetForCurrentThread() ?? throw new InvalidOperationException($"{nameof(IDispatcher)} cannot be null"));
var (_, playerViewController) = mediaManager.CreatePlatformView();
if (VirtualView.TryFindParent<Page>(out var page))
{
var parentViewController = (page.Handler as PageHandler)?.ViewController;
return new(playerViewController, parentViewController);
}
return new(playerViewController, null);
}
/// <inheritdoc/>
protected override void ConnectHandler(MauiMediaElement platformView)
{
base.ConnectHandler(platformView);
}
/// <inheritdoc/>
protected override void DisconnectHandler(MauiMediaElement platformView)
{
platformView.Dispose();
Dispose();
base.DisconnectHandler(platformView);
}
}
static class ParentPage
{
/// <summary>
/// Extension method to find the Parent of <see cref="VisualElement"/>.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="child"></param>
/// <param name="parent"></param>
/// <returns></returns>
public static bool TryFindParent<T>(this VisualElement? child, [NotNullWhen(true)] out T? parent) where T : VisualElement
{
while (true)
{
if (child is null)
{
parent = null;
return false;
}
if (child.Parent is T element)
{
parent = element;
return true;
}
child = child.Parent as VisualElement;
}
}
}
Views: MauiMediaElement
using System.ComponentModel;
using AVKit;
using CommunityToolkit.Maui.Views;
using Microsoft.Maui.Controls.Platform.Compatibility;
using Microsoft.Maui.Platform;
using UIKit;
namespace CommunityToolkit.Maui.Core.Views;
/// <summary>
/// The user-interface element that represents the <see cref="MediaElement"/> on iOS and macOS.
/// </summary>
public class MauiMediaElement : UIView
{
/// <summary>
/// Initializes a new instance of the <see cref="MauiMediaElement"/> class.
/// </summary>
/// <param name="playerViewController">The <see cref="AVPlayerViewController"/> that acts as the platform media player.</param>
/// <param name="parentViewController">The <see cref="UIViewController"/> that acts as the parent for <paramref name="playerViewController"/>.</param>
/// <exception cref="NullReferenceException">Thrown when <paramref name="playerViewController"/><c>.View</c> is <see langword="null"/>.</exception>
public MauiMediaElement(AVPlayerViewController playerViewController, UIViewController? parentViewController)
{
ArgumentNullException.ThrowIfNull(playerViewController.View);
playerViewController.View.Frame = Bounds;
#if IOS16_0_OR_GREATER || MACCATALYST16_1_OR_GREATER
// On iOS 16+ and macOS 13+ the AVPlayerViewController has to be added to a parent ViewController, otherwise the transport controls won't be displayed.
var viewController = parentViewController ?? WindowStateManager.Default.GetCurrentUIViewController();
// If we don't find the viewController, assume it's not Shell and still continue, the transport controls will still be displayed
if (viewController?.View is not null)
{
// Zero out the safe area insets of the AVPlayerViewController
UIEdgeInsets insets = viewController.View.SafeAreaInsets;
playerViewController.AdditionalSafeAreaInsets =
new UIEdgeInsets(insets.Top * -1, insets.Left, insets.Bottom * -1, insets.Right);
// Add the View from the AVPlayerViewController to the parent ViewController
viewController.AddChildViewController(playerViewController);
}
#endif
AddSubview(playerViewController.View);
}
}
Views: MediaManager
Only including CTOR as the rest is not directly relevant.
/// <summary>
/// Creates the corresponding platform view of <see cref="MediaElement"/> on iOS and macOS.
/// </summary>
/// <returns>The platform native counterpart of <see cref="MediaElement"/>.</returns>
public (PlatformMediaElement Player, AVPlayerViewController PlayerViewController) CreatePlatformView()
{
Player = new();
PlayerViewController = new()
{
Player = Player
};
// Pre-initialize Volume and Muted properties to the player object
Player.Muted = MediaElement.ShouldMute;
var volumeDiff = Math.Abs(Player.Volume - MediaElement.Volume);
if (volumeDiff > 0.01)
{
Player.Volume = (float)MediaElement.Volume;
}
AddStatusObservers();
AddPlayedToEndObserver();
AddErrorObservers();
return (Player, PlayerViewController);
}
The rest of code is part of The Maui Community toolkit Media Element. I am working on adding this feature. It is an open source project hosted on github. I have been working on various parts of media element for about 6 months and have been active in adding features and fixing bugs.