Merge pull request #17299 from Speenah/fix/hls-subtitle-timing

Fix fMP4 HLS subtitle timestamp map offset

CONTRIBUTORS.md

@@ -147,6 +147,7 @@
  - [Soumyadip Auddy](https://github.com/SoumyadipAuddy)
  - [st7105](https://github.com/st7105)
  - [sparky8251](https://github.com/sparky8251)
+ - [Speenah](https://github.com/Speenah)
  - [spookbits](https://github.com/spookbits)
  - [ssenart](https://github.com/ssenart)
  - [stanionascu](https://github.com/stanionascu)

Jellyfin.Api/Controllers/SubtitleController.cs

@@ -49,6 +49,12 @@ public class SubtitleController : BaseJellyfinApiController
     private readonly IFileSystem _fileSystem;
     private readonly ILogger<SubtitleController> _logger;
 
+    /// <summary>
+    /// The default MPEGTS value used in the WebVTT X-TIMESTAMP-MAP header, matching the legacy MPEG-TS
+    /// segment container's 10 second PTS offset.
+    /// </summary>
+    internal const long DefaultVttTimestampMapMpegts = 900000;
+
     /// <summary>
     /// Initializes a new instance of the <see cref="SubtitleController"/> class.
     /// </summary>
@@ -202,6 +208,7 @@ public class SubtitleController : BaseJellyfinApiController
     /// <param name="endPositionTicks">Optional. The end position of the subtitle in ticks.</param>
     /// <param name="copyTimestamps">Optional. Whether to copy the timestamps.</param>
     /// <param name="addVttTimeMap">Optional. Whether to add a VTT time map.</param>
+    /// <param name="vttTimestampMapMpegts">Optional. The VTT timestamp map MPEGTS value.</param>
     /// <param name="startPositionTicks">The start position of the subtitle in ticks.</param>
     /// <response code="200">File returned.</response>
     /// <returns>A <see cref="FileContentResult"/> with the subtitle file.</returns>
@@ -220,6 +227,7 @@ public class SubtitleController : BaseJellyfinApiController
         [FromQuery] long? endPositionTicks,
         [FromQuery] bool copyTimestamps = false,
         [FromQuery] bool addVttTimeMap = false,
+        [FromQuery][Range(typeof(long), "0", "9223372036854775807")] long vttTimestampMapMpegts = DefaultVttTimestampMapMpegts,
         [FromQuery] long startPositionTicks = 0)
     {
         // Set parameters to route value if not provided via query.
@@ -255,7 +263,7 @@ public class SubtitleController : BaseJellyfinApiController
 
                 var text = await reader.ReadToEndAsync().ConfigureAwait(false);
 
-                text = text.Replace("WEBVTT", "WEBVTT\nX-TIMESTAMP-MAP=MPEGTS:900000,LOCAL:00:00:00.000", StringComparison.Ordinal);
+                text = text.Replace("WEBVTT", "WEBVTT\n" + GetVttTimestampMap(vttTimestampMapMpegts), StringComparison.Ordinal);
 
                 return File(Encoding.UTF8.GetBytes(text), MimeTypes.GetMimeType("file." + format));
             }
@@ -289,6 +297,7 @@ public class SubtitleController : BaseJellyfinApiController
     /// <param name="endPositionTicks">Optional. The end position of the subtitle in ticks.</param>
     /// <param name="copyTimestamps">Optional. Whether to copy the timestamps.</param>
     /// <param name="addVttTimeMap">Optional. Whether to add a VTT time map.</param>
+    /// <param name="vttTimestampMapMpegts">Optional. The VTT timestamp map MPEGTS value.</param>
     /// <response code="200">File returned.</response>
     /// <returns>A <see cref="FileContentResult"/> with the subtitle file.</returns>
     [HttpGet("Videos/{routeItemId}/{routeMediaSourceId}/Subtitles/{routeIndex}/{routeStartPositionTicks}/Stream.{routeFormat}")]
@@ -307,7 +316,8 @@ public class SubtitleController : BaseJellyfinApiController
         [FromQuery, ParameterObsolete] string? format,
         [FromQuery] long? endPositionTicks,
         [FromQuery] bool copyTimestamps = false,
-        [FromQuery] bool addVttTimeMap = false)
+        [FromQuery] bool addVttTimeMap = false,
+        [FromQuery][Range(typeof(long), "0", "9223372036854775807")] long vttTimestampMapMpegts = DefaultVttTimestampMapMpegts)
     {
         return GetSubtitle(
             routeItemId,
@@ -321,6 +331,7 @@ public class SubtitleController : BaseJellyfinApiController
             endPositionTicks,
             copyTimestamps,
             addVttTimeMap,
+            vttTimestampMapMpegts,
             startPositionTicks ?? routeStartPositionTicks);
     }
 
@@ -331,6 +342,7 @@ public class SubtitleController : BaseJellyfinApiController
     /// <param name="index">The subtitle stream index.</param>
     /// <param name="mediaSourceId">The media source id.</param>
     /// <param name="segmentLength">The subtitle segment length.</param>
+    /// <param name="vttTimestampMapMpegts">Optional. The VTT timestamp map MPEGTS value.</param>
     /// <response code="200">Subtitle playlist retrieved.</response>
     /// <response code="404">Item not found.</response>
     /// <returns>A <see cref="FileContentResult"/> with the HLS subtitle playlist.</returns>
@@ -344,7 +356,8 @@ public class SubtitleController : BaseJellyfinApiController
         [FromRoute, Required] Guid itemId,
         [FromRoute, Required] int index,
         [FromRoute, Required] string mediaSourceId,
-        [FromQuery, Required] int segmentLength)
+        [FromQuery, Required] int segmentLength,
+        [FromQuery][Range(typeof(long), "0", "9223372036854775807")] long vttTimestampMapMpegts = DefaultVttTimestampMapMpegts)
     {
         var item = _libraryManager.GetItemById<Video>(itemId, User.GetUserId());
         if (item is null)
@@ -392,12 +405,7 @@ public class SubtitleController : BaseJellyfinApiController
 
             var endPositionTicks = Math.Min(runtime, positionTicks + segmentLengthTicks);
 
-            var url = string.Format(
-                CultureInfo.InvariantCulture,
-                "stream.vtt?CopyTimestamps=true&AddVttTimeMap=true&StartPositionTicks={0}&EndPositionTicks={1}&ApiKey={2}",
-                positionTicks.ToString(CultureInfo.InvariantCulture),
-                endPositionTicks.ToString(CultureInfo.InvariantCulture),
-                accessToken);
+            var url = GetSubtitleSegmentUrl(positionTicks, endPositionTicks, accessToken, vttTimestampMapMpegts);
 
             builder.AppendLine(url);
 
@@ -408,6 +416,38 @@ public class SubtitleController : BaseJellyfinApiController
         return File(Encoding.UTF8.GetBytes(builder.ToString()), MimeTypes.GetMimeType("playlist.m3u8"));
     }
 
+    /// <summary>
+    /// Builds the WebVTT X-TIMESTAMP-MAP header for the given MPEGTS offset.
+    /// </summary>
+    /// <param name="mpegTimestamp">The MPEGTS offset to embed.</param>
+    /// <returns>The X-TIMESTAMP-MAP header line.</returns>
+    internal static string GetVttTimestampMap(long mpegTimestamp)
+    {
+        return string.Format(
+            CultureInfo.InvariantCulture,
+            "X-TIMESTAMP-MAP=MPEGTS:{0},LOCAL:00:00:00.000",
+            mpegTimestamp);
+    }
+
+    /// <summary>
+    /// Builds the relative URL for a single WebVTT subtitle segment referenced from a subtitle HLS playlist.
+    /// </summary>
+    /// <param name="positionTicks">The segment start position, in ticks.</param>
+    /// <param name="endPositionTicks">The segment end position, in ticks.</param>
+    /// <param name="accessToken">The access token to include on the segment request.</param>
+    /// <param name="vttTimestampMapMpegts">The MPEGTS offset to request for this segment.</param>
+    /// <returns>The relative stream.vtt segment URL.</returns>
+    internal static string GetSubtitleSegmentUrl(long positionTicks, long endPositionTicks, string? accessToken, long vttTimestampMapMpegts)
+    {
+        return string.Format(
+            CultureInfo.InvariantCulture,
+            "stream.vtt?CopyTimestamps=true&AddVttTimeMap=true&StartPositionTicks={0}&EndPositionTicks={1}&ApiKey={2}&VttTimestampMapMpegts={3}",
+            positionTicks.ToString(CultureInfo.InvariantCulture),
+            endPositionTicks.ToString(CultureInfo.InvariantCulture),
+            accessToken,
+            vttTimestampMapMpegts.ToString(CultureInfo.InvariantCulture));
+    }
+
     /// <summary>
     /// Upload an external subtitle file.
     /// </summary>

Jellyfin.Api/Helpers/DynamicHlsHelper.cs

@@ -7,6 +7,7 @@ using System.Security.Claims;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using Jellyfin.Api.Controllers;
 using Jellyfin.Api.Extensions;
 using Jellyfin.Data.Enums;
 using Jellyfin.Database.Implementations.Entities;
@@ -691,11 +692,12 @@ public class DynamicHlsHelper
 
             var url = string.Format(
                 CultureInfo.InvariantCulture,
-                "{0}/Subtitles/{1}/subtitles.m3u8?SegmentLength={2}&ApiKey={3}",
+                "{0}/Subtitles/{1}/subtitles.m3u8?SegmentLength={2}&ApiKey={3}&VttTimestampMapMpegts={4}",
                 state.Request.MediaSourceId,
                 stream.Index.ToString(CultureInfo.InvariantCulture),
                 30.ToString(CultureInfo.InvariantCulture),
-                user.GetToken());
+                user.GetToken(),
+                GetVttTimestampMapMpegts(state.Request.SegmentContainer).ToString(CultureInfo.InvariantCulture));
 
             var line = string.Format(
                 CultureInfo.InvariantCulture,
@@ -710,6 +712,20 @@ public class DynamicHlsHelper
         }
     }
 
+    /// <summary>
+    /// Gets the WebVTT X-TIMESTAMP-MAP MPEGTS offset appropriate for the given HLS segment container.
+    /// fMP4 segments start at PTS 0, unlike legacy MPEG-TS segments which start at the historical 10 second offset.
+    /// </summary>
+    /// <param name="segmentContainer">The HLS segment container, e.g. "ts" or "mp4".</param>
+    /// <returns>The MPEGTS offset to embed in the WebVTT X-TIMESTAMP-MAP header.</returns>
+    internal static long GetVttTimestampMapMpegts(string? segmentContainer)
+    {
+        var segmentExtension = EncodingHelper.GetSegmentFileExtension(segmentContainer);
+        return string.Equals(segmentExtension, ".mp4", StringComparison.OrdinalIgnoreCase)
+            ? 0
+            : SubtitleController.DefaultVttTimestampMapMpegts;
+    }
+
     /// <summary>
     /// Appends EXT-X-IMAGE-STREAM-INF playlists for each available trickplay resolution.
     /// </summary>

tests/Jellyfin.Api.Tests/Controllers/SubtitleControllerTests.cs

@@ -134,4 +134,44 @@ public class SubtitleControllerTests
             fileSystem.Object,
             NullLogger<SubtitleController>.Instance);
     }
+
+    [Theory]
+    [MemberData(nameof(GetVttTimestampMap_Success_TestData))]
+    public static void GetVttTimestampMap_Success(long mpegTimestamp, string expected)
+    {
+        Assert.Equal(expected, SubtitleController.GetVttTimestampMap(mpegTimestamp));
+    }
+
+    public static TheoryData<long, string> GetVttTimestampMap_Success_TestData()
+    {
+        var data = new TheoryData<long, string>();
+        data.Add(900000, "X-TIMESTAMP-MAP=MPEGTS:900000,LOCAL:00:00:00.000");
+        data.Add(0, "X-TIMESTAMP-MAP=MPEGTS:0,LOCAL:00:00:00.000");
+        return data;
+    }
+
+    [Theory]
+    [MemberData(nameof(GetSubtitleSegmentUrl_Success_TestData))]
+    public static void GetSubtitleSegmentUrl_Success(long positionTicks, long endPositionTicks, string accessToken, long vttTimestampMapMpegts, string expected)
+    {
+        Assert.Equal(expected, SubtitleController.GetSubtitleSegmentUrl(positionTicks, endPositionTicks, accessToken, vttTimestampMapMpegts));
+    }
+
+    public static TheoryData<long, long, string, long, string> GetSubtitleSegmentUrl_Success_TestData()
+    {
+        var data = new TheoryData<long, long, string, long, string>();
+        data.Add(
+            0,
+            300000000,
+            "abc123",
+            0,
+            "stream.vtt?CopyTimestamps=true&AddVttTimeMap=true&StartPositionTicks=0&EndPositionTicks=300000000&ApiKey=abc123&VttTimestampMapMpegts=0");
+        data.Add(
+            300000000,
+            600000000,
+            "abc123",
+            900000,
+            "stream.vtt?CopyTimestamps=true&AddVttTimeMap=true&StartPositionTicks=300000000&EndPositionTicks=600000000&ApiKey=abc123&VttTimestampMapMpegts=900000");
+        return data;
+    }
 }

tests/Jellyfin.Api.Tests/Helpers/DynamicHlsHelperTests.cs

@@ -0,0 +1,26 @@
+using Jellyfin.Api.Helpers;
+using Xunit;
+
+namespace Jellyfin.Api.Tests.Helpers
+{
+    public static class DynamicHlsHelperTests
+    {
+        [Theory]
+        [MemberData(nameof(GetVttTimestampMapMpegts_Success_TestData))]
+        public static void GetVttTimestampMapMpegts_Success(string? segmentContainer, long expected)
+        {
+            Assert.Equal(expected, DynamicHlsHelper.GetVttTimestampMapMpegts(segmentContainer));
+        }
+
+        public static TheoryData<string?, long> GetVttTimestampMapMpegts_Success_TestData()
+        {
+            var data = new TheoryData<string?, long>();
+            data.Add(null, 900000);
+            data.Add(string.Empty, 900000);
+            data.Add("ts", 900000);
+            data.Add("mp4", 0);
+            data.Add("MP4", 0);
+            return data;
+        }
+    }
+}