Backport pull request #18264 from jellyfin/release-12.z

Refuse to start a set-up server whose database is missing or empty

Original-merge: d3e0e9abe4

Merged-by: crobibero <cody@robibe.ro>

Backported-by: Cody Robibero <cody@robibe.ro>

Jellyfin.Server/Migrations/JellyfinMigrationService.cs

@@ -132,6 +132,14 @@ internal class JellyfinMigrationService
         }
         else
         {
+            // Only a media server start is refused. A MigrateSystem run is left to the migrations, which another database
+            // provider may rely on to initialise its database, and a restore replaces the database later, in StartServer.
+            var startupMode = startupOptions.StartupMode ?? Configuration.StartupMode.MediaServer;
+            if (startupMode == Configuration.StartupMode.MediaServer && string.IsNullOrWhiteSpace(startupOptions.RestoreArchive))
+            {
+                await EnsureExistingDatabaseAsync(appPaths, logger).ConfigureAwait(false);
+            }
+
             // migrate any existing migration.xml files
             var migrationConfigPath = Path.Join(appPaths.ConfigurationDirectoryPath, "migrations.xml");
             var migrationOptions = File.Exists(migrationConfigPath)
@@ -183,6 +191,54 @@ internal class JellyfinMigrationService
         }
     }
 
+    /// <summary>
+    /// Stops the startup of a server that has been set up before but whose database is missing or empty. Running the migrations
+    /// against such a database fails part way through, and seeding it would leave a server nobody can log in to.
+    /// </summary>
+    private async Task EnsureExistingDatabaseAsync(IApplicationPaths appPaths, ILogger logger)
+    {
+        string? problem = null;
+        var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false);
+        await using (dbContext.ConfigureAwait(false))
+        {
+            var databaseCreator = dbContext.Database.GetService<IDatabaseCreator>() as IRelationalDatabaseCreator
+                ?? throw new InvalidOperationException("Jellyfin does only support relational databases.");
+
+            // Check existence first: opening a connection to a missing SQLite database creates an empty file.
+            if (!await databaseCreator.ExistsAsync().ConfigureAwait(false))
+            {
+                problem = "the database does not exist";
+            }
+            else
+            {
+                var historyRepository = dbContext.GetService<IHistoryRepository>();
+                if (!await historyRepository.ExistsAsync().ConfigureAwait(false))
+                {
+                    problem = "the database has no migration history";
+                }
+                else if ((await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false)).Count == 0)
+                {
+                    problem = "the migration history of the database is empty";
+                }
+            }
+        }
+
+        if (problem is null)
+        {
+            return;
+        }
+
+        var message = string.Format(
+            CultureInfo.InvariantCulture,
+            "This server has been set up before (IsStartupWizardCompleted is true in {0}), but {1}. Jellyfin will not start an existing server with an empty database. "
+            + "To continue, either restore the previous database; or start over and keep this server's settings by setting IsStartupWizardCompleted to false in {0} "
+            + "(users, watch history and everything else stored in the database will not come back); or set up a new server with empty configuration and data directories.",
+            appPaths.SystemConfigurationFilePath,
+            problem);
+        logger.LogCritical("{Message}", message);
+        throw new InvalidOperationException(message);
+    }
+
     /// <summary>
     /// Runs all pending migrations of the requested stage.
     /// </summary>

tests/Jellyfin.Server.Integration.Tests/JellyfinApplicationFactory.cs

@@ -29,6 +29,7 @@ namespace Jellyfin.Server.Integration.Tests
     {
         private static readonly string _testPathRoot = Path.Combine(Path.GetTempPath(), "jellyfin-test-data");
         private readonly ConcurrentBag<IDisposable> _disposableComponents = new ConcurrentBag<IDisposable>();
+        private readonly string _webHostPathRoot;
 
         /// <summary>
         /// Initializes static members of the <see cref="JellyfinApplicationFactory"/> class.
@@ -42,6 +43,23 @@ namespace Jellyfin.Server.Integration.Tests
             StartupHelpers.PerformStaticInitialization();
         }
 
+        /// <summary>
+        /// Initializes a new instance of the <see cref="JellyfinApplicationFactory"/> class using a new temporary directory.
+        /// </summary>
+        public JellyfinApplicationFactory()
+            : this(Path.Combine(_testPathRoot, "test-host-" + Path.GetFileNameWithoutExtension(Path.GetRandomFileName())))
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="JellyfinApplicationFactory"/> class.
+        /// </summary>
+        /// <param name="webHostPathRoot">The directory the application paths are created in. Reusing it starts the same server again.</param>
+        protected JellyfinApplicationFactory(string webHostPathRoot)
+        {
+            _webHostPathRoot = webHostPathRoot;
+        }
+
         /// <inheritdoc/>
         protected override IHostBuilder CreateHostBuilder()
         {
@@ -57,7 +75,7 @@ namespace Jellyfin.Server.Integration.Tests
             var commandLineOpts = new StartupOptions();
 
             // Use a temporary directory for the application paths
-            var webHostPathRoot = Path.Combine(_testPathRoot, "test-host-" + Path.GetFileNameWithoutExtension(Path.GetRandomFileName()));
+            var webHostPathRoot = _webHostPathRoot;
             Directory.CreateDirectory(Path.Combine(webHostPathRoot, "logs"));
             Directory.CreateDirectory(Path.Combine(webHostPathRoot, "config"));
             Directory.CreateDirectory(Path.Combine(webHostPathRoot, "cache"));

tests/Jellyfin.Server.Integration.Tests/StartOverTests.cs

@@ -0,0 +1,119 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Http.Json;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Emby.Server.Implementations.Serialization;
+using Jellyfin.Api.Models.StartupDtos;
+using Jellyfin.Database.Implementations;
+using Jellyfin.Extensions.Json;
+using MediaBrowser.Common.Configuration;
+using MediaBrowser.Model.Configuration;
+using Microsoft.Data.Sqlite;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.Extensions.DependencyInjection;
+using Xunit;
+
+namespace Jellyfin.Server.Integration.Tests;
+
+/// <summary>
+/// Covers starting a server that has been set up before after its database has been deleted.
+/// </summary>
+public sealed class StartOverTests : IDisposable
+{
+    private readonly string _root = Path.Combine(Path.GetTempPath(), "jellyfin-test-data", "start-over-" + Path.GetFileNameWithoutExtension(Path.GetRandomFileName()));
+    private readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options;
+
+    [Fact]
+    public async Task DeletedDatabase_WizardReset_StartsAsNewServer()
+    {
+        var paths = await SetUpServerAndDeleteDatabaseAsync();
+        SetWizardCompleted(paths, false);
+
+        using var factory = new SameRootApplicationFactory(_root);
+        using var client = factory.CreateClient();
+
+        using var response = await client.GetAsync("/Startup/Configuration", TestContext.Current.CancellationToken);
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+
+        var dbContextFactory = factory.Services.GetRequiredService<IDbContextFactory<JellyfinDbContext>>();
+        await using var context = await dbContextFactory.CreateDbContextAsync(TestContext.Current.CancellationToken);
+        var applied = (await context.Database.GetAppliedMigrationsAsync(TestContext.Current.CancellationToken)).ToHashSet(StringComparer.Ordinal);
+        Assert.All(context.GetService<IMigrationsAssembly>().Migrations.Keys, id => Assert.Contains(id, applied));
+    }
+
+    [Fact]
+    public async Task DeletedDatabase_WizardStillCompleted_RefusesToStart()
+    {
+        var paths = await SetUpServerAndDeleteDatabaseAsync();
+        var systemConfiguration = await File.ReadAllBytesAsync(paths.SystemConfigurationFilePath, TestContext.Current.CancellationToken);
+
+        using var factory = new SameRootApplicationFactory(_root);
+        var exception = Assert.ThrowsAny<Exception>(() => factory.CreateClient());
+
+        var guard = exception as InvalidOperationException ?? exception.InnerException as InvalidOperationException;
+        Assert.NotNull(guard);
+        Assert.Contains("the database does not exist", guard.Message, StringComparison.Ordinal);
+        Assert.Empty(Directory.GetFiles(paths.DataPath, "jellyfin.db*"));
+        Assert.Equal(systemConfiguration, await File.ReadAllBytesAsync(paths.SystemConfigurationFilePath, TestContext.Current.CancellationToken));
+    }
+
+    public void Dispose()
+    {
+        SqliteConnection.ClearAllPools();
+        try
+        {
+            Directory.Delete(_root, true);
+        }
+        catch (IOException)
+        {
+            // Best effort, a locked file must not fail the test.
+        }
+    }
+
+    private static void SetWizardCompleted(IApplicationPaths paths, bool completed)
+    {
+        var serializer = new MyXmlSerializer();
+        var configuration = (ServerConfiguration)serializer.DeserializeFromFile(typeof(ServerConfiguration), paths.SystemConfigurationFilePath)!;
+        configuration.IsStartupWizardCompleted = completed;
+        serializer.SerializeToFile(configuration, paths.SystemConfigurationFilePath);
+    }
+
+    private async Task<IApplicationPaths> SetUpServerAndDeleteDatabaseAsync()
+    {
+        IApplicationPaths paths;
+        using (var factory = new SameRootApplicationFactory(_root))
+        {
+            using var client = factory.CreateClient();
+            paths = factory.Services.GetRequiredService<IApplicationPaths>();
+
+            using (var response = await client.GetAsync("/Startup/User", TestContext.Current.CancellationToken))
+            {
+                Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+            }
+
+            var user = new StartupUserDto { Name = "StartOver", Password = "StartOver" };
+            using (var response = await client.PostAsJsonAsync("/Startup/User", user, _jsonOptions, TestContext.Current.CancellationToken))
+            {
+                Assert.Equal(HttpStatusCode.NoContent, response.StatusCode);
+            }
+        }
+
+        // Completing the wizard through the API would queue a library scan that can still hold the database file when
+        // it is deleted below, so the flag is set on the stopped server instead.
+        SetWizardCompleted(paths, true);
+        SqliteConnection.ClearAllPools();
+        foreach (var file in Directory.GetFiles(paths.DataPath, "jellyfin.db*"))
+        {
+            File.Delete(file);
+        }
+
+        return paths;
+    }
+
+    private sealed class SameRootApplicationFactory(string webHostPathRoot) : JellyfinApplicationFactory(webHostPathRoot);
+}

tests/Jellyfin.Server.Tests/Migrations/JellyfinMigrationServiceTests.cs

@@ -0,0 +1,187 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Emby.Server.Implementations;
+using Emby.Server.Implementations.Configuration;
+using Emby.Server.Implementations.Serialization;
+using Jellyfin.Database.Implementations;
+using Jellyfin.Server.Implementations.DatabaseConfiguration;
+using Jellyfin.Server.Implementations.Extensions;
+using Jellyfin.Server.Migrations;
+using Jellyfin.Server.ServerSetupApp;
+using MediaBrowser.Common.Configuration;
+using MediaBrowser.Model.Configuration;
+using Microsoft.Data.Sqlite;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+using Xunit;
+
+namespace Jellyfin.Server.Tests.Migrations;
+
+/// <summary>
+/// Covers how the migration service treats the database of a server that has been set up before.
+/// </summary>
+public sealed class JellyfinMigrationServiceTests : IDisposable
+{
+    private readonly string _root;
+    private readonly ServerApplicationPaths _paths;
+    private readonly List<ServiceProvider> _serviceProviders = [];
+
+    public JellyfinMigrationServiceTests()
+    {
+        _root = Path.Combine(Path.GetTempPath(), "jellyfin-migration-service-tests", Guid.NewGuid().ToString("N"));
+        _paths = new ServerApplicationPaths(
+            Path.Combine(_root, "data"),
+            Path.Combine(_root, "log"),
+            Path.Combine(_root, "config"),
+            Path.Combine(_root, "cache"),
+            Path.Combine(_root, "web"));
+        Directory.CreateDirectory(_paths.DataPath);
+        Directory.CreateDirectory(_paths.LogDirectoryPath);
+        Directory.CreateDirectory(_paths.ConfigurationDirectoryPath);
+        Directory.CreateDirectory(_paths.CachePath);
+    }
+
+    private string DatabasePath => Path.Combine(_paths.DataPath, "jellyfin.db");
+
+    [Fact]
+    public async Task CheckFirstTimeRunOrMigration_SetUpServerWithoutDatabase_ThrowsWithoutCreatingIt()
+    {
+        WriteServerConfiguration(wizardCompleted: true);
+        var service = CreateService();
+        var systemConfiguration = await File.ReadAllBytesAsync(_paths.SystemConfigurationFilePath, TestContext.Current.CancellationToken);
+
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(() => service.CheckFirstTimeRunOrMigration(_paths, new StartupOptions()));
+
+        Assert.Contains("the database does not exist", exception.Message, StringComparison.Ordinal);
+        Assert.Contains(_paths.SystemConfigurationFilePath, exception.Message, StringComparison.Ordinal);
+        Assert.Empty(Directory.GetFiles(_paths.DataPath, "jellyfin.db*"));
+        Assert.Equal(systemConfiguration, await File.ReadAllBytesAsync(_paths.SystemConfigurationFilePath, TestContext.Current.CancellationToken));
+    }
+
+    [Fact]
+    public async Task CheckFirstTimeRunOrMigration_SetUpServerWithEmptyDatabaseFile_Throws()
+    {
+        WriteServerConfiguration(wizardCompleted: true);
+        await File.WriteAllBytesAsync(DatabasePath, [], TestContext.Current.CancellationToken);
+        var service = CreateService();
+
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(() => service.CheckFirstTimeRunOrMigration(_paths, new StartupOptions()));
+
+        Assert.Contains("the database has no migration history", exception.Message, StringComparison.Ordinal);
+    }
+
+    [Fact]
+    public async Task CheckFirstTimeRunOrMigration_SetUpServerWithEmptyHistory_Throws()
+    {
+        WriteServerConfiguration(wizardCompleted: true);
+        var service = CreateService();
+        await using (var context = await CreateDbContextAsync())
+        {
+            await context.GetService<IHistoryRepository>().CreateIfNotExistsAsync(TestContext.Current.CancellationToken);
+        }
+
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(() => service.CheckFirstTimeRunOrMigration(_paths, new StartupOptions()));
+
+        Assert.Contains("the migration history of the database is empty", exception.Message, StringComparison.Ordinal);
+    }
+
+    [Fact]
+    public async Task CheckFirstTimeRunOrMigration_SetUpServerWithHistory_Passes()
+    {
+        WriteServerConfiguration(wizardCompleted: false);
+        await CreateService().CheckFirstTimeRunOrMigration(_paths, new StartupOptions());
+        var applied = await GetAppliedMigrationIdsAsync();
+        WriteServerConfiguration(wizardCompleted: true);
+
+        await CreateService().CheckFirstTimeRunOrMigration(_paths, new StartupOptions());
+
+        Assert.NotEmpty(applied);
+        Assert.Equal(applied, await GetAppliedMigrationIdsAsync());
+    }
+
+    [Fact]
+    public async Task CheckFirstTimeRunOrMigration_SeedSystemOnSetUpServerWithoutDatabase_SeedsCodeMigrations()
+    {
+        WriteServerConfiguration(wizardCompleted: true);
+
+        await CreateService().CheckFirstTimeRunOrMigration(_paths, new StartupOptions { StartupMode = Configuration.StartupMode.SeedSystem });
+
+        Assert.NotEmpty(await GetAppliedMigrationIdsAsync());
+    }
+
+    [Fact]
+    public async Task CheckFirstTimeRunOrMigration_MigrateSystemOnSetUpServerWithoutDatabase_Passes()
+    {
+        WriteServerConfiguration(wizardCompleted: true);
+
+        await CreateService().CheckFirstTimeRunOrMigration(_paths, new StartupOptions { StartupMode = Configuration.StartupMode.MigrateSystem });
+    }
+
+    [Fact]
+    public async Task CheckFirstTimeRunOrMigration_RestoreArchiveOnSetUpServerWithoutDatabase_Passes()
+    {
+        WriteServerConfiguration(wizardCompleted: true);
+
+        await CreateService().CheckFirstTimeRunOrMigration(_paths, new StartupOptions { RestoreArchive = Path.Combine(_root, "backup.zip") });
+    }
+
+    public void Dispose()
+    {
+        foreach (var serviceProvider in _serviceProviders)
+        {
+            serviceProvider.Dispose();
+        }
+
+        SqliteConnection.ClearAllPools();
+        try
+        {
+            Directory.Delete(_root, true);
+        }
+        catch (IOException)
+        {
+            // Best effort, a locked file must not fail the test.
+        }
+    }
+
+    private void WriteServerConfiguration(bool wizardCompleted)
+    {
+        new MyXmlSerializer().SerializeToFile(new ServerConfiguration { IsStartupWizardCompleted = wizardCompleted }, _paths.SystemConfigurationFilePath);
+    }
+
+    private JellyfinMigrationService CreateService()
+    {
+        var configurationManager = new ServerConfigurationManager(_paths, NullLoggerFactory.Instance, new MyXmlSerializer());
+        configurationManager.AddParts([new DatabaseConfigurationFactory()]);
+        var serviceProvider = new ServiceCollection()
+            .AddLogging()
+            .AddJellyfinDbContext(configurationManager, new ConfigurationBuilder().Build())
+            .AddSingleton<IApplicationPaths>(_paths)
+            .RegisterStartupLogger()
+            .BuildServiceProvider();
+        _serviceProviders.Add(serviceProvider);
+
+        var factory = serviceProvider.GetRequiredService<IDbContextFactory<JellyfinDbContext>>();
+        serviceProvider.GetRequiredService<IJellyfinDatabaseProvider>().DbContextFactory = factory;
+        return ActivatorUtilities.CreateInstance<JellyfinMigrationService>(serviceProvider);
+    }
+
+    private async Task<JellyfinDbContext> CreateDbContextAsync()
+    {
+        var factory = _serviceProviders[^1].GetRequiredService<IDbContextFactory<JellyfinDbContext>>();
+        return await factory.CreateDbContextAsync(TestContext.Current.CancellationToken);
+    }
+
+    private async Task<string[]> GetAppliedMigrationIdsAsync()
+    {
+        await using var context = await CreateDbContextAsync();
+        var applied = await context.Database.GetAppliedMigrationsAsync(TestContext.Current.CancellationToken);
+        return applied.Order(StringComparer.Ordinal).ToArray();
+    }
+}