< Summary - Jellyfin

Information
Class: Jellyfin.Server.Implementations.FullSystemBackup.BackupService
Assembly: Jellyfin.Server.Implementations
File(s): /srv/git/jellyfin/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs
Line coverage
16%
Covered lines: 8
Uncovered lines: 40
Coverable lines: 48
Total lines: 532
Line coverage: 16.6%
Branch coverage
0%
Covered branches: 0
Total branches: 2
Branch coverage: 0%
Method coverage

Feature is only available for sponsors

Upgrade to PRO version

Coverage history

Coverage history 0 25 50 75 100

Metrics

MethodBranch coverage Crap Score Cyclomatic complexity Line coverage
.cctor()100%210%
.ctor(...)100%11100%
ScheduleRestoreAndRestartServer(...)100%210%
TestBackupVersionCompatibility(...)0%620%
Map(...)100%210%
Map(...)100%210%
Map(...)100%210%
NormalizePathSeparator(...)100%210%

File(s)

/srv/git/jellyfin/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs

#LineLine coverage
 1using System;
 2using System.Collections.Generic;
 3using System.IO;
 4using System.IO.Compression;
 5using System.Linq;
 6using System.Text.Json;
 7using System.Text.Json.Nodes;
 8using System.Text.Json.Serialization;
 9using System.Threading;
 10using System.Threading.Tasks;
 11using Jellyfin.Database.Implementations;
 12using Jellyfin.Server.Implementations.StorageHelpers;
 13using Jellyfin.Server.Implementations.SystemBackupService;
 14using MediaBrowser.Controller;
 15using MediaBrowser.Controller.SystemBackupService;
 16using Microsoft.EntityFrameworkCore;
 17using Microsoft.EntityFrameworkCore.Infrastructure;
 18using Microsoft.EntityFrameworkCore.Migrations;
 19using Microsoft.Extensions.Hosting;
 20using Microsoft.Extensions.Logging;
 21
 22namespace Jellyfin.Server.Implementations.FullSystemBackup;
 23
 24/// <summary>
 25/// Contains methods for creating and restoring backups.
 26/// </summary>
 27public class BackupService : IBackupService
 28{
 29    private const string ManifestEntryName = "manifest.json";
 30    private readonly ILogger<BackupService> _logger;
 31    private readonly IDbContextFactory<JellyfinDbContext> _dbProvider;
 32    private readonly IServerApplicationHost _applicationHost;
 33    private readonly IServerApplicationPaths _applicationPaths;
 34    private readonly IJellyfinDatabaseProvider _jellyfinDatabaseProvider;
 35    private readonly IHostApplicationLifetime _hostApplicationLifetime;
 036    private static readonly JsonSerializerOptions _serializerSettings = new JsonSerializerOptions(JsonSerializerDefaults
 037    {
 038        AllowTrailingCommas = true,
 039        ReferenceHandler = ReferenceHandler.IgnoreCycles,
 040    };
 41
 2142    private readonly Version _backupEngineVersion = Version.Parse("0.2.0");
 43
 44    /// <summary>
 45    /// Initializes a new instance of the <see cref="BackupService"/> class.
 46    /// </summary>
 47    /// <param name="logger">A logger.</param>
 48    /// <param name="dbProvider">A Database Factory.</param>
 49    /// <param name="applicationHost">The Application host.</param>
 50    /// <param name="applicationPaths">The application paths.</param>
 51    /// <param name="jellyfinDatabaseProvider">The Jellyfin database Provider in use.</param>
 52    /// <param name="applicationLifetime">The SystemManager.</param>
 53    public BackupService(
 54        ILogger<BackupService> logger,
 55        IDbContextFactory<JellyfinDbContext> dbProvider,
 56        IServerApplicationHost applicationHost,
 57        IServerApplicationPaths applicationPaths,
 58        IJellyfinDatabaseProvider jellyfinDatabaseProvider,
 59        IHostApplicationLifetime applicationLifetime)
 60    {
 2161        _logger = logger;
 2162        _dbProvider = dbProvider;
 2163        _applicationHost = applicationHost;
 2164        _applicationPaths = applicationPaths;
 2165        _jellyfinDatabaseProvider = jellyfinDatabaseProvider;
 2166        _hostApplicationLifetime = applicationLifetime;
 2167    }
 68
 69    /// <inheritdoc/>
 70    public void ScheduleRestoreAndRestartServer(string archivePath)
 71    {
 072        _applicationHost.RestoreBackupPath = archivePath;
 073        _applicationHost.ShouldRestart = true;
 074        _applicationHost.NotifyPendingRestart();
 075        _ = Task.Run(async () =>
 076        {
 077            await Task.Delay(500).ConfigureAwait(false);
 078            _hostApplicationLifetime.StopApplication();
 079        });
 080    }
 81
 82    /// <inheritdoc/>
 83    public async Task RestoreBackupAsync(string archivePath)
 84    {
 85        _logger.LogWarning("Begin restoring system to {BackupArchive}", archivePath); // Info isn't cutting it
 86        if (!File.Exists(archivePath))
 87        {
 88            throw new FileNotFoundException($"Requested backup file '{archivePath}' does not exist.");
 89        }
 90
 91        StorageHelper.TestCommonPathsForStorageCapacity(_applicationPaths, _logger);
 92
 93        var fileStream = File.OpenRead(archivePath);
 94        await using (fileStream.ConfigureAwait(false))
 95        {
 96            using var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read, false);
 97            var zipArchiveEntry = zipArchive.GetEntry(ManifestEntryName);
 98
 99            if (zipArchiveEntry is null)
 100            {
 101                throw new NotSupportedException($"The loaded archive '{archivePath}' does not appear to be a Jellyfin ba
 102            }
 103
 104            BackupManifest? manifest;
 105            var manifestStream = zipArchiveEntry.Open();
 106            await using (manifestStream.ConfigureAwait(false))
 107            {
 108                manifest = await JsonSerializer.DeserializeAsync<BackupManifest>(manifestStream, _serializerSettings).Co
 109            }
 110
 111            if (manifest!.ServerVersion > _applicationHost.ApplicationVersion) // newer versions of Jellyfin should be a
 112            {
 113                throw new NotSupportedException($"The loaded archive '{archivePath}' is made for a newer version of Jell
 114            }
 115
 116            if (!TestBackupVersionCompatibility(manifest.BackupEngineVersion))
 117            {
 118                throw new NotSupportedException($"The loaded archive '{archivePath}' is made for a newer version of Jell
 119            }
 120
 121            void CopyDirectory(string source, string target)
 122            {
 123                var fullSourcePath = NormalizePathSeparator(Path.GetFullPath(source) + Path.DirectorySeparatorChar);
 124                var fullTargetRoot = Path.GetFullPath(target) + Path.DirectorySeparatorChar;
 125                foreach (var item in zipArchive.Entries)
 126                {
 127                    var sourcePath = NormalizePathSeparator(Path.GetFullPath(item.FullName));
 128                    var targetPath = Path.GetFullPath(Path.Combine(target, Path.GetRelativePath(source, item.FullName)))
 129
 130                    if (!sourcePath.StartsWith(fullSourcePath, StringComparison.Ordinal)
 131                        || !targetPath.StartsWith(fullTargetRoot, StringComparison.Ordinal))
 132                    {
 133                        continue;
 134                    }
 135
 136                    _logger.LogInformation("Restore and override {File}", targetPath);
 137
 138                    Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!);
 139                    item.ExtractToFile(targetPath, overwrite: true);
 140                }
 141            }
 142
 143            CopyDirectory("Config", _applicationPaths.ConfigurationDirectoryPath);
 144            CopyDirectory("Data", _applicationPaths.DataPath);
 145            CopyDirectory("Root", _applicationPaths.RootFolderPath);
 146
 147            if (manifest.Options.Database)
 148            {
 149                _logger.LogInformation("Begin restoring Database");
 150                var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false);
 151                await using (dbContext.ConfigureAwait(false))
 152                {
 153                    // restore migration history manually
 154                    var historyEntry = zipArchive.GetEntry(NormalizePathSeparator(Path.Combine("Database", $"{nameof(His
 155                    if (historyEntry is null)
 156                    {
 157                        _logger.LogInformation("No backup of the history table in archive. This is required for Jellyfin
 158                        throw new InvalidOperationException("Cannot restore backup that has no History data.");
 159                    }
 160
 161                    HistoryRow[] historyEntries;
 162                    var historyArchive = historyEntry.Open();
 163                    await using (historyArchive.ConfigureAwait(false))
 164                    {
 165                        historyEntries = await JsonSerializer.DeserializeAsync<HistoryRow[]>(historyArchive).ConfigureAw
 166                            throw new InvalidOperationException("Cannot restore backup that has no History data.");
 167                    }
 168
 169                    var historyRepository = dbContext.GetService<IHistoryRepository>();
 170                    await historyRepository.CreateIfNotExistsAsync().ConfigureAwait(false);
 171
 172                    foreach (var item in await historyRepository.GetAppliedMigrationsAsync(CancellationToken.None).Confi
 173                    {
 174                        var insertScript = historyRepository.GetDeleteScript(item.MigrationId);
 175                        await dbContext.Database.ExecuteSqlRawAsync(insertScript).ConfigureAwait(false);
 176                    }
 177
 178                    foreach (var item in historyEntries)
 179                    {
 180                        var insertScript = historyRepository.GetInsertScript(item);
 181                        await dbContext.Database.ExecuteSqlRawAsync(insertScript).ConfigureAwait(false);
 182                    }
 183
 184                    dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
 185                    var entityTypes = typeof(JellyfinDbContext).GetProperties(System.Reflection.BindingFlags.Public | Sy
 186                        .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable)))
 187                        .Select(e => (Type: e, Set: e.GetValue(dbContext) as IQueryable))
 188                        .ToArray();
 189
 190                    var tableNames = entityTypes.Select(f => dbContext.Model.FindEntityType(f.Type.PropertyType.GetGener
 191                    _logger.LogInformation("Begin purging database");
 192                    await _jellyfinDatabaseProvider.PurgeDatabase(dbContext, tableNames).ConfigureAwait(false);
 193                    _logger.LogInformation("Database Purged");
 194
 195                    foreach (var entityType in entityTypes)
 196                    {
 197                        _logger.LogInformation("Read backup of {Table}", entityType.Type.Name);
 198
 199                        var zipEntry = zipArchive.GetEntry(NormalizePathSeparator(Path.Combine("Database", $"{entityType
 200                        if (zipEntry is null)
 201                        {
 202                            _logger.LogInformation("No backup of expected table {Table} is present in backup. Continue a
 203                            continue;
 204                        }
 205
 206                        var zipEntryStream = zipEntry.Open();
 207                        await using (zipEntryStream.ConfigureAwait(false))
 208                        {
 209                            _logger.LogInformation("Restore backup of {Table}", entityType.Type.Name);
 210                            var records = 0;
 211                            await foreach (var item in JsonSerializer.DeserializeAsyncEnumerable<JsonObject>(zipEntryStr
 212                            {
 213                                var entity = item.Deserialize(entityType.Type.PropertyType.GetGenericArguments()[0]);
 214                                if (entity is null)
 215                                {
 216                                    throw new InvalidOperationException($"Cannot deserialize entity '{item}'");
 217                                }
 218
 219                                try
 220                                {
 221                                    records++;
 222                                    dbContext.Add(entity);
 223                                }
 224                                catch (Exception ex)
 225                                {
 226                                    _logger.LogError(ex, "Could not store entity {Entity} continue anyway.", item);
 227                                }
 228                            }
 229
 230                            _logger.LogInformation("Prepared to restore {Number} entries for {Table}", records, entityTy
 231                        }
 232                    }
 233
 234                    _logger.LogInformation("Try restore Database");
 235                    await dbContext.SaveChangesAsync().ConfigureAwait(false);
 236                    _logger.LogInformation("Restored database.");
 237                }
 238            }
 239
 240            _logger.LogInformation("Restored Jellyfin system from {Date}.", manifest.DateCreated);
 241        }
 242    }
 243
 244    private bool TestBackupVersionCompatibility(Version backupEngineVersion)
 245    {
 0246        if (backupEngineVersion == _backupEngineVersion)
 247        {
 0248            return true;
 249        }
 250
 0251        return false;
 252    }
 253
 254    /// <inheritdoc/>
 255    public async Task<BackupManifestDto> CreateBackupAsync(BackupOptionsDto backupOptions)
 256    {
 257        var manifest = new BackupManifest()
 258        {
 259            DateCreated = DateTime.UtcNow,
 260            ServerVersion = _applicationHost.ApplicationVersion,
 261            DatabaseTables = null!,
 262            BackupEngineVersion = _backupEngineVersion,
 263            Options = Map(backupOptions)
 264        };
 265
 266        await _jellyfinDatabaseProvider.RunScheduledOptimisation(CancellationToken.None).ConfigureAwait(false);
 267
 268        var backupFolder = Path.Combine(_applicationPaths.BackupPath);
 269
 270        if (!Directory.Exists(backupFolder))
 271        {
 272            Directory.CreateDirectory(backupFolder);
 273        }
 274
 275        var backupStorageSpace = StorageHelper.GetFreeSpaceOf(_applicationPaths.BackupPath);
 276
 277        const long FiveGigabyte = 5_368_709_115;
 278        if (backupStorageSpace.FreeSpace < FiveGigabyte)
 279        {
 280            throw new InvalidOperationException($"The backup directory '{backupStorageSpace.Path}' does not have at leas
 281        }
 282
 283        var backupPath = Path.Combine(backupFolder, $"jellyfin-backup-{manifest.DateCreated.ToLocalTime():yyyyMMddHHmmss
 284        _logger.LogInformation("Attempt to create a new backup at {BackupPath}", backupPath);
 285        var fileStream = File.OpenWrite(backupPath);
 286        await using (fileStream.ConfigureAwait(false))
 287        using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, false))
 288        {
 289            _logger.LogInformation("Start backup process.");
 290            var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false);
 291            await using (dbContext.ConfigureAwait(false))
 292            {
 293                dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
 294                static IAsyncEnumerable<object> GetValues(IQueryable dbSet)
 295                {
 296                    var method = dbSet.GetType().GetMethod(nameof(DbSet<object>.AsAsyncEnumerable))!;
 297                    var enumerable = method.Invoke(dbSet, null)!;
 298                    return (IAsyncEnumerable<object>)enumerable;
 299                }
 300
 301                // include the migration history as well
 302                var historyRepository = dbContext.GetService<IHistoryRepository>();
 303                var migrations = await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false);
 304
 305                ICollection<(Type Type, string SourceName, Func<IAsyncEnumerable<object>> ValueFactory)> entityTypes = [
 306                    .. typeof(JellyfinDbContext)
 307                    .GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance)
 308                    .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable)))
 309                    .Select(e => (Type: e.PropertyType, dbContext.Model.FindEntityType(e.PropertyType.GetGenericArgument
 310                    (Type: typeof(HistoryRow), SourceName: nameof(HistoryRow), ValueFactory: () => migrations.ToAsyncEnu
 311                ];
 312                manifest.DatabaseTables = entityTypes.Select(e => e.Type.Name).ToArray();
 313                var transaction = await dbContext.Database.BeginTransactionAsync().ConfigureAwait(false);
 314
 315                await using (transaction.ConfigureAwait(false))
 316                {
 317                    _logger.LogInformation("Begin Database backup");
 318
 319                    foreach (var entityType in entityTypes)
 320                    {
 321                        _logger.LogInformation("Begin backup of entity {Table}", entityType.SourceName);
 322                        var zipEntry = zipArchive.CreateEntry(NormalizePathSeparator(Path.Combine("Database", $"{entityT
 323                        var entities = 0;
 324                        var zipEntryStream = zipEntry.Open();
 325                        await using (zipEntryStream.ConfigureAwait(false))
 326                        {
 327                            var jsonSerializer = new Utf8JsonWriter(zipEntryStream);
 328                            await using (jsonSerializer.ConfigureAwait(false))
 329                            {
 330                                jsonSerializer.WriteStartArray();
 331
 332                                var set = entityType.ValueFactory().ConfigureAwait(false);
 333                                await foreach (var item in set.ConfigureAwait(false))
 334                                {
 335                                    entities++;
 336                                    try
 337                                    {
 338                                        JsonSerializer.SerializeToDocument(item, _serializerSettings).WriteTo(jsonSerial
 339                                    }
 340                                    catch (Exception ex)
 341                                    {
 342                                        _logger.LogError(ex, "Could not load entity {Entity}", item);
 343                                        throw;
 344                                    }
 345                                }
 346
 347                                jsonSerializer.WriteEndArray();
 348                            }
 349                        }
 350
 351                        _logger.LogInformation("backup of entity {Table} with {Number} created", entityType.Type.Name, e
 352                    }
 353                }
 354            }
 355
 356            _logger.LogInformation("Backup of folder {Table}", _applicationPaths.ConfigurationDirectoryPath);
 357            foreach (var item in Directory.EnumerateFiles(_applicationPaths.ConfigurationDirectoryPath, "*.xml", SearchO
 358              .Union(Directory.EnumerateFiles(_applicationPaths.ConfigurationDirectoryPath, "*.json", SearchOption.TopDi
 359            {
 360                zipArchive.CreateEntryFromFile(item, NormalizePathSeparator(Path.Combine("Config", Path.GetFileName(item
 361            }
 362
 363            void CopyDirectory(string source, string target, string filter = "*")
 364            {
 365                if (!Directory.Exists(source))
 366                {
 367                    return;
 368                }
 369
 370                _logger.LogInformation("Backup of folder {Table}", source);
 371
 372                foreach (var item in Directory.EnumerateFiles(source, filter, SearchOption.AllDirectories))
 373                {
 374                    zipArchive.CreateEntryFromFile(item, NormalizePathSeparator(Path.Combine(target, Path.GetRelativePat
 375                }
 376            }
 377
 378            CopyDirectory(Path.Combine(_applicationPaths.ConfigurationDirectoryPath, "users"), Path.Combine("Config", "u
 379            CopyDirectory(Path.Combine(_applicationPaths.ConfigurationDirectoryPath, "ScheduledTasks"), Path.Combine("Co
 380            CopyDirectory(Path.Combine(_applicationPaths.RootFolderPath), "Root");
 381            CopyDirectory(Path.Combine(_applicationPaths.DataPath, "collections"), Path.Combine("Data", "collections"));
 382            CopyDirectory(Path.Combine(_applicationPaths.DataPath, "playlists"), Path.Combine("Data", "playlists"));
 383            CopyDirectory(Path.Combine(_applicationPaths.DataPath, "ScheduledTasks"), Path.Combine("Data", "ScheduledTas
 384            if (backupOptions.Subtitles)
 385            {
 386                CopyDirectory(Path.Combine(_applicationPaths.DataPath, "subtitles"), Path.Combine("Data", "subtitles"));
 387            }
 388
 389            if (backupOptions.Trickplay)
 390            {
 391                CopyDirectory(Path.Combine(_applicationPaths.DataPath, "trickplay"), Path.Combine("Data", "trickplay"));
 392            }
 393
 394            if (backupOptions.Metadata)
 395            {
 396                CopyDirectory(Path.Combine(_applicationPaths.InternalMetadataPath), Path.Combine("Data", "metadata"));
 397            }
 398
 399            var manifestStream = zipArchive.CreateEntry(ManifestEntryName).Open();
 400            await using (manifestStream.ConfigureAwait(false))
 401            {
 402                await JsonSerializer.SerializeAsync(manifestStream, manifest).ConfigureAwait(false);
 403            }
 404        }
 405
 406        _logger.LogInformation("Backup created");
 407        return Map(manifest, backupPath);
 408    }
 409
 410    /// <inheritdoc/>
 411    public async Task<BackupManifestDto?> GetBackupManifest(string archivePath)
 412    {
 413        if (!File.Exists(archivePath))
 414        {
 415            return null;
 416        }
 417
 418        BackupManifest? manifest;
 419        try
 420        {
 421            manifest = await GetManifest(archivePath).ConfigureAwait(false);
 422        }
 423        catch (Exception ex)
 424        {
 425            _logger.LogError(ex, "Tried to load archive from {Path} but failed.", archivePath);
 426            return null;
 427        }
 428
 429        if (manifest is null)
 430        {
 431            return null;
 432        }
 433
 434        return Map(manifest, archivePath);
 435    }
 436
 437    /// <inheritdoc/>
 438    public async Task<BackupManifestDto[]> EnumerateBackups()
 439    {
 440        if (!Directory.Exists(_applicationPaths.BackupPath))
 441        {
 442            return [];
 443        }
 444
 445        var archives = Directory.EnumerateFiles(_applicationPaths.BackupPath, "*.zip");
 446        var manifests = new List<BackupManifestDto>();
 447        foreach (var item in archives)
 448        {
 449            try
 450            {
 451                var manifest = await GetManifest(item).ConfigureAwait(false);
 452
 453                if (manifest is null)
 454                {
 455                    continue;
 456                }
 457
 458                manifests.Add(Map(manifest, item));
 459            }
 460            catch (Exception ex)
 461            {
 462                _logger.LogError(ex, "Could not load {BackupArchive} path.", item);
 463            }
 464        }
 465
 466        return manifests.ToArray();
 467    }
 468
 469    private static async ValueTask<BackupManifest?> GetManifest(string archivePath)
 470    {
 471        var archiveStream = File.OpenRead(archivePath);
 472        await using (archiveStream.ConfigureAwait(false))
 473        {
 474            using var zipStream = new ZipArchive(archiveStream, ZipArchiveMode.Read);
 475            var manifestEntry = zipStream.GetEntry(ManifestEntryName);
 476            if (manifestEntry is null)
 477            {
 478                return null;
 479            }
 480
 481            var manifestStream = manifestEntry.Open();
 482            await using (manifestStream.ConfigureAwait(false))
 483            {
 484                return await JsonSerializer.DeserializeAsync<BackupManifest>(manifestStream, _serializerSettings).Config
 485            }
 486        }
 487    }
 488
 489    private static BackupManifestDto Map(BackupManifest manifest, string path)
 490    {
 0491        return new BackupManifestDto()
 0492        {
 0493            BackupEngineVersion = manifest.BackupEngineVersion,
 0494            DateCreated = manifest.DateCreated,
 0495            ServerVersion = manifest.ServerVersion,
 0496            Path = path,
 0497            Options = Map(manifest.Options)
 0498        };
 499    }
 500
 501    private static BackupOptionsDto Map(BackupOptions options)
 502    {
 0503        return new BackupOptionsDto()
 0504        {
 0505            Metadata = options.Metadata,
 0506            Subtitles = options.Subtitles,
 0507            Trickplay = options.Trickplay,
 0508            Database = options.Database
 0509        };
 510    }
 511
 512    private static BackupOptions Map(BackupOptionsDto options)
 513    {
 0514        return new BackupOptions()
 0515        {
 0516            Metadata = options.Metadata,
 0517            Subtitles = options.Subtitles,
 0518            Trickplay = options.Trickplay,
 0519            Database = options.Database
 0520        };
 521    }
 522
 523    /// <summary>
 524    /// Windows is able to handle '/' as a path seperator in zip files
 525    /// but linux isn't able to handle '\' as a path seperator in zip files,
 526    /// So normalize to '/'.
 527    /// </summary>
 528    /// <param name="path">The path to normalize.</param>
 529    /// <returns>The normalized path. </returns>
 530    private static string NormalizePathSeparator(string path)
 0531        => path.Replace('\\', '/');
 532}