/**
* Grasscutter Tools
* Copyright (C) 2023 jie65535
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
**/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
namespace GrasscutterTools.Utils
{
///
/// Downloads only the game data files used by the DEBUG tools page.
/// The returned directory has the same layout as a local GC_Resources directory.
///
internal sealed class RemoteResourceLoader : IDisposable
{
private static readonly string[] AllExcelFiles =
{
"AchievementExcelConfigData.json",
"AvatarExcelConfigData.json",
"CutsceneExcelConfigData.json",
"DungeonExcelConfigData.json",
"EquipAffixExcelConfigData.json",
"GadgetExcelConfigData.json",
"GatherExcelConfigData.json",
"HomeWorldBgmExcelConfigData.json",
"HomeWorldFurnitureExcelConfigData.json",
"MainQuestExcelConfigData.json",
"MaterialExcelConfigData.json",
"MonsterExcelConfigData.json",
"QuestExcelConfigData.json",
"ReliquaryExcelConfigData.json",
"ReliquarySetExcelConfigData.json",
"SceneExcelConfigData.json",
"SceneTagConfigData.json",
"WeaponExcelConfigData.json",
"WeatherExcelConfigData.json",
};
private static readonly string[] FallbackTextMapFiles =
{
"TextMapCHS.json", "TextMap_MediumCHS.json",
"TextMapCHT.json", "TextMap_MediumCHT.json",
"TextMapEN.json", "TextMap_MediumEN.json",
"TextMapRU_0.json", "TextMapRU_1.json",
"TextMap_MediumRU_0.json", "TextMap_MediumRU_1.json",
};
private readonly HttpClient _httpClient;
private readonly string _host;
private readonly string _projectPath;
private readonly string _branch;
private readonly string _tempDirectory;
private bool _disposed;
private sealed class RepositoryTreeItem
{
public string path { get; set; }
public string type { get; set; }
}
private RemoteResourceLoader(string repositoryUrl, string branch, string accessToken)
{
if (!TryParseRepository(repositoryUrl, out _host, out _projectPath))
throw new ArgumentException("远程仓库必须是 GitLab 仓库地址,例如 https://gitlab.com/Dimbreath/animegamedata2。", nameof(repositoryUrl));
_branch = string.IsNullOrWhiteSpace(branch) ? "main" : branch.Trim();
_httpClient = new HttpClient
{
Timeout = TimeSpan.FromMinutes(5),
};
_httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("GrasscutterTools/DebugResourceLoader");
if (!string.IsNullOrWhiteSpace(accessToken))
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("PRIVATE-TOKEN", accessToken.Trim());
_tempDirectory = Path.Combine(Path.GetTempPath(), "GrasscutterTools", "remote-resources", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(Path.Combine(_tempDirectory, "ExcelBinOutput"));
Directory.CreateDirectory(Path.Combine(_tempDirectory, "TextMap"));
}
public string DirectoryPath => _tempDirectory;
///
/// Creates a temporary resource tree and downloads the files needed by one tools action.
///
public static RemoteResourceLoader Download(string repositoryUrl, string branch, string accessToken,
bool includeAllExcelFiles, bool includeActivityFile, bool includeManualTextMap,
Action reportProgress = null)
{
return DownloadAsync(repositoryUrl, branch, accessToken, includeAllExcelFiles,
includeActivityFile, includeManualTextMap, reportProgress, CancellationToken.None)
.GetAwaiter().GetResult();
}
public static async Task DownloadAsync(string repositoryUrl, string branch,
string accessToken, bool includeAllExcelFiles, bool includeActivityFile,
bool includeManualTextMap, Action reportProgress,
CancellationToken cancellationToken)
{
var loader = new RemoteResourceLoader(repositoryUrl, branch, accessToken);
try
{
var requiredFiles = new List();
if (includeManualTextMap)
requiredFiles.Add("ExcelBinOutput/ManualTextMapConfigData.json");
if (includeActivityFile)
requiredFiles.Add("ExcelBinOutput/NewActivityExcelConfigData.json");
if (includeAllExcelFiles)
{
foreach (var file in AllExcelFiles)
requiredFiles.Add("ExcelBinOutput/" + file);
}
await loader.DownloadFilesAsync(requiredFiles, required: true, 0, 35,
"ExcelBinOutput", reportProgress, cancellationToken);
await loader.DownloadTextMapsAsync(reportProgress, cancellationToken);
return loader;
}
catch
{
loader.Dispose();
throw;
}
}
private async Task DownloadFilesAsync(IReadOnlyCollection repositoryPaths, bool required,
int progressStart, int progressEnd, string description, Action reportProgress,
CancellationToken cancellationToken)
{
if (repositoryPaths.Count == 0)
return;
var completed = 0;
Report(reportProgress, progressStart,
$"开始下载 {description}({repositoryPaths.Count} 个文件,并发下载)");
var tasks = repositoryPaths.Select(async path =>
{
await DownloadFileAsync(path, required, cancellationToken);
var current = Interlocked.Increment(ref completed);
var progress = progressStart + (progressEnd - progressStart) * current / repositoryPaths.Count;
Report(reportProgress, progress,
$"下载 {description} {current}/{repositoryPaths.Count}: {Path.GetFileName(path)}");
}).ToArray();
await Task.WhenAll(tasks);
}
private async Task DownloadTextMapsAsync(Action reportProgress,
CancellationToken cancellationToken)
{
Report(reportProgress, 36, "查询远程 TextMap 文件列表");
var paths = await FindTextMapPathsAsync(cancellationToken);
if (paths.Count == 0)
{
// Older mirrors do not expose the repository tree API. Try the conventional
// file names in that case; missing optional variants are harmless.
paths = FallbackTextMapFiles.Select(file => "TextMap/" + file).ToList();
Report(reportProgress, 37, "远程文件列表不可用,使用标准 TextMap 文件名");
}
var textMapPaths = paths
.Where(IsSupportedTextMap)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
var downloaded = 0;
var completed = 0;
Report(reportProgress, 38, $"开始下载 TextMap({textMapPaths.Count} 个文件,并发下载)");
var tasks = textMapPaths.Select(async path =>
{
if (await DownloadFileAsync(path, required: false, cancellationToken))
Interlocked.Increment(ref downloaded);
var current = Interlocked.Increment(ref completed);
var progress = 38 + 10 * current / Math.Max(1, textMapPaths.Count);
Report(reportProgress, progress,
$"下载 TextMap {current}/{textMapPaths.Count}: {Path.GetFileName(path)}");
}).ToArray();
await Task.WhenAll(tasks);
if (downloaded == 0)
throw new FileNotFoundException("远程仓库中未找到 CHS、CHT、EN 或 RU TextMap 文件。");
}
private async Task> FindTextMapPathsAsync(CancellationToken cancellationToken)
{
var result = new List();
try
{
for (var page = 1; page <= 20; page++)
{
var url = BuildApiUrl("repository/tree") +
"?path=TextMap&recursive=true&per_page=100&page=" + page +
"&ref=" + Uri.EscapeDataString(_branch);
using (var response = await _httpClient.GetAsync(url, cancellationToken))
{
if (!response.IsSuccessStatusCode) break;
var json = await response.Content.ReadAsStringAsync();
var items = JsonConvert.DeserializeObject>(json);
if (items == null || items.Count == 0) break;
result.AddRange(items.Where(item => item != null && string.Equals(item.type, "blob", StringComparison.OrdinalIgnoreCase))
.Select(item => item.path)
.Where(path => !string.IsNullOrWhiteSpace(path)));
if (items.Count < 100) break;
}
}
}
catch (OperationCanceledException)
{
throw;
}
catch
{
// Fall back to the stable names below. The file download will report a useful
// error if the repository uses a layout that cannot be resolved.
}
return result;
}
private static bool IsSupportedTextMap(string path)
{
var name = Path.GetFileName(path);
return Regex.IsMatch(name, @"^TextMap.*(?:CHS|CHT|EN|RU)(?:_\d+)?\.json$", RegexOptions.IgnoreCase);
}
private async Task DownloadFileAsync(string repositoryPath, bool required,
CancellationToken cancellationToken)
{
var rawUrl = BuildRawUrl(repositoryPath);
using (var response = await _httpClient.GetAsync(rawUrl, cancellationToken))
{
if (!response.IsSuccessStatusCode)
{
if (required)
throw new HttpRequestException($"下载远程资源失败 ({(int)response.StatusCode}): {repositoryPath}");
return false;
}
var localPath = Path.GetFullPath(Path.Combine(_tempDirectory,
repositoryPath.Replace('/', Path.DirectorySeparatorChar)));
var tempRoot = Path.GetFullPath(_tempDirectory).TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar;
if (!localPath.StartsWith(tempRoot, StringComparison.OrdinalIgnoreCase))
throw new InvalidDataException("远程仓库返回了无效的资源路径。");
Directory.CreateDirectory(Path.GetDirectoryName(localPath));
File.WriteAllBytes(localPath, await response.Content.ReadAsByteArrayAsync());
return true;
}
}
private static void Report(Action reportProgress, int value, string message)
{
reportProgress?.Invoke(Math.Max(0, Math.Min(100, value)), message);
}
private string BuildApiUrl(string endpoint)
{
return _host + "/api/v4/projects/" + Uri.EscapeDataString(_projectPath) + "/" + endpoint;
}
private string BuildRawUrl(string repositoryPath)
{
var escapedBranch = string.Join("/", _branch.Split('/').Select(Uri.EscapeDataString));
var escapedPath = string.Join("/", repositoryPath.Split('/').Select(Uri.EscapeDataString));
return _host + "/" + _projectPath + "/-/raw/" + escapedBranch + "/" + escapedPath;
}
private static bool TryParseRepository(string value, out string host, out string projectPath)
{
host = null;
projectPath = null;
if (!Uri.TryCreate(value?.Trim(), UriKind.Absolute, out var uri) ||
(uri.Scheme != Uri.UriSchemeHttps && uri.Scheme != Uri.UriSchemeHttp))
return false;
var path = uri.AbsolutePath.Trim('/');
var separator = path.IndexOf("/-/", StringComparison.Ordinal);
if (separator >= 0)
path = path.Substring(0, separator);
if (path.EndsWith(".git", StringComparison.OrdinalIgnoreCase))
path = path.Substring(0, path.Length - 4);
if (string.IsNullOrWhiteSpace(path)) return false;
host = uri.GetLeftPart(UriPartial.Authority).TrimEnd('/');
projectPath = path;
return true;
}
public void Dispose()
{
if (_disposed) return;
_disposed = true;
_httpClient.Dispose();
try
{
if (Directory.Exists(_tempDirectory))
Directory.Delete(_tempDirectory, true);
}
catch
{
// Best effort cleanup. The OS will reclaim the directory eventually if a file
// is still held by a failed download.
}
}
}
}