/** * Grasscutter Tools * Copyright (C) 2023 jie65535 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published * by the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. **/ using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json; namespace GrasscutterTools.Utils { /// /// Downloads only the game data files used by the DEBUG tools page. /// The returned directory has the same layout as a local GC_Resources directory. /// internal sealed class RemoteResourceLoader : IDisposable { private static readonly string[] AllExcelFiles = { "AchievementExcelConfigData.json", "AvatarExcelConfigData.json", "CutsceneExcelConfigData.json", "DungeonExcelConfigData.json", "EquipAffixExcelConfigData.json", "GadgetExcelConfigData.json", "GatherExcelConfigData.json", "HomeWorldBgmExcelConfigData.json", "HomeWorldFurnitureExcelConfigData.json", "MainQuestExcelConfigData.json", "MaterialExcelConfigData.json", "MonsterExcelConfigData.json", "QuestExcelConfigData.json", "ReliquaryExcelConfigData.json", "ReliquarySetExcelConfigData.json", "SceneExcelConfigData.json", "SceneTagConfigData.json", "WeaponExcelConfigData.json", "WeatherExcelConfigData.json", }; private static readonly string[] FallbackTextMapFiles = { "TextMapCHS.json", "TextMap_MediumCHS.json", "TextMapCHT.json", "TextMap_MediumCHT.json", "TextMapEN.json", "TextMap_MediumEN.json", "TextMapRU_0.json", "TextMapRU_1.json", "TextMap_MediumRU_0.json", "TextMap_MediumRU_1.json", }; private readonly HttpClient _httpClient; private readonly string _host; private readonly string _projectPath; private readonly string _branch; private readonly string _tempDirectory; private bool _disposed; private sealed class RepositoryTreeItem { public string path { get; set; } public string type { get; set; } } private RemoteResourceLoader(string repositoryUrl, string branch, string accessToken) { if (!TryParseRepository(repositoryUrl, out _host, out _projectPath)) throw new ArgumentException("远程仓库必须是 GitLab 仓库地址,例如 https://gitlab.com/Dimbreath/animegamedata2。", nameof(repositoryUrl)); _branch = string.IsNullOrWhiteSpace(branch) ? "main" : branch.Trim(); _httpClient = new HttpClient { Timeout = TimeSpan.FromMinutes(5), }; _httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("GrasscutterTools/DebugResourceLoader"); if (!string.IsNullOrWhiteSpace(accessToken)) _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("PRIVATE-TOKEN", accessToken.Trim()); _tempDirectory = Path.Combine(Path.GetTempPath(), "GrasscutterTools", "remote-resources", Guid.NewGuid().ToString("N")); Directory.CreateDirectory(Path.Combine(_tempDirectory, "ExcelBinOutput")); Directory.CreateDirectory(Path.Combine(_tempDirectory, "TextMap")); } public string DirectoryPath => _tempDirectory; /// /// Creates a temporary resource tree and downloads the files needed by one tools action. /// public static RemoteResourceLoader Download(string repositoryUrl, string branch, string accessToken, bool includeAllExcelFiles, bool includeActivityFile, bool includeManualTextMap, Action reportProgress = null) { return DownloadAsync(repositoryUrl, branch, accessToken, includeAllExcelFiles, includeActivityFile, includeManualTextMap, reportProgress, CancellationToken.None) .GetAwaiter().GetResult(); } public static async Task DownloadAsync(string repositoryUrl, string branch, string accessToken, bool includeAllExcelFiles, bool includeActivityFile, bool includeManualTextMap, Action reportProgress, CancellationToken cancellationToken) { var loader = new RemoteResourceLoader(repositoryUrl, branch, accessToken); try { var requiredFiles = new List(); if (includeManualTextMap) requiredFiles.Add("ExcelBinOutput/ManualTextMapConfigData.json"); if (includeActivityFile) requiredFiles.Add("ExcelBinOutput/NewActivityExcelConfigData.json"); if (includeAllExcelFiles) { foreach (var file in AllExcelFiles) requiredFiles.Add("ExcelBinOutput/" + file); } await loader.DownloadFilesAsync(requiredFiles, required: true, 0, 35, "ExcelBinOutput", reportProgress, cancellationToken); await loader.DownloadTextMapsAsync(reportProgress, cancellationToken); return loader; } catch { loader.Dispose(); throw; } } private async Task DownloadFilesAsync(IReadOnlyCollection repositoryPaths, bool required, int progressStart, int progressEnd, string description, Action reportProgress, CancellationToken cancellationToken) { if (repositoryPaths.Count == 0) return; var completed = 0; Report(reportProgress, progressStart, $"开始下载 {description}({repositoryPaths.Count} 个文件,并发下载)"); var tasks = repositoryPaths.Select(async path => { await DownloadFileAsync(path, required, cancellationToken); var current = Interlocked.Increment(ref completed); var progress = progressStart + (progressEnd - progressStart) * current / repositoryPaths.Count; Report(reportProgress, progress, $"下载 {description} {current}/{repositoryPaths.Count}: {Path.GetFileName(path)}"); }).ToArray(); await Task.WhenAll(tasks); } private async Task DownloadTextMapsAsync(Action reportProgress, CancellationToken cancellationToken) { Report(reportProgress, 36, "查询远程 TextMap 文件列表"); var paths = await FindTextMapPathsAsync(cancellationToken); if (paths.Count == 0) { // Older mirrors do not expose the repository tree API. Try the conventional // file names in that case; missing optional variants are harmless. paths = FallbackTextMapFiles.Select(file => "TextMap/" + file).ToList(); Report(reportProgress, 37, "远程文件列表不可用,使用标准 TextMap 文件名"); } var textMapPaths = paths .Where(IsSupportedTextMap) .Distinct(StringComparer.OrdinalIgnoreCase) .ToList(); var downloaded = 0; var completed = 0; Report(reportProgress, 38, $"开始下载 TextMap({textMapPaths.Count} 个文件,并发下载)"); var tasks = textMapPaths.Select(async path => { if (await DownloadFileAsync(path, required: false, cancellationToken)) Interlocked.Increment(ref downloaded); var current = Interlocked.Increment(ref completed); var progress = 38 + 10 * current / Math.Max(1, textMapPaths.Count); Report(reportProgress, progress, $"下载 TextMap {current}/{textMapPaths.Count}: {Path.GetFileName(path)}"); }).ToArray(); await Task.WhenAll(tasks); if (downloaded == 0) throw new FileNotFoundException("远程仓库中未找到 CHS、CHT、EN 或 RU TextMap 文件。"); } private async Task> FindTextMapPathsAsync(CancellationToken cancellationToken) { var result = new List(); try { for (var page = 1; page <= 20; page++) { var url = BuildApiUrl("repository/tree") + "?path=TextMap&recursive=true&per_page=100&page=" + page + "&ref=" + Uri.EscapeDataString(_branch); using (var response = await _httpClient.GetAsync(url, cancellationToken)) { if (!response.IsSuccessStatusCode) break; var json = await response.Content.ReadAsStringAsync(); var items = JsonConvert.DeserializeObject>(json); if (items == null || items.Count == 0) break; result.AddRange(items.Where(item => item != null && string.Equals(item.type, "blob", StringComparison.OrdinalIgnoreCase)) .Select(item => item.path) .Where(path => !string.IsNullOrWhiteSpace(path))); if (items.Count < 100) break; } } } catch (OperationCanceledException) { throw; } catch { // Fall back to the stable names below. The file download will report a useful // error if the repository uses a layout that cannot be resolved. } return result; } private static bool IsSupportedTextMap(string path) { var name = Path.GetFileName(path); return Regex.IsMatch(name, @"^TextMap.*(?:CHS|CHT|EN|RU)(?:_\d+)?\.json$", RegexOptions.IgnoreCase); } private async Task DownloadFileAsync(string repositoryPath, bool required, CancellationToken cancellationToken) { var rawUrl = BuildRawUrl(repositoryPath); using (var response = await _httpClient.GetAsync(rawUrl, cancellationToken)) { if (!response.IsSuccessStatusCode) { if (required) throw new HttpRequestException($"下载远程资源失败 ({(int)response.StatusCode}): {repositoryPath}"); return false; } var localPath = Path.GetFullPath(Path.Combine(_tempDirectory, repositoryPath.Replace('/', Path.DirectorySeparatorChar))); var tempRoot = Path.GetFullPath(_tempDirectory).TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar; if (!localPath.StartsWith(tempRoot, StringComparison.OrdinalIgnoreCase)) throw new InvalidDataException("远程仓库返回了无效的资源路径。"); Directory.CreateDirectory(Path.GetDirectoryName(localPath)); File.WriteAllBytes(localPath, await response.Content.ReadAsByteArrayAsync()); return true; } } private static void Report(Action reportProgress, int value, string message) { reportProgress?.Invoke(Math.Max(0, Math.Min(100, value)), message); } private string BuildApiUrl(string endpoint) { return _host + "/api/v4/projects/" + Uri.EscapeDataString(_projectPath) + "/" + endpoint; } private string BuildRawUrl(string repositoryPath) { var escapedBranch = string.Join("/", _branch.Split('/').Select(Uri.EscapeDataString)); var escapedPath = string.Join("/", repositoryPath.Split('/').Select(Uri.EscapeDataString)); return _host + "/" + _projectPath + "/-/raw/" + escapedBranch + "/" + escapedPath; } private static bool TryParseRepository(string value, out string host, out string projectPath) { host = null; projectPath = null; if (!Uri.TryCreate(value?.Trim(), UriKind.Absolute, out var uri) || (uri.Scheme != Uri.UriSchemeHttps && uri.Scheme != Uri.UriSchemeHttp)) return false; var path = uri.AbsolutePath.Trim('/'); var separator = path.IndexOf("/-/", StringComparison.Ordinal); if (separator >= 0) path = path.Substring(0, separator); if (path.EndsWith(".git", StringComparison.OrdinalIgnoreCase)) path = path.Substring(0, path.Length - 4); if (string.IsNullOrWhiteSpace(path)) return false; host = uri.GetLeftPart(UriPartial.Authority).TrimEnd('/'); projectPath = path; return true; } public void Dispose() { if (_disposed) return; _disposed = true; _httpClient.Dispose(); try { if (Directory.Exists(_tempDirectory)) Directory.Delete(_tempDirectory, true); } catch { // Best effort cleanup. The OS will reclaim the directory eventually if a file // is still held by a failed download. } } } }