xianglin_admin/iFileProxy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

支持任务队列

Browse source
This commit is contained in:
root 2024-11-23 22:40:57 +08:00
parent e530cb46fe
commit 2327a7ec3e
11 changed files with 259 additions and 81 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Config/AppConfig.cs
View file

@ -76,6 +76,7 @@ namespace iFileProxy.Config
public uint ThreadNum { get; set; } = 1;
public uint MaxAllowedFileSize { get; set; }
public uint MaxParallelTasks { get; set; } = 4;
public uint MaxQueueLength {get; set; } = 60;
public string Aria2cPath { get; set; } = "./bin/aria2c";
public int CacheLifetime { get; set; } = 3600;
}

12
src/Controllers/iProxyController.cs
View file

@ -32,6 +32,8 @@ namespace iFileProxy.Controllers
TaskAddState.ErrIPForbidden => (ActionResult<CommonRsp>)Ok(new CommonRsp() { retcode = (int)TaskAddState.ErrIPForbidden, message = "请求次数超过限制!" }),
TaskAddState.ErrTargetHostForbidden => (ActionResult<CommonRsp>)Ok(new CommonRsp() { retcode = (int)TaskAddState.ErrTargetHostForbidden, message = "目标主机不在服务白名单内!" }),
TaskAddState.ErrGetFileInfo => (ActionResult<CommonRsp>)Ok(new CommonRsp() { retcode = (int)TaskAddState.ErrGetFileInfo, message = "目标文件信息获取失败!" }),
TaskAddState.ErrQueueLengthLimit => (ActionResult<CommonRsp>)Ok(new CommonRsp() { retcode = (int)TaskAddState.ErrQueueLengthLimit, message = "服务器任务队列已满 请稍候重试!" }),
TaskAddState.Pending => (ActionResult<CommonRsp>)Ok(new CommonRsp() { retcode = (int)TaskAddState.Pending, message = "已经添加到任务队列!" }),
_ => (ActionResult<CommonRsp>)Ok(new CommonRsp() { retcode = (int)TaskAddState.Success, message = "succ default" }),
};
@ -42,7 +44,15 @@ namespace iFileProxy.Controllers
[Route("/GetMyTasks")]
public ActionResult<CommonRsp> GetMyTasks()
{
return Ok(new CommonRsp() { retcode = 0, data = taskManager.GetTaskListByIpAddr(HttpContext),message = "succ" });
var data = taskManager.GetTaskListByIpAddr(HttpContext);
foreach (var d in data)
{
if (d.Status == TaskState.Queuing)
{
d.QueuePosition = taskManager.GetQueuePosition(d.TaskId);
}
}
return Ok(new CommonRsp() { retcode = 0, data = data ,message = "succ" });
}
[HttpGet]

2
src/Helpers/MasterHelper.cs
View file

@ -88,7 +88,5 @@ namespace iFileProxy.Helpers
return null;
}
}
}

4
src/Models/Db.cs
View file

@ -48,6 +48,10 @@ namespace iFileProxy.Models
[JsonProperty("tag")]
[JsonPropertyName("tag")]
public string Tag { get; set; }
[JsonProperty("queue_position")]
[JsonPropertyName("queue_position")]
public int QueuePosition { get; set; }
}
public class DbConfigName {

5
src/Models/Task.cs
View file

@ -7,6 +7,7 @@
End = 3, // 任务正常结束
Cached = 4, // 要下载的内容已经缓存
Cleaned =5, // 内容过期已被清理
Queuing = 6, // 正在排队
}
public enum TaskAddState {
Success = 0,
@ -20,7 +21,9 @@
ErrTargetHostForbidden = 8,
ErrFileNameForbidden = 9,
ErrIPForbidden = 10,
ErrGetFileInfo = 11
ErrGetFileInfo = 11,
Pending = 12,
ErrQueueLengthLimit = 13,
}
public enum FileHashAlgorithm
{

5
src/SerilogConfig.cs
View file

@ -2,7 +2,6 @@
{
using Serilog;
using Serilog.Events;
using Serilog.Filters;
using System.Net;
public static class SerilogConfig
@ -12,7 +11,11 @@
var filePath = Path.Combine(AppContext.BaseDirectory, $"logs/dispatch.api.log");
Log.Logger = new LoggerConfiguration()
#if RELEASE
.MinimumLevel.Information()
#else
.MinimumLevel.Debug()
#endif
.MinimumLevel.Override("Microsoft", LogEventLevel.Information)
.MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning)
.Enrich.FromLogContext()

182
src/Services/TaskManager.cs
View file

@ -1,9 +1,9 @@
using iFileProxy.Config;
using iFileProxy.Helpers;
using iFileProxy.Models;
using Org.BouncyCastle.Asn1.Tsp;
using Serilog;
using System.Diagnostics;
using System.Security.Policy;
using System.Text.Json;
namespace iFileProxy.Services
@ -13,10 +13,48 @@ namespace iFileProxy.Services
/// </summary>
public class TaskManager
{
// 定义事件
public event EventHandler<TaskInfo>? TaskCompleted;
protected virtual void OnTaskCompleted(TaskInfo taskInfo)
{
EventHandler<TaskInfo>? handler = TaskCompleted; // 创建事件的副本
handler?.Invoke(this, taskInfo);
}
// 事件处理程序：任务完成后调度下一任务
private void HandleTaskCompleted(object? sender, TaskInfo taskInfo)
{
_logger.Debug($"[TaskId: {taskInfo.TaskId}] End.");
_logger.Information($"Running Task Num: {_runningTasks.Count}");
_logger.Information($"Queue Task Num: {_pendingTasks.Count}");
// 等待队列中有内容并且当前正在运行的任务小于最大并行任务
if (_pendingTasks.Count > 0 && _runningTasks.Count < _appConfig.DownloadOptions.MaxParallelTasks)
{
lock (_taskLock) // 线程安全
{
int add_task_num = (int)(_appConfig.DownloadOptions.MaxParallelTasks - _runningTasks.Count);
if (add_task_num > _pendingTasks.Count)
add_task_num = _pendingTasks.Count;
for (int i = 0; i < add_task_num; i++) // 运行的任务中不足最大并行数并且有正在队列的task时候添加足够多的任务
{
TaskInfo nextTask = _pendingTasks.Dequeue();
Task.Run(() => StartTask(nextTask)).ConfigureAwait(false);
}
}
}
}
private readonly static Serilog.ILogger _logger = Log.Logger.ForContext<TaskManager>();
private readonly AppConfig? _appConfig = AppConfig.GetCurrConfig("iFileProxy.json");
private readonly DatabaseHelper _dbHelper;
private Dictionary<string, TaskInfo> runningTasks = [];
private Dictionary<string, TaskInfo> _runningTasks = [];
private Queue<TaskInfo> _pendingTasks = new();
private readonly object _taskLock = new();
public TaskManager()
{
_logger.Information("Initializing TaskManager...");
@ -27,6 +65,11 @@ namespace iFileProxy.Services
_logger.Fatal($"Failed to load application configuration");
Environment.Exit(1);
}
// 绑定任务完成事件的处理程序
TaskCompleted -= HandleTaskCompleted;
TaskCompleted += HandleTaskCompleted;
_logger.Information("TaskManager init succ.");
}
/// <summary>
@ -39,10 +82,18 @@ namespace iFileProxy.Services
string? clientIp = MasterHelper.GetClientIPAddr(c);
string? t_url = c.Request.Query["url"].FirstOrDefault() ?? c.Request.Form["url"].FirstOrDefault();
if (_appConfig.DownloadOptions.MaxParallelTasks != 0 && runningTasks.Count >= _appConfig.DownloadOptions.MaxParallelTasks)
return TaskAddState.ErrMaxParallelTasksLimit;
bool queue_task = false;
if (runningTasks.Where(x => x.Value.Url == t_url).Any())
// 如果当前并行任务量已经达到设定并行任务和列队上限
if (_appConfig.DownloadOptions.MaxParallelTasks != 0 && _runningTasks.Count >= _appConfig.DownloadOptions.MaxParallelTasks)
{
if (_pendingTasks.Count >= _appConfig.DownloadOptions.MaxQueueLength)
return TaskAddState.ErrMaxParallelTasksLimit;
else
queue_task = true;
}
if (_runningTasks.Values.Any(x => x.Url == t_url))
return TaskAddState.ErrUrlRepeat;
if (!MasterHelper.CheckUrlIsValid(t_url))
@ -73,6 +124,19 @@ namespace iFileProxy.Services
UpdateTime = DateTime.Now
};
// 如果是等待中任务或者列队不是空
if (queue_task || _pendingTasks.Count != 0) // 判断一下队列长度 防止被插队
{
lock(_taskLock)
if (_pendingTasks.Count >= _appConfig.DownloadOptions.MaxQueueLength)
return TaskAddState.ErrQueueLengthLimit;
_pendingTasks.Enqueue(taskInfo); // 加入等待队列
var status = AddTaskInfoToDb(taskInfo,true);
_logger.Information($"[TaskId: {taskInfo.TaskId}] Queuing...");
if (status != TaskAddState.Success) { return status; }
return TaskAddState.Pending;
}
if (MasterHelper.CheckDownloadFileExists(taskInfo.FileName))
{
@ -85,50 +149,77 @@ namespace iFileProxy.Services
taskInfo.Tag = $"REDIRECT:{r.TaskId}";
}
}
return AddTaskInfoToDb(taskInfo);
}
public TaskAddState AddTaskInfoToDb(TaskInfo taskInfo, bool queuing = false)
{
if (_dbHelper.InsertTaskData(taskInfo))
{
if (taskInfo.Status != TaskState.Cached)
if (!queuing) // 如果不是正在排队的任务
{
StartTask(taskInfo);
taskInfo.Status = TaskState.Running;
_dbHelper.UpdateTaskStatus(taskInfo);
if (taskInfo.Status != TaskState.Cached)
{
StartTask(taskInfo);
taskInfo.Status = TaskState.Running;
}
_logger.Debug($"[TaskId: {taskInfo.TaskId}] Add to Database Successful.");
}
_logger.Debug("任务添加成功.");
else
{
taskInfo.Status = TaskState.Queuing;
}
_dbHelper.UpdateTaskStatus(taskInfo);
return TaskAddState.Success;
}
else
return TaskAddState.ErrDbFail;
}
public async void StartTask(TaskInfo taskInfo)
public async Task StartTask(TaskInfo taskInfo)
{
if (runningTasks.ContainsKey(taskInfo.TaskId))
if (_runningTasks.ContainsKey(taskInfo.TaskId))
{
_logger.Error($"指定的task已经存在!!!");
return;
}
Process aria2c = new();
aria2c.StartInfo = new ProcessStartInfo
Process aria2c = new()
{
FileName = _appConfig.DownloadOptions.Aria2cPath,
WorkingDirectory = _appConfig.DownloadOptions.SavePath,
Arguments = $"-x {_appConfig.DownloadOptions.ThreadNum} -s {_appConfig.DownloadOptions.ThreadNum} {taskInfo.Url}",
RedirectStandardOutput = true,
RedirectStandardError = true,
RedirectStandardInput = true,
UseShellExecute = false,
Environment = { { "TaskId", taskInfo.TaskId } }
StartInfo = new ProcessStartInfo
{
FileName = _appConfig.DownloadOptions.Aria2cPath,
WorkingDirectory = _appConfig.DownloadOptions.SavePath,
Arguments = $"-x {_appConfig.DownloadOptions.ThreadNum} -s {_appConfig.DownloadOptions.ThreadNum} {taskInfo.Url}",
RedirectStandardOutput = true,
RedirectStandardError = true,
RedirectStandardInput = true,
UseShellExecute = false,
Environment = { { "TaskId", taskInfo.TaskId } }
}
};
try
{
_runningTasks.Add(taskInfo.TaskId, taskInfo);
aria2c.Start();
if (taskInfo.Status != TaskState.Running)
{
taskInfo.Status = TaskState.Running;
_dbHelper.UpdateTaskStatus(taskInfo);
}
_logger.Information($"[TaskId: {taskInfo.TaskId}] Started.");
aria2c.BeginOutputReadLine();
aria2c.BeginErrorReadLine();
aria2c.OutputDataReceived += Aria2c_OutputDataReceived;
aria2c.ErrorDataReceived += Aria2c_ErrorDataReceived;
runningTasks.Add(taskInfo.TaskId, taskInfo);
await aria2c.WaitForExitAsync();
runningTasks.Remove(taskInfo.TaskId);
_runningTasks.Remove(taskInfo.TaskId);
if (aria2c.ExitCode != 0)
{
@ -143,14 +234,18 @@ namespace iFileProxy.Services
taskInfo.Hash = MasterHelper.GetFileHash(Path.Combine(_appConfig.DownloadOptions.SavePath, taskInfo.FileName), FileHashAlgorithm.MD5);
_dbHelper.UpdateTaskHash(taskInfo);
}
// 触发任务完成事件
OnTaskCompleted(taskInfo);
}
catch (Exception)
catch (Exception ex)
{
_logger.Fatal("执行下载任务时候出现致命问题");
_logger.Fatal($"执行下载任务时候出现致命问题: {ex.Message}");
throw;
}
}
private void Aria2c_ErrorDataReceived(object sender, DataReceivedEventArgs e)
{
if (e.Data == null || e.Data.Trim() == "")
@ -179,22 +274,31 @@ namespace iFileProxy.Services
return JsonSerializer.Deserialize<List<TaskInfo>>(_dbHelper.GetTaskInfoByTid(taskId)) ?? [];
}
public Dictionary<string, TaskInfo> GetRunningTasks() => _runningTasks;
public int GetQueuePosition(string taskId)
{
int position = -1; // 默认值表示未找到
int index = 0;
Queue<TaskInfo> tempQueue = new( _pendingTasks);
while (tempQueue.Count > 0)
{
TaskInfo current = tempQueue.Dequeue();
if (current.TaskId == taskId)
{
position = index;
break;
}
index++;
}
return position;
}
//public bool DeleteTask(HttpContext c)
//{
//}
//public bool UpdateTaskStatus(HttpContext c)
//{
//}
//public List<TaskInfo> GetAllTaskInfo(HttpContext c)
//{
//}
//public TaskInfo GetTaskInfo(HttpContext c)
//{
//}
}
}

1
src/iFileProxy.json
View file

@ -18,6 +18,7 @@
"ThreadNum": 4, // 下载线程数
"MaxAllowedFileSize": 65536, // 允许代理的最大文件尺寸
"MaxParallelTasks": 4, // 同一时间最大并行任务数
"MaxQueueLength": 60, // 最大等待队列长度
"Aria2cPath": "./lib/aria2c",
"CacheLifetime": 3600 // 缓存生命周期(秒) 超出此范围的缓存文件将被删除
},

7
src/wwwroot/index.html
View file

@ -14,12 +14,13 @@
<div class="container mt-5">
<div class="row justify-content-center">
<div id="loading-mask" style="display: none;">
<div class="loading-spinner"></div>
<p class="loading-text">数据提交中，请稍等...</p>
</div>
<div class="col-md-6">
<div class="card">
<div id="from_data" class="card-body">
<div id="loading-mask" style="display: none;">
<div class="loading-spinner"><p>加载中</p></div>
</div>
<h5 class="card-title text-center">Github文件下载加速</h5>
<!-- URL 输入框 -->

105
src/wwwroot/query_download_task.html
View file

@ -30,9 +30,6 @@
.table {
width: 100%;
max-width: 100%;
margin-bottom: 1rem;
background-color: transparent;
border-collapse: collapse;
border: 1px solid #dee2e6;
border-radius: 8px;
@ -41,21 +38,21 @@
.table th,
.table td {
padding: 15px;
padding: 10px;
vertical-align: middle;
border: 1px solid #dee2e6;
text-align: center;
max-width: 256px;
/* 设置最大宽度 */
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.table thead th {
.table th {
background-color: #007bff;
color: white;
font-weight: bold;
white-space: nowrap;
}
.table td {
word-break: break-word;
}
.table tbody tr:nth-child(even) {
@ -66,40 +63,85 @@
background-color: #e2e6ea;
}
.hidden-on-small {
display: table-cell;
}
@media (max-width: 768px) {
.table th,
.table td {
font-size: 0.9em;
padding: 8px;
}
.hidden-on-small {
display: none;
}
.table td {
white-space: normal;
word-wrap: break-word;
}
.statusData {
width: 20%;
}
}
@media (max-width: 576px) {
.table th,
.table td {
font-size: 0.8em;
padding: 5px;
}
.table-responsive {
overflow-x: auto;
}
.table th,
.table td {
padding: 10px;
font-size: 0.9em;
}
@media (min-width: 1200px) {
.timeData {
width: 16%;
}
.hidden-on-small {
display: none !important;
.hashData {
width: 25%;
}
}
.containerCustom {
max-width: 95%
}
</style>
</head>
<body>
<div class="container">
<div class="container containerCustom">
<h1 class="text-center">离线下载任务管理</h1>
<div class="table-responsive">
<div id="loading-mask" style="display: none;">
<div class="loading-spinner"></div>
<p class="loading-text">数据加载中，请稍等...</p>
</div>
<table id="data-table" class="table table-striped table-bordered">
<thead>
<tr>
<th>文件名</th>
<th class="hidden-on-small">大小</th>
<th>提交时间</th>
<th>状态</th>
<th class="hidden-on-small">哈希</th>
<th style="width: 10%" class="hidden-on-small">大小</th>
<th class="timeData">提交时间</th>
<th class="timeData hidden-on-small">结束时间</th>
<th class="statusData">状态</th>
<th class="hidden-on-small hashData">哈希</th>
</tr>
</thead>
<tbody id="taskTableBody">
@ -107,11 +149,9 @@
</tbody>
</table>
<p class="more-content"><a href="/index.html">返回主页</a> | 捐赠</p>
</div>
</div>
<!-- 优先加载jq一类的三方库 -->
<script src="static/js/bootstarp/5/bootstrap.bundle.min.js" crossorigin="anonymous"></script>
<script src="static/js/jquery/2.1.4/jquery.min.js"></script>
<script src="static/js/custom/Common.js"></script>
@ -124,8 +164,9 @@
[2, "错误"],
[3, "已完成"],
[4, "已缓存"],
[5, "已被清理"]
])
[5, "已被清理"],
[6, "排队中"]
]);
data = [];
$.ajax({
type: "GET",
@ -133,18 +174,18 @@
dataType: "json",
success: function (response) {
hideLoadingMask();
if (response.retcode == 0) {
if (response.retcode === 0) {
data = response.data;
populateTable(data);
}
else
} else {
alert(response.message);
}
},
error(xhr, status, error) {
alert(error);
}
});
function formatBytes(bytes, decimals = 2) {
if (bytes === 0) return '0 Bytes';
const k = 1024;
@ -161,16 +202,16 @@
data.forEach(item => {
const row = document.createElement('tr');
row.innerHTML = `
<td>${item.status == 3 || item.status == 4 ? `<a href="/Download/${item.tid}">${item.file_name}</a>` : item.file_name}</td>
<td>${item.status === 3 || item.status === 4 ? `<a href="/Download/${item.tid}">${item.file_name}</a>` : item.file_name}</td>
<td class="hidden-on-small">${formatBytes(item.size)}</td>
<td>${item.add_time}</td>
<td>${statusStrDict.get(item.status)}</td>
<td class="hidden-on-small">${item.update_time}</td>
<td>${statusStrDict.get(item.status) + (item.status == 6 ? " #" + (item.queue_position + 1) : "")}</td>
<td class="hidden-on-small">${item.hash || 'N/A'}</td>
`;
tableBody.appendChild(row);
});
}
</script>
</body>

16
src/wwwroot/static/css/custom/Common.css
View file

@ -14,10 +14,20 @@ a {
width: 100%;
height: 100%;
background-color: rgba(255, 255, 255, 0.8);
backdrop-filter: blur(7px);
display: flex;
justify-content: center;
align-items: center;
justify-content: center; /* 水平居中 */
align-items: center; /* 垂直居中 */
flex-direction: column; /* 确保垂直方向排列，加载动画在上，文字在下 */
z-index: 1000;
text-align: center; /* 文字居中 */
}
.loading-text {
margin-top: 10px; /* 加载动画和文字之间的间距 */
font-size: 1.2em;
color: #333;
font-weight: bold;
}
.loading-spinner {
@ -31,6 +41,8 @@ a {
animation: spin 2s linear infinite;
}
@keyframes spin {
0% {
transform: rotate(0deg);