add archive download

lib/network/app_dio.dart

@@ -97,6 +97,9 @@ class MyLogInterceptor implements Interceptor {
 
   @override
   void onRequest(RequestOptions options, RequestInterceptorHandler handler) {
+    Log.info("Network", "${options.method} ${options.uri}\n"
+        "headers:\n${options.headers}\n"
+        "data:\n${options.data}");
     options.connectTimeout = const Duration(seconds: 15);
     options.receiveTimeout = const Duration(seconds: 15);
     options.sendTimeout = const Duration(seconds: 15);

lib/network/download.dart

@@ -1,4 +1,5 @@
 import 'dart:async';
+import 'dart:isolate';
 
 import 'package:flutter/widgets.dart' show ChangeNotifier;
 import 'package:venera/foundation/appdata.dart';
@@ -11,13 +12,14 @@ import 'package:venera/network/images.dart';
 import 'package:venera/utils/ext.dart';
 import 'package:venera/utils/file_type.dart';
 import 'package:venera/utils/io.dart';
+import 'package:zip_flutter/zip_flutter.dart';
+
+import 'file_downloader.dart';
 
 abstract class DownloadTask with ChangeNotifier {
   /// 0-1
   double get progress;
 
-  bool get isComplete;
-
   bool get isError;
 
   bool get isPaused;
@@ -106,10 +108,7 @@ class ImagesDownloadTask extends DownloadTask with _TransferSpeedMixin {
   }
 
   @override
-  String? get cover => _cover;
-
-  @override
-  bool get isComplete => _totalCount == _downloadedCount;
+  String? get cover => _cover ?? comic?.cover;
 
   @override
   String get message => _message;
@@ -159,7 +158,8 @@ class ImagesDownloadTask extends DownloadTask with _TransferSpeedMixin {
 
   var tasks = <int, _ImageDownloadWrapper>{};
 
-  int get _maxConcurrentTasks => (appdata.settings["downloadThreads"] as num).toInt();
+  int get _maxConcurrentTasks =>
+      (appdata.settings["downloadThreads"] as num).toInt();
 
   void _scheduleTasks() {
     var images = _images![_images!.keys.elementAt(_chapter)]!;
@@ -268,7 +268,7 @@ class ImagesDownloadTask extends DownloadTask with _TransferSpeedMixin {
         var fileType = detectFileType(data);
         var file = File(FilePath.join(path!, "cover${fileType.ext}"));
         file.writeAsBytesSync(data);
-        return file.path;
+        return "file://${file.path}";
       });
       if (res.error) {
         _setError("Error: ${res.errorMessage}");
@@ -382,7 +382,7 @@ class ImagesDownloadTask extends DownloadTask with _TransferSpeedMixin {
   int get speed => currentSpeed;
 
   @override
-  String get title => comic?.title ?? comicTitle ??  "Loading...";
+  String get title => comic?.title ?? comicTitle ?? "Loading...";
 
   @override
   Map<String, dynamic> toJson() {
@@ -448,7 +448,7 @@ class ImagesDownloadTask extends DownloadTask with _TransferSpeedMixin {
       }).toList(),
       directory: Directory(path!).name,
       chapters: comic!.chapters,
-      cover: File(_cover!).uri.pathSegments.last,
+      cover: File(_cover!.split("file://").last).uri.pathSegments.last,
       comicType: ComicType(source.key.hashCode),
       downloadedChapters: chapters ?? [],
       createdAt: DateTime.now(),
@@ -577,7 +577,7 @@ abstract mixin class _TransferSpeedMixin {
 
   void onData(int length) {
     if (timer == null) return;
-    if(length < 0) {
+    if (length < 0) {
       return;
     }
     _bytesSinceLastSecond += length;
@@ -603,3 +603,217 @@ abstract mixin class _TransferSpeedMixin {
     _bytesSinceLastSecond = 0;
   }
 }
+
+class ArchiveDownloadTask extends DownloadTask {
+  final String archiveUrl;
+
+  final ComicDetails comic;
+
+  late ComicSource source;
+
+  /// Download comic by archive url
+  ///
+  /// Currently only support zip file and comics without chapters
+  ArchiveDownloadTask(this.archiveUrl, this.comic) {
+    source = ComicSource.find(comic.sourceKey)!;
+  }
+
+  FileDownloader? _downloader;
+
+  String _message = "Fetching comic info...";
+
+  bool _isRunning = false;
+
+  bool _isError = false;
+
+  void _setError(String message) {
+    _isRunning = false;
+    _isError = true;
+    _message = message;
+    notifyListeners();
+    Log.error("Download", message);
+  }
+
+  @override
+  void cancel() async {
+    _isRunning = false;
+    await _downloader?.stop();
+    if (path != null) {
+      Directory(path!).deleteIgnoreError(recursive: true);
+    }
+    path = null;
+    LocalManager().removeTask(this);
+  }
+
+  @override
+  ComicType get comicType => ComicType(source.key.hashCode);
+
+  @override
+  String? get cover => comic.cover;
+
+  @override
+  String get id => comic.id;
+
+  @override
+  bool get isError => _isError;
+
+  @override
+  bool get isPaused => !_isRunning;
+
+  @override
+  String get message => _message;
+
+  int _currentBytes = 0;
+
+  int _expectedBytes = 0;
+
+  int _speed = 0;
+
+  @override
+  void pause() {
+    _isRunning = false;
+    _message = "Paused";
+    _downloader?.stop();
+    notifyListeners();
+  }
+
+  @override
+  double get progress =>
+      _expectedBytes == 0 ? 0 : _currentBytes / _expectedBytes;
+
+  @override
+  void resume() async {
+    if (_isRunning) {
+      return;
+    }
+    _isError = false;
+    _isRunning = true;
+    notifyListeners();
+    _message = "Downloading...";
+
+    if (path == null) {
+      var dir = await LocalManager().findValidDirectory(
+        comic.id,
+        comicType,
+        comic.title,
+      );
+      if (!(await dir.exists())) {
+        try {
+          await dir.create();
+        } catch (e) {
+          _setError("Error: $e");
+          return;
+        }
+      }
+      path = dir.path;
+    }
+
+    var resultFile = File(FilePath.join(path!, "archive.zip"));
+
+    Log.info("Download", "Downloading $archiveUrl");
+
+    _downloader = FileDownloader(archiveUrl, resultFile.path);
+
+    bool isDownloaded = false;
+
+    try {
+      await for (var status in _downloader!.start()) {
+        _currentBytes = status.downloadedBytes;
+        _expectedBytes = status.totalBytes;
+        _message =
+        "${bytesToReadableString(_currentBytes)}/${bytesToReadableString(_expectedBytes)}";
+        _speed = status.bytesPerSecond;
+        isDownloaded = status.isFinished;
+        notifyListeners();
+      }
+    }
+    catch(e) {
+      _setError("Error: $e");
+      return;
+    }
+
+    if (!_isRunning) {
+      return;
+    }
+
+    if (!isDownloaded) {
+      _setError("Error: Download failed");
+      return;
+    }
+
+    try {
+      await extractArchive(path!);
+    } catch (e) {
+      _setError("Failed to extract archive: $e");
+      return;
+    }
+
+    await resultFile.deleteIgnoreError();
+
+    LocalManager().completeTask(this);
+  }
+
+  static Future<void> extractArchive(String path) async {
+    var resultFile = FilePath.join(path, "archive.zip");
+    await Isolate.run(() {
+      ZipFile.openAndExtract(resultFile, path);
+    });
+  }
+
+  @override
+  int get speed => _speed;
+
+  @override
+  String get title => comic.title;
+
+  @override
+  Map<String, dynamic> toJson() {
+    return {
+      "type": "ArchiveDownloadTask",
+      "archiveUrl": archiveUrl,
+      "comic": comic.toJson(),
+      "path": path,
+    };
+  }
+
+  static ArchiveDownloadTask? fromJson(Map<String, dynamic> json) {
+    if (json["type"] != "ArchiveDownloadTask") {
+      return null;
+    }
+    return ArchiveDownloadTask(
+      json["archiveUrl"],
+      ComicDetails.fromJson(json["comic"]),
+    )..path = json["path"];
+  }
+
+  String _findCover() {
+    var files = Directory(path!).listSync();
+    for (var f in files) {
+      if (f.name.startsWith('cover')) {
+        return f.name;
+      }
+    }
+    files.sort((a, b) {
+      return a.name.compareTo(b.name);
+    });
+    return files.first.name;
+  }
+
+  @override
+  LocalComic toLocalComic() {
+    return LocalComic(
+      id: comic.id,
+      title: title,
+      subtitle: comic.subTitle ?? '',
+      tags: comic.tags.entries.expand((e) {
+        return e.value.map((v) => "${e.key}:$v");
+      }).toList(),
+      directory: Directory(path!).name,
+      chapters: null,
+      cover: _findCover(),
+      comicType: ComicType(source.key.hashCode),
+      downloadedChapters: [],
+      createdAt: DateTime.now(),
+    );
+  }
+}

lib/network/file_downloader.dart

@@ -0,0 +1,298 @@
+import 'dart:async';
+import 'dart:io';
+
+import 'package:dio/io.dart';
+import 'package:venera/network/app_dio.dart';
+import 'package:venera/utils/ext.dart';
+
+class FileDownloader {
+  final String url;
+  final String savePath;
+  final int maxConcurrent;
+
+  FileDownloader(this.url, this.savePath, {this.maxConcurrent = 4});
+
+  int _currentBytes = 0;
+
+  int _lastBytes = 0;
+
+  late int _fileSize;
+
+  final _dio = Dio();
+
+  RandomAccessFile? _file;
+
+  bool _isWriting = false;
+
+  int _kChunkSize = 16 * 1024 * 1024;
+
+  bool _canceled = false;
+
+  late List<_DownloadBlock> _blocks;
+
+  Future<void> _writeStatus() async {
+    var file = File("$savePath.download");
+    await file.writeAsString(_blocks.map((e) => e.toString()).join("\n"));
+  }
+
+  Future<void> _readStatus() async {
+    var file = File("$savePath.download");
+    if (!await file.exists()) {
+      return;
+    }
+
+    var lines = await file.readAsLines();
+    _blocks = lines.map((e) => _DownloadBlock.fromString(e)).toList();
+  }
+
+  /// create file and write empty bytes
+  Future<void> _prepareFile() async {
+    var file = File(savePath);
+    if (await file.exists()) {
+      if (file.lengthSync() == _fileSize &&
+          File("$savePath.download").existsSync()) {
+        _file = await file.open(mode: FileMode.append);
+        return;
+      } else {
+        await file.delete();
+      }
+    }
+
+    await file.create(recursive: true);
+    _file = await file.open(mode: FileMode.append);
+    await _file!.truncate(_fileSize);
+  }
+
+  Future<void> _createTasks() async {
+    var res = await _dio.head(url);
+    var length = res.headers["content-length"]?.first;
+    _fileSize = length == null ? 0 : int.parse(length);
+
+    await _prepareFile();
+
+    if (File("$savePath.download").existsSync()) {
+      await _readStatus();
+      _currentBytes = _blocks.fold<int>(0,
+          (previousValue, element) => previousValue + element.downloadedBytes);
+    } else {
+      if (_fileSize > 1024 * 1024 * 1024) {
+        _kChunkSize = 64 * 1024 * 1024;
+      } else if (_fileSize > 512 * 1024 * 1024) {
+        _kChunkSize = 32 * 1024 * 1024;
+      }
+
+      _blocks = [];
+      for (var i = 0; i < _fileSize; i += _kChunkSize) {
+        var end = i + _kChunkSize;
+        if (end > _fileSize) {
+          _blocks.add(_DownloadBlock(i, _fileSize, 0, false));
+        } else {
+          _blocks.add(_DownloadBlock(i, i + _kChunkSize, 0, false));
+        }
+      }
+    }
+  }
+
+  Stream<DownloadingStatus> start() {
+    var stream = StreamController<DownloadingStatus>();
+    _download(stream);
+    return stream.stream;
+  }
+
+  void _reportStatus(StreamController<DownloadingStatus> stream) {
+    stream.add(DownloadingStatus(_currentBytes, _fileSize, 0));
+  }
+
+  void _download(StreamController<DownloadingStatus> resultStream) async {
+    try {
+      var proxy = await AppDio.getProxy();
+      _dio.httpClientAdapter = IOHttpClientAdapter(
+        createHttpClient: () {
+          return HttpClient()
+            ..findProxy = (uri) => proxy == null ? "DIRECT" : "PROXY $proxy";
+        },
+      );
+
+      // get file size
+      await _createTasks();
+
+      if (_canceled) return;
+
+      // check if file is downloaded
+      if (_currentBytes >= _fileSize) {
+        await _file!.close();
+        _file = null;
+        _reportStatus(resultStream);
+        resultStream.close();
+        return;
+      }
+
+      _reportStatus(resultStream);
+
+      Timer.periodic(const Duration(seconds: 1), (timer) {
+        if (_canceled || _currentBytes >= _fileSize) {
+          timer.cancel();
+          return;
+        }
+        resultStream.add(DownloadingStatus(
+            _currentBytes, _fileSize, _currentBytes - _lastBytes));
+        _lastBytes = _currentBytes;
+      });
+
+      // start downloading
+      await _scheduleDownload();
+      if (_canceled) {
+        resultStream.close();
+        return;
+      }
+      await _file!.close();
+      _file = null;
+      await File("$savePath.download").delete();
+
+      // check if download is finished
+      if (_currentBytes < _fileSize) {
+        resultStream
+            .addError(Exception("Download failed: Expected $_fileSize bytes, "
+                "but only $_currentBytes bytes downloaded."));
+        resultStream.close();
+      }
+
+      resultStream.add(DownloadingStatus(_currentBytes, _fileSize, 0, true));
+      resultStream.close();
+    } catch (e, s) {
+      await _file?.close();
+      _file = null;
+      resultStream.addError(e, s);
+      resultStream.close();
+    }
+  }
+
+  Future<void> _scheduleDownload() async {
+    var tasks = <Future>[];
+    while (true) {
+      if (_canceled) return;
+      if (tasks.length >= maxConcurrent) {
+        await Future.any(tasks);
+      }
+      final block = _blocks.firstWhereOrNull((element) =>
+          !element.downloading &&
+          element.end - element.start > element.downloadedBytes);
+      if (block == null) {
+        break;
+      }
+      block.downloading = true;
+      var task = _fetchBlock(block);
+      task.then((value) => tasks.remove(task), onError: (e) {
+        if(_canceled) return;
+        throw e;
+      });
+      tasks.add(task);
+    }
+    await Future.wait(tasks);
+  }
+
+  Future<void> _fetchBlock(_DownloadBlock block) async {
+    final start = block.start;
+    final end = block.end;
+
+    if (start > _fileSize) {
+      return;
+    }
+
+    var options = Options(
+      responseType: ResponseType.stream,
+      headers: {
+        "Range": "bytes=${start + block.downloadedBytes}-${end - 1}",
+        "Accept": "*/*",
+        "Accept-Encoding": "deflate, gzip",
+      },
+      preserveHeaderCase: true,
+    );
+    var res = await _dio.get<ResponseBody>(url, options: options);
+    if (_canceled) return;
+    if (res.data == null) {
+      throw Exception("Failed to block $start-$end");
+    }
+
+    var buffer = <int>[];
+    await for (var data in res.data!.stream) {
+      if (_canceled) return;
+      buffer.addAll(data);
+      if (buffer.length > 16 * 1024) {
+        if (_isWriting) continue;
+        _currentBytes += buffer.length;
+        _isWriting = true;
+        await _file!.setPosition(start + block.downloadedBytes);
+        await _file!.writeFrom(buffer);
+        block.downloadedBytes += buffer.length;
+        buffer.clear();
+        await _writeStatus();
+        _isWriting = false;
+      }
+    }
+
+    if (buffer.isNotEmpty) {
+      while (_isWriting) {
+        await Future.delayed(const Duration(milliseconds: 10));
+      }
+      _isWriting = true;
+      _currentBytes += buffer.length;
+      await _file!.setPosition(start + block.downloadedBytes);
+      await _file!.writeFrom(buffer);
+      block.downloadedBytes += buffer.length;
+      await _writeStatus();
+      _isWriting = false;
+    }
+
+    block.downloading = false;
+  }
+
+  Future<void> stop() async {
+    _canceled = true;
+    await _file?.close();
+    _file = null;
+  }
+}
+
+class DownloadingStatus {
+  /// The current downloaded bytes
+  final int downloadedBytes;
+
+  /// The total bytes of the file
+  final int totalBytes;
+
+  /// Whether the download is finished
+  final bool isFinished;
+
+  /// The download speed in bytes per second
+  final int bytesPerSecond;
+
+  const DownloadingStatus(
+      this.downloadedBytes, this.totalBytes, this.bytesPerSecond,
+      [this.isFinished = false]);
+
+  @override
+  String toString() {
+    return "Downloaded: $downloadedBytes/$totalBytes ${isFinished ? "Finished" : ""}";
+  }
+}
+
+class _DownloadBlock {
+  final int start;
+  final int end;
+  int downloadedBytes;
+  bool downloading;
+
+  _DownloadBlock(this.start, this.end, this.downloadedBytes, this.downloading);
+
+  @override
+  String toString() {
+    return "$start-$end-$downloadedBytes";
+  }
+
+  _DownloadBlock.fromString(String str)
+      : start = int.parse(str.split("-")[0]),
+        end = int.parse(str.split("-")[1]),
+        downloadedBytes = int.parse(str.split("-")[2]),
+        downloading = false;
+}