Improve image cache. Close #326

lib/foundation/app.dart

@@ -13,7 +13,7 @@ export "widget_utils.dart";
 export "context.dart";
 
 class _App {
-  final version = "1.4.0";
+  final version = "1.4.1";
 
   bool get isAndroid => Platform.isAndroid;
 

lib/foundation/cache_manager.dart

@@ -33,10 +33,13 @@ class CacheManager {
         type TEXT
       )
     ''');
-    compute((path) => Directory(path).size, cachePath)
+    compute((path) => Directory(path).size, cachePath).then((value) {
-        .then((value) => _currentSize = value);
+      _currentSize = value;
+      checkCache();
+    });
   }
 
+  /// Get the singleton instance of CacheManager.
   factory CacheManager() => instance ??= CacheManager._create();
 
   /// set cache size limit in MB
@@ -44,35 +47,14 @@ class CacheManager {
     _limitSize = size * 1024 * 1024;
   }
 
-  void setType(String key, String? type){
+  /// Write cache to disk.
-    _db.execute('''
+  Future<void> writeCache(String key, List<int> data,
-      UPDATE cache
+      [int duration = 7 * 24 * 60 * 60 * 1000]) async {
-      SET type = ?
-      WHERE key = ?
-    ''', [type, key]);
-  }
-
-  String? getType(String key){
-    var res = _db.select('''
-      SELECT type FROM cache
-      WHERE key = ?
-    ''', [key]);
-    if(res.isEmpty){
-      return null;
-    }
-    return res.first[0];
-  }
-
-  Future<void> writeCache(String key, List<int> data, [int duration = 7 * 24 * 60 * 60 * 1000]) async{
     this.dir++;
     this.dir %= 100;
     var dir = this.dir;
-    var name = md5.convert(Uint8List.fromList(key.codeUnits)).toString();
+    var name = md5.convert(key.codeUnits).toString();
     var file = File('$cachePath/$dir/$name');
-    while(await file.exists()){
-      name = md5.convert(Uint8List.fromList(name.codeUnits)).toString();
-      file = File('$cachePath/$dir/$name');
-    }
     await file.create(recursive: true);
     await file.writeAsBytes(data);
     var expires = DateTime.now().millisecondsSinceEpoch + duration;
@@ -85,20 +67,10 @@ class CacheManager {
     checkCacheIfRequired();
   }
 
-  Future<CachingFile> openWrite(String key) async{
+  /// Find cache by key.
-    this.dir++;
+  /// If cache is expired, it will be deleted and return null.
-    this.dir %= 100;
+  /// If cache is not found, it will return null.
-    var dir = this.dir;
+  /// If cache is found, it will return the file, and update the expires time.
-    var name = md5.convert(Uint8List.fromList(key.codeUnits)).toString();
-    var file = File('$cachePath/$dir/$name');
-    while(await file.exists()){
-      name = md5.convert(Uint8List.fromList(name.codeUnits)).toString();
-      file = File('$cachePath/$dir/$name');
-    }
-    await file.create(recursive: true);
-    return CachingFile._(key, dir.toString(), name, file);
-  }
-
   Future<File?> findCache(String key) async {
     var res = _db.select('''
       SELECT * FROM cache
@@ -110,21 +82,51 @@ class CacheManager {
     var row = res.first;
     var dir = row[1] as String;
     var name = row[2] as String;
+    var expires = row[3] as int;
     var file = File('$cachePath/$dir/$name');
+    var now = DateTime.now().millisecondsSinceEpoch;
+    if (expires < now) {
+      // expired
+      _db.execute('''
+        DELETE FROM cache
+        WHERE key = ?
+      ''', [key]);
       if (await file.exists()) {
+        await file.delete();
+      }
+      return null;
+    }
+    if (await file.exists()) {
+      // update time
+      var expires = now + 7 * 24 * 60 * 60 * 1000;
+      _db.execute('''
+        UPDATE cache
+        SET expires = ?
+        WHERE key = ?
+      ''', [expires, key]);
       return file;
+    } else {
+      _db.execute('''
+        DELETE FROM cache
+        WHERE key = ?
+      ''', [key]);
     }
     return null;
   }
 
   bool _isChecking = false;
 
+  /// Check cache size and delete expired cache.
+  /// Only check cache if current size is greater than limit size.
   void checkCacheIfRequired() {
     if (_currentSize != null && _currentSize! > _limitSize) {
       checkCache();
     }
   }
 
+  /// Check cache size and delete expired cache.
+  /// If current size is greater than limit size,
+  /// delete cache until current size is less than limit size.
   Future<void> checkCache() async {
     if (_isChecking) {
       return;
@@ -139,6 +141,8 @@ class CacheManager {
       var name = row[2] as String;
       var file = File('$cachePath/$dir/$name');
       if (await file.exists()) {
+        var size = await file.length();
+        _currentSize = _currentSize! - size;
         await file.delete();
       }
     }
@@ -147,15 +151,7 @@ class CacheManager {
       WHERE expires < ?
     ''', [DateTime.now().millisecondsSinceEpoch]);
 
-    int count = 0;
+    while (_currentSize != null && _currentSize! > _limitSize) {
-    var res2 = _db.select('''
-      SELECT COUNT(*) FROM cache
-    ''');
-    if(res2.isNotEmpty){
-      count = res2.first[0] as int;
-    }
-
-    while((_currentSize != null && _currentSize! > _limitSize) ||  count > 2000){
       var res = _db.select('''
         SELECT * FROM cache
         ORDER BY expires ASC
@@ -183,12 +179,12 @@ class CacheManager {
             WHERE key = ?
           ''', [key]);
         }
-        count--;
       }
     }
     _isChecking = false;
   }
 
+  /// Delete cache by key.
   Future<void> delete(String key) async {
     var res = _db.select('''
       SELECT * FROM cache
@@ -215,6 +211,7 @@ class CacheManager {
     }
   }
 
+  /// Delete all cache.
   Future<void> clear() async {
     await Directory(cachePath).delete(recursive: true);
     Directory(cachePath).createSync(recursive: true);
@@ -223,75 +220,4 @@ class CacheManager {
     ''');
     _currentSize = 0;
   }
-
-  Future<void> deleteKeyword(String keyword) async{
-    var res = _db.select('''
-      SELECT * FROM cache
-      WHERE key LIKE ?
-    ''', ['%$keyword%']);
-    for(var row in res){
-      var key = row[0] as String;
-      var dir = row[1] as String;
-      var name = row[2] as String;
-      var file = File('$cachePath/$dir/$name');
-      var fileSize = 0;
-      if(await file.exists()){
-        fileSize = await file.length();
-        try {
-          await file.delete();
-        }
-        finally {}
-      }
-      _db.execute('''
-        DELETE FROM cache
-        WHERE key = ?
-      ''', [key]);
-      if(_currentSize != null) {
-        _currentSize = _currentSize! - fileSize;
-      }
-    }
-  }
-}
-
-class CachingFile{
-  CachingFile._(this.key, this.dir, this.name, this.file);
-
-  final String key;
-
-  final String dir;
-
-  final String name;
-
-  final File file;
-
-  final List<int> _buffer = [];
-
-  Future<void> writeBytes(List<int> data) async{
-    _buffer.addAll(data);
-    if(_buffer.length > 1024 * 1024){
-      await file.writeAsBytes(_buffer, mode: FileMode.append);
-      _buffer.clear();
-    }
-  }
-
-  Future<void> close() async{
-    if(_buffer.isNotEmpty){
-      await file.writeAsBytes(_buffer, mode: FileMode.append);
-    }
-    CacheManager()._db.execute('''
-      INSERT OR REPLACE INTO cache (key, dir, name, expires) VALUES (?, ?, ?, ?)
-    ''', [key, dir, name, DateTime.now().millisecondsSinceEpoch + 7 * 24 * 60 * 60 * 1000]);
-    CacheManager().checkCacheIfRequired();
-  }
-
-  Future<void> cancel() async{
-    await file.deleteIgnoreError();
-  }
-
-  void reset() {
-    _buffer.clear();
-    if(file.existsSync()) {
-      file.deleteSync();
-    }
-  }
 }

lib/network/images.dart

@@ -1,3 +1,4 @@
+import 'dart:async';
 import 'dart:typed_data';
 
 import 'package:flutter_qjs/flutter_qjs.dart';
@@ -8,7 +9,7 @@ import 'package:venera/utils/image.dart';
 
 import 'app_dio.dart';
 
-class ImageDownloader {
+abstract class ImageDownloader {
   static Stream<ImageDownloadProgress> loadThumbnail(
       String url, String? sourceKey,
       [String? cid]) async* {
@@ -82,7 +83,35 @@ class ImageDownloader {
     );
   }
 
+  static final _loadingImages = <String, _StreamWrapper<ImageDownloadProgress>>{};
+
+  /// Cancel all loading images.
+  static void cancelAllLoadingImages() {
+    for (var wrapper in _loadingImages.values) {
+      wrapper.cancel();
+    }
+    _loadingImages.clear();
+  }
+
+  /// Load a comic image from the network or cache.
+  /// The function will prevent multiple requests for the same image.
   static Stream<ImageDownloadProgress> loadComicImage(
+      String imageKey, String? sourceKey, String cid, String eid) {
+    final cacheKey = "$imageKey@$sourceKey@$cid@$eid";
+    if (_loadingImages.containsKey(cacheKey)) {
+      return _loadingImages[cacheKey]!.stream;
+    }
+    final stream = _StreamWrapper<ImageDownloadProgress>(
+      _loadComicImage(imageKey, sourceKey, cid, eid),
+      (wrapper) {
+        _loadingImages.remove(cacheKey);
+      },
+    );
+    _loadingImages[cacheKey] = stream;
+    return stream.stream;
+  }
+
+  static Stream<ImageDownloadProgress> _loadComicImage(
       String imageKey, String? sourceKey, String cid, String eid) async* {
     final cacheKey = "$imageKey@$sourceKey@$cid@$eid";
     final cache = await CacheManager().findCache(cacheKey);
@@ -189,6 +218,63 @@ class ImageDownloader {
   }
 }
 
+/// A wrapper class for a stream that
+/// allows multiple listeners to listen to the same stream.
+class _StreamWrapper<T> {
+  final Stream<T> _stream;
+
+  final List<StreamController> controllers = [];
+
+  final void Function(_StreamWrapper<T> wrapper) onClosed;
+
+  bool isClosed = false;
+
+  _StreamWrapper(this._stream, this.onClosed) {
+    _listen();
+  }
+
+  void _listen() async {
+    await for (var data in _stream) {
+      if (isClosed) {
+        break;
+      }
+      for (var controller in controllers) {
+        if (!controller.isClosed) {
+          controller.add(data);
+        }
+      }
+    }
+    for (var controller in controllers) {
+      if (!controller.isClosed) {
+        controller.close();
+      }
+    }
+    controllers.clear();
+    isClosed = true;
+    onClosed(this);
+  }
+
+  Stream<T> get stream {
+    if (isClosed) {
+      throw Exception('Stream is closed');
+    }
+    var controller = StreamController<T>();
+    controllers.add(controller);
+    controller.onCancel = () {
+      controllers.remove(controller);
+    };
+    return controller.stream;
+  }
+
+  void cancel() {
+    for (var controller in controllers) {
+      controller.close();
+    }
+    controllers.clear();
+    isClosed = true;
+  }
+}
+
 class ImageDownloadProgress {
   final int currentBytes;
 

lib/pages/reader/images.dart

@@ -21,6 +21,12 @@ class _ReaderImagesState extends State<_ReaderImages> {
     super.initState();
   }
 
+  @override
+  void dispose() {
+    super.dispose();
+    ImageDownloader.cancelAllLoadingImages();
+  }
+
   void load() async {
     if (inProgress) return;
     inProgress = true;
@@ -104,14 +110,14 @@ class _GalleryModeState extends State<_GalleryMode>
     implements _ImageViewController {
   late PageController controller;
 
-  late List<bool> cached;
-
   int get preCacheCount => appdata.settings["preloadImageCount"];
 
   var photoViewControllers = <int, PhotoViewController>{};
 
   late _ReaderState reader;
 
+  /// [totalPages] is the total number of pages in the current chapter.
+  /// More than one images can be displayed on one page.
   int get totalPages => (reader.images!.length / reader.imagesPerPage).ceil();
 
   var imageStates = <State<ComicImage>>{};
@@ -125,24 +131,36 @@ class _GalleryModeState extends State<_GalleryMode>
     reader = context.reader;
     controller = PageController(initialPage: reader.page);
     reader._imageViewController = this;
-    cached = List.filled(reader.maxPage + 2, false);
     Future.microtask(() {
       context.readerScaffold.setFloatingButton(0);
     });
     super.initState();
   }
 
-  void cache(int current) {
+  /// [cache] is used to cache the images.
-    for (int i = current + 1; i <= current + preCacheCount; i++) {
+  /// The count of images to cache is determined by the [preCacheCount] setting.
-      if (i <= totalPages && !cached[i]) {
+  /// For previous page and next page, it will do a memory cache.
-        int startIndex = (i - 1) * reader.imagesPerPage;
+  /// For current page, it will do nothing because it is already on the screen.
+  /// For other pages, it will do a pre-download cache.
+  void cache(int startPage) {
+    print("Cache page $startPage");
+    for (int i = startPage - 1; i <= startPage + preCacheCount; i++) {
+      if (i == startPage || i <= 0 || i > totalPages) continue;
+      bool shouldPreCache = i == startPage + 1 || i == startPage - 1;
+      _cachePage(i, shouldPreCache);
+    }
+  }
+
+  void _cachePage(int page, bool shouldPreCache) {
+    int startIndex = (page - 1) * reader.imagesPerPage;
     int endIndex =
     math.min(startIndex + reader.imagesPerPage, reader.images!.length);
+    print("Cache page $page: $startIndex-$endIndex");
     for (int i = startIndex; i < endIndex; i++) {
-          precacheImage(
+      if (shouldPreCache) {
-              _createImageProviderFromKey(reader.images![i], context), context);
+        _precacheImage(i+1, context);
-        }
+      } else {
-        cached[i] = true;
+        _preDownloadImage(i+1, context);
       }
     }
   }
@@ -192,7 +210,6 @@ class _GalleryModeState extends State<_GalleryMode>
             List<String> pageImages =
                 reader.images!.sublist(startIndex, endIndex);
 
-            cached[index] = true;
             cache(index);
 
             photoViewControllers[index] ??= PhotoViewController();
@@ -201,8 +218,11 @@ class _GalleryModeState extends State<_GalleryMode>
               return PhotoViewGalleryPageOptions(
                 filterQuality: FilterQuality.medium,
                 controller: photoViewControllers[index],
-                imageProvider:
+                imageProvider: _createImageProviderFromKey(
-                    _createImageProviderFromKey(pageImages[0], context),
+                  pageImages[0],
+                  context,
+                  startIndex + 1,
+                ),
                 fit: BoxFit.contain,
                 errorBuilder: (_, error, s, retry) {
                   return NetworkError(message: error.toString(), retry: retry);
@@ -214,7 +234,7 @@ class _GalleryModeState extends State<_GalleryMode>
               controller: photoViewControllers[index],
               minScale: PhotoViewComputedScale.contained * 1.0,
               maxScale: PhotoViewComputedScale.covered * 10.0,
-              child: buildPageImages(pageImages),
+              child: buildPageImages(pageImages, startIndex),
             );
           }
         },
@@ -249,7 +269,7 @@ class _GalleryModeState extends State<_GalleryMode>
     );
   }
 
-  Widget buildPageImages(List<String> images) {
+  Widget buildPageImages(List<String> images, int startIndex) {
     Axis axis = (reader.mode == ReaderMode.galleryTopToBottom)
         ? Axis.vertical
         : Axis.horizontal;
@@ -267,7 +287,11 @@ class _GalleryModeState extends State<_GalleryMode>
           child: ComicImage(
             width: double.infinity,
             height: double.infinity,
-            image: _createImageProviderFromKey(images[0], context),
+            image: _createImageProviderFromKey(
+              images[0],
+              context,
+              startIndex + 1,
+            ),
             fit: BoxFit.contain,
             alignment: axis == Axis.vertical
                 ? Alignment.bottomCenter
@@ -280,7 +304,11 @@ class _GalleryModeState extends State<_GalleryMode>
           child: ComicImage(
             width: double.infinity,
             height: double.infinity,
-            image: _createImageProviderFromKey(images[1], context),
+            image: _createImageProviderFromKey(
+              images[1],
+              context,
+              startIndex + 2,
+            ),
             fit: BoxFit.contain,
             alignment: axis == Axis.vertical
                 ? Alignment.topCenter
@@ -292,8 +320,9 @@ class _GalleryModeState extends State<_GalleryMode>
       ];
     } else {
       imageWidgets = images.map((imageKey) {
+        startIndex++;
         ImageProvider imageProvider =
-            _createImageProviderFromKey(imageKey, context);
+            _createImageProviderFromKey(imageKey, context, startIndex);
         return Expanded(
           child: ComicImage(
             image: imageProvider,
@@ -402,34 +431,22 @@ class _GalleryModeState extends State<_GalleryMode>
         keyRepeatTimer = null;
       }
       if (forward == true) {
-        controller.nextPage(
+        reader.toPage(reader.page+1);
-          duration: const Duration(milliseconds: 200),
-          curve: Curves.ease,
-        );
       } else if (forward == false) {
-        controller.previousPage(
+        reader.toPage(reader.page-1);
-          duration: const Duration(milliseconds: 200),
-          curve: Curves.ease,
-        );
       }
     }
     if (event is KeyRepeatEvent && keyRepeatTimer == null) {
       keyRepeatTimer = Timer.periodic(
-        const Duration(milliseconds: 100),
+        const Duration(milliseconds: 200),
         (timer) {
           if (!mounted) {
             timer.cancel();
             return;
           } else if (forward == true) {
-            controller.nextPage(
+            reader.toPage(reader.page+1);
-              duration: const Duration(milliseconds: 100),
-              curve: Curves.ease,
-            );
           } else if (forward == false) {
-            controller.previousPage(
+            reader.toPage(reader.page-1);
-              duration: const Duration(milliseconds: 100),
-              curve: Curves.ease,
-            );
           }
         },
       );
@@ -599,7 +616,7 @@ class _ContinuousModeState extends State<_ContinuousMode>
   void cacheImages(int current) {
     for (int i = current + 1; i <= current + preCacheCount; i++) {
       if (i <= reader.maxPage && !cached[i]) {
-        _precacheImage(i, context);
+        _preDownloadImage(i, context);
         cached[i] = true;
       }
     }
@@ -1016,7 +1033,10 @@ class _ContinuousModeState extends State<_ContinuousMode>
 }
 
 ImageProvider _createImageProviderFromKey(
-    String imageKey, BuildContext context) {
+  String imageKey,
+  BuildContext context,
+  int page,
+) {
   var reader = context.reader;
   return ReaderImageProvider(
     imageKey,
@@ -1030,16 +1050,38 @@ ImageProvider _createImageProviderFromKey(
 ImageProvider _createImageProvider(int page, BuildContext context) {
   var reader = context.reader;
   var imageKey = reader.images![page - 1];
-  return _createImageProviderFromKey(imageKey, context);
+  return _createImageProviderFromKey(imageKey, context, page);
 }
 
+/// [_precacheImage] is used to precache the image for the given page.
+/// The image is cached using the flutter's [precacheImage] method.
+/// The image will be downloaded and decoded into memory.
 void _precacheImage(int page, BuildContext context) {
+  if (page <= 0 || page > context.reader.images!.length) {
+    return;
+  }
+  print("Precache image for page $page");
   precacheImage(
     _createImageProvider(page, context),
     context,
   );
 }
 
+/// [_preDownloadImage] is used to download the image for the given page.
+/// The image is downloaded using the [CacheManager] and saved to the local storage.
+void _preDownloadImage(int page, BuildContext context) {
+  if (page <= 0 || page > context.reader.images!.length) {
+    return;
+  }
+  print("Preload image for page $page");
+  var reader = context.reader;
+  var imageKey = reader.images![page - 1];
+  var cid = reader.cid;
+  var eid = reader.eid;
+  var sourceKey = reader.type.comicSource?.key;
+  ImageDownloader.loadComicImage(imageKey, sourceKey, cid, eid);
+}
+
 class _SwipeChangeChapterProgress extends StatefulWidget {
   const _SwipeChangeChapterProgress({
     this.controller,

lib/pages/reader/reader.dart

@@ -29,6 +29,7 @@ import 'package:venera/foundation/image_provider/reader_image.dart';
 import 'package:venera/foundation/local.dart';
 import 'package:venera/foundation/log.dart';
 import 'package:venera/foundation/res.dart';
+import 'package:venera/network/images.dart';
 import 'package:venera/pages/settings/settings_page.dart';
 import 'package:venera/utils/clipboard_image.dart';
 import 'package:venera/utils/data_sync.dart';

pubspec.lock

@@ -45,10 +45,10 @@ packages:
     dependency: transitive
     description:
       name: async
-      sha256: "758e6d74e971c3e5aceb4110bfd6698efc7f501675bcfe0c775459a8140750eb"
+      sha256: d2872f9c19731c2e5f10444b14686eb7cc85c76274bd6c16e1816bff9a3bab63
       url: "https://pub.dev"
     source: hosted
-    version: "2.13.0"
+    version: "2.12.0"
   battery_plus:
     dependency: "direct main"
     description:
@@ -182,10 +182,10 @@ packages:
     dependency: transitive
     description:
       name: fake_async
-      sha256: "5368f224a74523e8d2e7399ea1638b37aecfca824a3cc4dfdf77bf1fa905ac44"
+      sha256: "6a95e56b2449df2273fd8c45a662d6947ce1ebb7aafe80e550a3f68297f3cacc"
       url: "https://pub.dev"
     source: hosted
-    version: "1.3.3"
+    version: "1.3.2"
   ffi:
     dependency: transitive
     description:
@@ -516,10 +516,10 @@ packages:
     dependency: "direct main"
     description:
       name: intl
-      sha256: "3df61194eb431efc39c4ceba583b95633a403f46c9fd341e550ce0bfa50e9aa5"
+      sha256: d6f56758b7d3014a48af9701c085700aac781a92a87a62b1333b46d8879661cf
       url: "https://pub.dev"
     source: hosted
-    version: "0.20.2"
+    version: "0.19.0"
   io:
     dependency: transitive
     description:
@@ -540,10 +540,10 @@ packages:
     dependency: transitive
     description:
       name: leak_tracker
-      sha256: "6bb818ecbdffe216e81182c2f0714a2e62b593f4a4f13098713ff1685dfb6ab0"
+      sha256: c35baad643ba394b40aac41080300150a4f08fd0fd6a10378f8f7c6bc161acec
       url: "https://pub.dev"
     source: hosted
-    version: "10.0.9"
+    version: "10.0.8"
   leak_tracker_flutter_testing:
     dependency: transitive
     description:
@@ -1029,10 +1029,10 @@ packages:
     dependency: transitive
     description:
       name: vm_service
-      sha256: ddfa8d30d89985b96407efce8acbdd124701f96741f2d981ca860662f1c0dc02
+      sha256: "0968250880a6c5fe7edc067ed0a13d4bae1577fe2771dcf3010d52c4a9d3ca14"
       url: "https://pub.dev"
     source: hosted
-    version: "15.0.0"
+    version: "14.3.1"
   web:
     dependency: transitive
     description:

pubspec.yaml

@@ -2,7 +2,7 @@ name: venera
 description: "A comic app."
 publish_to: 'none'
 
-version: 1.4.0+140
+version: 1.4.1+141
 
 environment:
   sdk: '>=3.6.0 <4.0.0'