From b0c151906386b5cea2e369e8abe46dcdd8fc01cb Mon Sep 17 00:00:00 2001 From: marc barbier Date: Fri, 9 Jul 2021 19:26:04 +0200 Subject: [PATCH] nb thread fix + corruption check --- ModsListDownloader/scripts/release.sh | 4 +- ModsListDownloader/src/downloader/Log.java | 4 +- ModsListDownloader/src/downloader/Main.java | 1 + .../src/downloader/ModUpdater.java | 50 ++++++++++++++++--- .../src/downloader/helper/HttpHelper.java | 15 +++--- 5 files changed, 56 insertions(+), 18 deletions(-) diff --git a/ModsListDownloader/scripts/release.sh b/ModsListDownloader/scripts/release.sh index 8f418ed..2946cc6 100755 --- a/ModsListDownloader/scripts/release.sh +++ b/ModsListDownloader/scripts/release.sh @@ -4,8 +4,8 @@ then fi -rm release-${VER}.zip 2> /dev/null mv mcDownloader.jar ModPackDl.jar -echo "java -jar ModPackDl.jar -t -v" > run.sh -echo "java -jar ModPackDl.jar -t -v" > run.bat +echo "java -jar ModPackDl.jar --thread 2 -v" > run.sh +echo "java -jar ModPackDl.jar --thread 2 -v" > run.bat echo "pause" >> run.bat zip release-${VER}.zip ModPackDl.jar run.sh run.bat rm run.sh diff --git a/ModsListDownloader/src/downloader/Log.java b/ModsListDownloader/src/downloader/Log.java index e23fb69..79960ff 100644 --- a/ModsListDownloader/src/downloader/Log.java +++ b/ModsListDownloader/src/downloader/Log.java @@ -19,12 +19,12 @@ public class Log { //erase last stream since we used \r lastStream.print(""); stream.println(line); - stream.print(staticLine + "\r"); + stream.print(staticLine + " \r"); lastStream = stream; } public static void setStaticPrint(String staticString) { staticLine = staticString; - System.out.print(staticString + "\r"); + System.out.print(staticString + " \r"); } } diff --git a/ModsListDownloader/src/downloader/Main.java b/ModsListDownloader/src/downloader/Main.java index 8dad0c6..ef0bd86 100644 --- a/ModsListDownloader/src/downloader/Main.java +++ b/ModsListDownloader/src/downloader/Main.java @@ -22,6 +22,7 @@ public class Main { interpretArgs(args); //no threading option called => only one thread will be used if( threadNb == null)threadNb = 1; + Log.i("Main", "Running on " + threadNb + " thread"); new ModUpdater().update(); } diff --git a/ModsListDownloader/src/downloader/ModUpdater.java b/ModsListDownloader/src/downloader/ModUpdater.java index dd84662..38a951e 100644 --- a/ModsListDownloader/src/downloader/ModUpdater.java +++ b/ModsListDownloader/src/downloader/ModUpdater.java @@ -9,6 +9,7 @@ import java.util.ArrayList; import java.util.List; import downloader.forgeSvc.ForgeSvcFile; +import downloader.helper.ArchiveHelper; import downloader.helper.HttpHelper; public class ModUpdater { @@ -61,6 +62,31 @@ public class ModUpdater { Thread.sleep(100); } catch (InterruptedException e) {} } + + if(! UpdaterThread.failedLines.isEmpty()) { + List failures = new ArrayList<>(UpdaterThread.failedLines); + UpdaterThread.failedLines.clear(); + + Log.e(ME, "filed to download " + failures.size()); + Log.e(ME, "Retrying but slower"); + + for(String line : failures) { + Thread t = new UpdaterThread(line, db, directUpdateManager, curseUpdateManager, threads); + t.start(); + try { + t.join(); + } catch (InterruptedException e) {} + } + + if(! UpdaterThread.failedLines.isEmpty()) { + Log.e(ME, "could not download the following mods : "); + for(String s: failures) { + Log.e(ME, " " + s); + } + } + } + + Log.i(ME, "finished"); curseUpdateManager.updateFile(); } @@ -79,6 +105,7 @@ final class UpdaterThread extends Thread { private final CurseUpdateManager curseUpdateManager; private final List threads; private static final String ME = "ModUpdaterThread"; + public static final List failedLines = new ArrayList<>(); public UpdaterThread(String line, Database db, DirectUpdateManager directUpdateManager, CurseUpdateManager curseUpdateManager, List threads) { super(line); @@ -91,9 +118,11 @@ final class UpdaterThread extends Thread { @Override public void run() { + //used for post download checks + File downloadedFile = null; if(line.startsWith("direct=")) { - handleDirectDownload(line); + downloadedFile = handleDirectDownload(line); } else if (line.startsWith("del=")) { String[] delete = line.split("del="); if(new File("mods/" + delete[1]).delete()) { @@ -102,15 +131,21 @@ final class UpdaterThread extends Thread { } else { if (line.split("/").length >= 5) { - handleCurseDownload(line); + downloadedFile = handleCurseDownload(line); } } + + if(downloadedFile != null && !ArchiveHelper.checkJarIntegrity(downloadedFile)) { + downloadedFile.delete(); + failedLines.add(line); + } + super.run(); //on se retire de la pool d'execution this.threads.remove(this); } - private void handleDirectDownload(String line) { + private File handleDirectDownload(String line) { String url = line.replaceFirst("direct=.*@", ""); String filename=extractNameFromLink(line); @@ -122,15 +157,16 @@ final class UpdaterThread extends Thread { { Log.i(ME,"downloading "+ filename); try { - HttpHelper.readFileFromUrlToFolder(url,"mods",filename); Log.i(ME,"done"); + return HttpHelper.readFileFromUrlToFolder(url,"mods",filename); } catch (MalformedURLException e) { Log.e(ME, "could not update mod, error while downloading"); } } + return null; } - private void handleCurseDownload(String line) { + private File handleCurseDownload(String line) { String name = line.split("/")[5]; int modID = db.findModBySlug(name); ProjectInfo pj = db.getProjectInfo(modID); @@ -157,13 +193,14 @@ final class UpdaterThread extends Thread { if (!upToDate) { Log.i("MOD", "downloading " + pj.getName().replace(" ", "-")); - HttpHelper.readFileFromUrlToFolder(downloadUrl, "mods"); + File downloaded = HttpHelper.readFileFromUrlToFolder(downloadUrl, "mods"); Log.i("MOD", "download finished"); if (!this.curseUpdateManager.isModIdKnown(modID)) { this.curseUpdateManager.addModsToTheList(HttpHelper.getFileNameFromURL(downloadUrl), modID); Log.i("DB", "added a mod to the registry"); } + return downloaded; } } catch (MalformedURLException e) { e.printStackTrace(); @@ -171,6 +208,7 @@ final class UpdaterThread extends Thread { } else { Log.e("DB", "no file in server"); } + return null; } private static String extractNameFromLink(String line) { diff --git a/ModsListDownloader/src/downloader/helper/HttpHelper.java b/ModsListDownloader/src/downloader/helper/HttpHelper.java index c6d7cd8..1d313df 100644 --- a/ModsListDownloader/src/downloader/helper/HttpHelper.java +++ b/ModsListDownloader/src/downloader/helper/HttpHelper.java @@ -121,14 +121,13 @@ public class HttpHelper { int bytesRead = in.read(buffer); if (bytesRead < 0) break; - { - double elapsed = ( System.currentTimeMillis() - start ) / 1000; - if(elapsed != 0) { - long estimatedBandwith = (long) (bytesRead / elapsed); - //display the estimate - Log.setStaticPrint(formatBandwidth(estimatedBandwith)); - start = System.currentTimeMillis(); - } + + double elapsed = ( System.currentTimeMillis() - start ) / 1000; + if(elapsed != 0) { + long estimatedBandwith = (long) (bytesRead / elapsed); + //display the estimate + Log.setStaticPrint(formatBandwidth(estimatedBandwith)); + start = System.currentTimeMillis(); } downloadingWriter.write(buffer, 0, bytesRead); }