bandwidth precision improvement

Create LICENSE

LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org>

ModsListDownloader/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "java.project.referencedLibraries": [
+        "lib/**/*.jar"
+    ]
+}

ModsListDownloader/scripts/release.sh

@@ -4,8 +4,8 @@ then
 fi
 -rm release-${VER}.zip 2> /dev/null
 mv mcDownloader.jar ModPackDl.jar
-echo "java -jar ModPackDl.jar -t -v" > run.sh
+echo "java -jar ModPackDl.jar --thread 2 -v" > run.sh
-echo "java -jar ModPackDl.jar -t -v" > run.bat
+echo "java -jar ModPackDl.jar --thread 2 -v" > run.bat
 echo "pause" >> run.bat
 zip release-${VER}.zip ModPackDl.jar run.sh run.bat
 rm run.sh

ModsListDownloader/src/downloader/CurseForgeModFile.java

@@ -4,25 +4,21 @@ public class CurseForgeModFile extends ProjectInfo {
 	private int projectID;
 	private int fileID;
 	private boolean clientOnly;	
-	
 	public CurseForgeModFile(ProjectInfo pj, int projectID, int fileID, boolean clientOnly) {
 		this(pj.getSlug(), pj.getName(), pj.getDesc(), projectID, fileID, clientOnly);
 	}	
-
 	public CurseForgeModFile(String slug, String name, String desc, int projectID, int fileID, boolean clientOnly) {
 		super(slug, name, desc);
 		this.projectID = projectID;
 		this.fileID = fileID;
 		this.clientOnly = clientOnly;
 	}	
-
 	public int getFileID() {
 		return this.fileID;
 	}
 	public int getProjectID() {
 		return this.projectID;
 	}
-
 	public boolean isClientOnly() {
 		return this.clientOnly;
 	}

ModsListDownloader/src/downloader/CurseUpdateManager.java

@@ -6,14 +6,13 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
-import java.util.SortedMap;
+import java.util.concurrent.ConcurrentHashMap;
-import java.util.TreeMap;
 
 import downloader.helper.ArchiveHelper;
 import downloader.helper.SlugHelper;
 
 public class CurseUpdateManager {
-	private SortedMap<Integer, String> installedMods;
+	private ConcurrentHashMap<Integer, String> installedMods;
 	
 	public CurseUpdateManager()
 	{
@@ -27,13 +26,13 @@ public class CurseUpdateManager {
 		{
 			try {
 				ObjectInputStream ois = new ObjectInputStream(new FileInputStream(modsMapFile));
-				installedMods = (SortedMap<Integer, String>) ois.readObject();
+				installedMods = (ConcurrentHashMap<Integer, String>) ois.readObject();
 				ois.close();
 			} catch (Exception e) {
 				modsMapFile.delete();
 			}
-			if(installedMods == null)installedMods = new TreeMap<>();
 		}
+		if(installedMods == null)installedMods = new ConcurrentHashMap<>();
 		this.updateFile();
 	}
 
@@ -61,7 +60,7 @@ public class CurseUpdateManager {
 
 		if (alredyInstalledMods.exists()) {
 			//we only check if the jar file still works
-			if(ArchiveHelper.checkJarIntegrity(alredyInstalledMods)) {
+			if(!ArchiveHelper.checkJarIntegrity(alredyInstalledMods)) {
 				alredyInstalledMods.delete();
 				installedMods.remove(modid);
 				updateFile();
@@ -110,18 +109,17 @@ public class CurseUpdateManager {
 	 * @param fileName
 	 * @param modID
 	 */
-	public void addModsToTheList(String fileName,int modID)
+	public synchronized void addModsToTheList(String fileName,int modID)
 	{
 		installedMods.put(modID, fileName);	
-		updateFile();
 	}
 	
-	public boolean isModIdKnown(int modId)
+	public synchronized boolean isModIdKnown(int modId)
 	{
 		return installedMods.get(modId) != null;
 	}
 	
-	private void updateFile()
+	public synchronized void updateFile()
 	{
 		File modsMap = new File("mods/modsMap.sav");
 		try {

ModsListDownloader/src/downloader/Database.java

@@ -13,14 +13,20 @@ import downloader.forgeSvc.ForgeSvcFile;
 import downloader.helper.HttpHelper;
 
 public class Database {
-	DBConnector connector;
+	private DBConnector connector;
 
-	Gson gson;
+	private Gson gson;
+
+	private static final String ME = "Db";
 
 	public Database() {
 		try {
+			//work around to avoid
 			Class.forName("org.sqlite.JDBC");
-		} catch (ClassNotFoundException e1) {} //it will crash anyway
+		} catch (ClassNotFoundException e1) {
+			Log.e(ME, "Error sqLite was not found");
+			System.exit(1);
+		}
 
 		this.connector = new DBConnector();
 		try {
@@ -34,12 +40,12 @@ public class Database {
 	
 	private int findProjectBySlug(String slug, int ptype) {
 		int pjId = -1;
-		ResultSet rs = this.connector.executeRequest("select projectid from projects where type =" + ptype + " and slug LIKE \"%" + slug.trim().toLowerCase() + "%\"");
+		ResultSet rs = this.connector.executeRequest("select projectid from projects where type =" + ptype + " and slug LIKE \"" + slug.trim().toLowerCase() + "\"");
 		try {
 			if (rs.next()) {
 				pjId = rs.getInt("projectid");
 			} else {
-				Log.e("DB", "not found " + slug);
+				Log.e(ME, "not found " + slug);
 				return -1;
 			} 
 		} catch (SQLException e) {

ModsListDownloader/src/downloader/DatabaseVersionManager.java

@@ -4,7 +4,9 @@ import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
+import java.io.FileWriter;
 import java.io.IOException;
+import java.io.PrintWriter;
 import java.net.MalformedURLException;
 
 import downloader.helper.ArchiveHelper;
@@ -29,11 +31,22 @@ public class DatabaseVersionManager {
         if( version == null || !latestVersion.equals(version) ) {
             Log.i(ME, "new dbVersion avaliable downloading...");
             downloadDatabase(latestVersion);
+            saveDbVersion(latestVersion);
         } else {
             Log.i(ME, "update to Date");
         }
     }
 
+    private void saveDbVersion(String latestVersion) {
+        File versionFile = new File("dbVersion");
+        PrintWriter out;
+        try {
+            out = new PrintWriter(new FileWriter(versionFile));
+            out.println(latestVersion);
+            out.close();
+        } catch (IOException e) {}
+    }
+
     private void downloadDatabase(String dbVersion) {
         Log.i(ME, "fetching=" + "http://files.mcdex.net/data/mcdex-v5-" + dbVersion + ".dat.bz2");
         File archive;
@@ -81,8 +94,7 @@ public class DatabaseVersionManager {
                 versionFile.delete();
                 return null;
             }
-        } else {
+        }
         throw new FileNotFoundException();
     }
 }
-}

ModsListDownloader/src/downloader/DirectUpdateManager.java

@@ -13,12 +13,7 @@ public class DirectUpdateManager {
      * @return upToDate
      */
     public boolean checkAndDelete(String filename, String slug) {
-        File mod = new File("mods/"+filename);
+        File mod = new File("mods/"+filename.toLowerCase());
-
-        if (!mod.isFile()) {
-            mod.delete();
-            return false;
-        }
         
         if (!mod.exists()) {
             SlugHelper.deleteBySlug(slug);

ModsListDownloader/src/downloader/Log.java

@@ -1,13 +1,34 @@
 package downloader;
 
+import java.io.PrintStream;
+
 public class Log {
 	private Log() {}
+	private static String staticLine = "";
+	private static PrintStream lastStream = System.out;
 
 	public static void e(String who, String msg) {
-		System.err.println(String.valueOf(who) + "  :  " + msg);
+		printNonStatic("[ERROR]" + who + "  :  " + msg, System.err);
+	}
+
+	public static void w(String who, String msg) {
+		printNonStatic("[WARN]" + who + "  :  " + msg, System.out);
     }
 	
 	public static void i(String who, String msg) {
-		if(Main.verbose)System.out.println(String.valueOf(who) + "  :  " + msg);
+		if(Main.verbose) printNonStatic("[INFO]" + who + "  :  " + msg, System.out);
+	}
+
+	private static void printNonStatic(String line, PrintStream stream) {
+		//erase last stream since we used \r
+		lastStream.print("");
+		stream.println(line);
+		stream.print(staticLine + "                 \r");
+		lastStream = stream;
+	}
+
+	public static void setStaticPrint(String staticString) {
+		staticLine = staticString;
+		System.out.print(staticString + "                    \r");
 	}
 }

ModsListDownloader/src/downloader/Main.java

@@ -10,18 +10,25 @@ import java.util.Iterator;
 public class Main {	
 
 	public static boolean verbose;
-	public static boolean checkingonly;
 	public static Integer threadNb;
     public static String mcVersion;
 
 	public static void main(String[] args) {
+		if(mcVersion == null) {
 			mcVersion = "1.12.2";
-
+		}
+		else {
+			if(!isVersionValid()) {
+				Log.e("main", "the version number provided is invalid");
+				System.exit(1);
+			}
+		}
 		checkModsDirectory();
 		new DatabaseVersionManager().updateIfNeeded();
 		interpretArgs(args);
 		//no threading option called => only one thread will be used
 		if( threadNb == null)threadNb = 1;
+		Log.i("Main", "Running on " + threadNb + " thread");
 		new ModUpdater().update();
 	}
 
@@ -44,55 +51,73 @@ public class Main {
 		while (it.hasNext()) {
 			String cmd = it.next();
 			switch (cmd) {
-				case "--check":
-					System.out.println("--check is not avaliable");
-					checkingonly = true;
-					continue;
 				case "-t":
+					Log.w("main", "Warning using -t option can lead to corruption during download if you have a lot of cores or a slow internet");
+
 					if (threadNb != null) {
 						Log.e("main", "error conflicting arguments --thread and -t");
 						System.exit(1);
 					}
 
 					if (cores <= 0) {
-						Log.i("main","java think you have no cpu core... sooo lets go for 4");
+						Log.i("main","java think you have no cpu core... sooo lets go for 2");
-						threadNb = Integer.valueOf(4);
+						threadNb = Integer.valueOf(2);
-						continue;
+						break;
 					} 
 					Log.i("main", "running on " + cores + " thread");
 					threadNb = Integer.valueOf(cores);
-					continue;
+					break;
 
 				case "-v":
 					verbose = true;
-					continue;
+					break;
+
+				case "-version":
+					checkNext(it);
+					mcVersion = it.next();
+					break;
 					
 				case "--thread":
 					if (threadNb != null) {
 						Log.e("main", "error conflicting arguments --thread and -t");
 						System.exit(1);
 					} 
-					if (!it.hasNext()) {
+					checkNext(it);
-						Log.e("main", "arguments invalid please refer to --help");
-						System.exit(1);
-					} 
 					String threadNumber = it.next();
 					if (threadNumber.matches("[0-9]*")) {
 						threadNb = Integer.valueOf(Integer.parseInt(threadNumber));
-						continue;
+						break;
 					} 
 					Log.e("main","arguments invalid please refer to --help");
 					System.exit(1);
-					continue;
+					break;
 
 				case "--help":
-					Log.i("main", "use --check to check without updating\nuse --help to see this help\nuse -v to see verbose\nuse --threads to specify the number of threads\nuse -t to set the number of thread automaticaly");
+					Log.i("main", "use --help to see this help\nuse -v to see verbose\nuse --threads to specify the number of threads\nuse -t to set the number of thread automaticaly");
 					System.exit(0);
-					continue;
+					break;
 				default:
 					Log.e("main", "unknown args " + cmd + "\nsee --help for help");
 					System.exit(1);
 			} 
 		} 
 	}
+
+	private static boolean isVersionValid() {
+		switch(mcVersion.split(".").length) {
+			case 2:
+				return mcVersion.matches("[0-9]*\\.[0-9]*");
+			case 3:
+				return mcVersion.matches("[0-9]*\\.[0-9]*\\.[0-9]*");
+			default:
+				return false;
+		}
+	}
+
+	private static void checkNext(Iterator<String> it) {
+		if (!it.hasNext()) {
+			Log.e("main", "arguments invalid please refer to --help");
+			System.exit(1);
+		} 
+	}
 }

ModsListDownloader/src/downloader/ModUpdater.java

@@ -7,16 +7,16 @@ import java.io.IOException;
 import java.net.MalformedURLException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.function.Function;
 
 import downloader.forgeSvc.ForgeSvcFile;
+import downloader.helper.ArchiveHelper;
 import downloader.helper.HttpHelper;
 
 public class ModUpdater {
 	private DirectUpdateManager directUpdateManager;
 	private CurseUpdateManager curseUpdateManager;
 	private Database db;
-	private final String ME = "ModUpdater";
+	private static final String ME = "ModUpdater";
 
 	public ModUpdater() {
 		this.db = new Database();
@@ -33,6 +33,14 @@ public class ModUpdater {
         }
 
         //parse and download
+		startProcessingThreads(modsList);
+		checkForFailures();
+
+		Log.i(ME, "finished");
+		curseUpdateManager.updateFile();
+	}
+
+	private void startProcessingThreads(File modsList) {
 		List<Thread> threads = new ArrayList<>(Main.threadNb);
 		try {
 			BufferedReader in = new BufferedReader(new FileReader(modsList));
@@ -44,19 +52,54 @@ public class ModUpdater {
 					threads.add(t);
 				}
 
-				while(Main.threadNb == threads.size()) {
+				while(Main.threadNb >= threads.size()) {
-					try {
+					threadSleep();
-						Thread.sleep(10);
-					} catch (InterruptedException e) {}
 				}
 			}
 			in.close();
-            Log.i(ME, "finished");
-
 		} catch( IOException e ) {
 			e.printStackTrace();
 			System.exit(1);
 		}
+		waitThreadsEnd(threads);
+	}
+
+	private void checkForFailures() {
+		if(! UpdaterThread.failedLines.isEmpty()) {
+			List<String> failures = new ArrayList<>(UpdaterThread.failedLines);
+			UpdaterThread.failedLines.clear();
+
+
+			Log.e(ME, "filed to download " + failures.size());
+			Log.e(ME, "Retrying but slower");
+
+			for(String line : failures) {
+				Thread t = new UpdaterThread(line, db, directUpdateManager, curseUpdateManager, null);
+				t.start();
+				try {
+					t.join();
+				} catch (InterruptedException e) {}
+			}
+
+			if(! UpdaterThread.failedLines.isEmpty()) {
+				Log.e(ME, "could not download the following mods : ");
+				for(String s: failures) {
+					Log.e(ME, "    " + s);
+				}
+			}
+		}
+	}
+
+	private void waitThreadsEnd(List<Thread> threads) {
+		while( !threads.isEmpty() ) {
+			threadSleep();
+		}
+	}
+
+	private void threadSleep() {
+		try {
+			Thread.sleep(10);
+		} catch (InterruptedException e) {}
 	}
 
 	public static boolean isAComment(String line)
@@ -73,9 +116,10 @@ final class UpdaterThread extends Thread {
 	private final CurseUpdateManager curseUpdateManager;
 	private final List<Thread> threads;
 	private static final String ME = "ModUpdaterThread";
+	protected static final List<String> failedLines = new ArrayList<>();
 
 	public UpdaterThread(String line, Database db, DirectUpdateManager directUpdateManager, CurseUpdateManager curseUpdateManager, List<Thread> threads) {
-		super();
+		super(line);
 		this.line = line;
 		this.db = db;
 		this.directUpdateManager = directUpdateManager;
@@ -85,9 +129,12 @@ final class UpdaterThread extends Thread {
 
 	@Override
 	public void run() {
-		if(line.startsWith("direct="))
+		try {
+				//used for post download checks
+			File downloadedFile = null;
+			if(line.startsWith("direct=") && line.contains(";")  && line.contains("@"))
 			{
-			handleDirectDownload(line);
+				downloadedFile = handleDirectDownload(line);
 			} else if (line.startsWith("del=")) {
 				String[] delete = line.split("del=");
 				if(new File("mods/" + delete[1]).delete()) {
@@ -95,16 +142,26 @@ final class UpdaterThread extends Thread {
 				}
 			}
 			else {
-			if (line.split("/").length >= 5) {
+				if (line.split("/").length >= 5 && line.contains("curseforge")) {
-				handleCurseDownload(line);
+					downloadedFile = handleCurseDownload(line);
+				} else {
+					Log.e(ME, "unrecognized line: " + line);
 				}
 			}
-		super.run();
-		//on se retire de la pool d'execution
-		this.threads.remove(this);
-	}
 
-	private void handleDirectDownload(String line) {
+			if(downloadedFile != null && !ArchiveHelper.checkJarIntegrity(downloadedFile)) {
+				downloadedFile.delete();
+				failedLines.add(line);
+			}
+		} catch(Exception e) { e.printStackTrace(); }
+		
+
+		super.run();
+		//on se retire de la pool d'execution
+		if(this.threads != null)this.threads.remove(this);
+	}
+
+	private File handleDirectDownload(String line) {
 		String url = line.replaceFirst("direct=.*@", "");
 		String filename=extractNameFromLink(line);
 		
@@ -114,17 +171,18 @@ final class UpdaterThread extends Thread {
 		boolean upToDate = directUpdateManager.checkAndDelete(filename, slug);
 		if(!upToDate)
 		{
-			Log.i("main","downloading "+ filename);
+			Log.i(ME,"downloading "+ filename);
 			try {
-                HttpHelper.readFileFromUrlToFolder(url,"mods",filename);
+				Log.i(ME,"done");
-                Log.i("main","done");
+				return HttpHelper.readFileFromUrlToFolder(url,"mods",filename);
 			} catch (MalformedURLException e) {
-                Log.e("ModUpdater", "could not update mod, error while downloading");
+				Log.e(ME, "could not update mod, error while downloading");
 			}
 		}
+		return null;
 	}
 
-    private void handleCurseDownload(String line) {
+	private File handleCurseDownload(String line) {
 		String name = line.split("/")[5];
 		int modID = db.findModBySlug(name);
 		ProjectInfo pj = db.getProjectInfo(modID);
@@ -143,18 +201,22 @@ final class UpdaterThread extends Thread {
 				else {
 					Log.i("DB", "NEW MOD DETECTED " + modID);
 					upToDate = this.curseUpdateManager.checkAndDelete(new File("mods/" + HttpHelper.getFileNameFromURL(downloadUrl)), pj.getSlug());
+					if(upToDate) {
+						Log.i("who", "file found");
+						this.curseUpdateManager.addModsToTheList("mods/" + HttpHelper.getFileNameFromURL(downloadUrl), modID);
+					}
 				} 
 
 				if (!upToDate) {
 					Log.i("MOD", "downloading  " + pj.getName().replace(" ", "-")); 
-					
+					File downloaded = HttpHelper.readFileFromUrlToFolder(downloadUrl, "mods"); 
-					HttpHelper.readFileFromUrlToFolder(downloadUrl, "mods"); 
 					
 					Log.i("MOD", "download finished");
 					if (!this.curseUpdateManager.isModIdKnown(modID)) {
 						this.curseUpdateManager.addModsToTheList(HttpHelper.getFileNameFromURL(downloadUrl), modID);
 						Log.i("DB", "added a mod to the registry"); 
 					} 
+					return downloaded;
 				} 
 			} catch (MalformedURLException e) {
 				e.printStackTrace();
@@ -162,6 +224,7 @@ final class UpdaterThread extends Thread {
 		} else {
 			Log.e("DB", "no file in server");
 		} 
+		return null;
     }
 
 	private static String extractNameFromLink(String line) {

ModsListDownloader/src/downloader/helper/HttpHelper.java

@@ -10,8 +10,10 @@ import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.Locale;
 
 import downloader.Log;
+import downloader.Main;
 
 public class HttpHelper {
 	private HttpHelper() {}
@@ -90,13 +92,11 @@ public class HttpHelper {
 		}
 
 		File downloadingFile = destination;
-		System.out.println(downloadingFile.getAbsolutePath());
 
 		if (downloadingFile.exists()) {
 			return downloadingFile;
 		} else {
 			try {
-				System.out.println(downloadingFile);
 				downloadingFile.createNewFile();
 			} catch (IOException e) {
 				Log.e("HTTPHelper","permission error could not write file");
@@ -113,6 +113,7 @@ public class HttpHelper {
 			// imposible vu qu'on l'a crée precedament
 		}
 
+		double start = System.currentTimeMillis();
 		try (InputStream in = con.getInputStream()) {
 			// on lit par tranche de 4mo
 			byte[] buffer = new byte[4096];
@@ -121,6 +122,14 @@ public class HttpHelper {
 				int bytesRead = in.read(buffer);
 				if (bytesRead < 0)
 					break;
+					
+				double elapsed = ( System.currentTimeMillis() - start ) / 1000;
+				if(elapsed != 0) {
+					long estimatedBandwith = (long) (bytesRead / elapsed);
+					//display the estimate
+					Log.setStaticPrint("estimated bandwidth :" + formatBandwidth(estimatedBandwith * Main.threadNb));
+					start = System.currentTimeMillis();
+				}
 				downloadingWriter.write(buffer, 0, 	bytesRead);
 			}
 			downloadingWriter.close();
@@ -130,6 +139,35 @@ public class HttpHelper {
 
 	}
 
+	/**
+	 *
+	 * @param estimatedBandwith in o/s
+	 * @return
+	 */
+	private static String formatBandwidth(long estimatedBandwith) {
+		//is ko/s
+		if(estimatedBandwith > 1000) {
+			//is mo/s
+			if(estimatedBandwith > 1000 * 1000) {
+				return String.valueOf(estimatedBandwith/ (1000 * 1000)) + "M" + getLocalizedByte() + "/s";
+			} else {
+				return String.valueOf(estimatedBandwith/ 1000) + "K" + getLocalizedByte() + "/s";
+			}
+		} else {
+			return String.valueOf(estimatedBandwith) + getLocalizedByte() + "/s";
+		}
+	}
+
+	private static String getLocalizedByte() {
+		Locale l = Locale.getDefault();
+
+		if (l.equals(Locale.FRANCE) | l.equals(Locale.FRENCH)) {
+			return "O";
+		} else {
+			return "B";
+		}
+	}
+
 	public static String getFileNameFromURL(String url)
 	{
 		String[] urlsplit = url.split("/");