From 3d6224e3aff0edb7f5cfe8eeb15b3041bf7ed5f0 Mon Sep 17 00:00:00 2001 From: marc barbier Date: Wed, 13 Jan 2021 23:41:32 +0100 Subject: [PATCH] refactor + cache de la bdd --- downloader/DBConnector.java | 2 +- downloader/Database.java | 111 ++++++++++++++++---------- downloader/Downloader.java | 44 ---------- downloader/HttpHelper.java | 71 ++++++---------- downloader/IntegrityChecker.java | 39 +++++++++ downloader/Log.java | 12 +++ downloader/Main.java | 60 ++++++++++++++ downloader/forgeSvc/ForgeSvcFile.java | 6 -- 8 files changed, 205 insertions(+), 140 deletions(-) delete mode 100644 downloader/Downloader.java create mode 100644 downloader/IntegrityChecker.java create mode 100644 downloader/Log.java create mode 100644 downloader/Main.java diff --git a/downloader/DBConnector.java b/downloader/DBConnector.java index 1bb32d7..163ff64 100644 --- a/downloader/DBConnector.java +++ b/downloader/DBConnector.java @@ -18,7 +18,7 @@ public class DBConnector { try { // db parameters String url = "jdbc:sqlite:" + dbFile; - System.out.println(url); + Log.i("DB",url); // create a connection to the database conn = DriverManager.getConnection(url); diff --git a/downloader/Database.java b/downloader/Database.java index 9f1ff8d..7ee87d6 100644 --- a/downloader/Database.java +++ b/downloader/Database.java @@ -1,10 +1,13 @@ package downloader; import java.io.BufferedInputStream; +import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.FileReader; import java.io.IOException; +import java.io.PrintWriter; import java.net.MalformedURLException; import java.sql.ResultSet; import java.sql.SQLException; @@ -22,70 +25,88 @@ public class Database { String version; DBConnector connector; Gson gson; + IntegrityChecker integrityChecker; + File dbVer; public Database() { + dbVer = new File("dbVersion"); + if (dbVer.exists()) { + try { + BufferedReader bfr = new BufferedReader(new FileReader(dbVer)); + this.version = bfr.readLine(); + bfr.close(); - // Todo save ver - this.version = ""; + connector = new DBConnector(); + connector.connect(new File("database.dat").getAbsolutePath()); + + } catch (IOException | SQLException e) { + dbVer.delete(); + version = ""; + } + } else { + version = ""; + } gson = new Gson(); - + integrityChecker = new IntegrityChecker(); + } public void updateDatabase() { try { String dbVersion = HttpHelper.readStringFromUrl("http://files.mcdex.net/data/latest.v5"); if (dbVersion.equals(version)) { - println("alredy up to date"); + Log.i("DB", "alredy up to date"); return; } if (dbVersion.equals("error")) { - println("an error ocured"); + Log.i("DB", "an error ocured"); return; } - println("newer version found"); + Log.i("DB", "newer version found"); if (!this.version.isEmpty()) { - println("current=" + this.version + " remote=" + dbVersion); + Log.i("DB", "current=" + this.version + " remote=" + dbVersion); } else { - println("new=" + dbVersion); + Log.i("DB", "new=" + dbVersion); } - println("fetching=" + "http://files.mcdex.net/data/mcdex-v5-" + dbVersion + ".dat.bz2"); - File archive = HttpHelper.readFileFromUrl(null,"http://files.mcdex.net/data/mcdex-v5-" + dbVersion + ".dat.bz2"); - println("download finished"); - println("decompressing db"); + Log.i("DB", "fetching=" + "http://files.mcdex.net/data/mcdex-v5-" + dbVersion + ".dat.bz2"); + File archive = HttpHelper.readFileFromUrl("http://files.mcdex.net/data/mcdex-v5-" + dbVersion + ".dat.bz2"); + Log.i("DB", "download finished"); + Log.i("DB", "decompressing db"); File decompressedDatabase; try { decompressedDatabase = decompressBz2(archive, "database.dat"); } catch (IOException e) { - println("cannot extract database"); + Log.i("DB", "cannot extract database"); return; } - println("done"); - println("loading the database"); + Log.i("DB", "done"); + Log.i("DB", "loading the database"); connector = new DBConnector(); try { connector.connect(decompressedDatabase.getAbsolutePath()); } catch (SQLException e) { return; } - println("database loaded"); + Log.i("DB", "database loaded"); this.version = dbVersion; - + Log.i("DB", "saving db data"); + dbVer.delete(); + try { + dbVer.createNewFile(); + PrintWriter p = new PrintWriter(dbVer); + p.print(version); + p.close(); + } catch (Exception e) { + Log.e("DB", "failed to save db"); + } } catch (MalformedURLException e) { - println("database link is dead"); + Log.i("DB", "database link is dead"); System.exit(1); } } - private void print(String message) { - System.out.print("db : " + message); - } - - private void println(String message) { - print(message + "\n"); - } - private File decompressBz2(File inputFile, String outputFile) throws IOException { BZip2CompressorInputStream input = new BZip2CompressorInputStream( new BufferedInputStream(new FileInputStream(inputFile))); @@ -102,9 +123,9 @@ public class Database { try { if (rs.next()) { modID = rs.getInt("projectid"); - System.out.println("modid:" + modID); + Log.i("DB", "modid:" + modID); } else { - System.out.println("not found " + slug); + Log.e("DB", "not found " + slug); return -1; } } catch (SQLException e) { @@ -119,8 +140,6 @@ public class Database { } private ProjectInfo getProjectInfo(int projectId) { - System.out.println( - "select slug, name, description from projects where projectid = " + projectId + " and type = 0"); ResultSet rs = connector.executeRequest( "select slug, name, description from projects where projectid = " + projectId + " and type = 0"); try { @@ -135,24 +154,34 @@ public class Database { public boolean fetchMod(String string) { String name = string.split("/")[5]; int modID = findModBySlug(name); - if (modID == -1) + if (modID == -1) { + Log.e("DB", "could not find mod id"); return false; + } + ProjectInfo pj = getProjectInfo(modID); if (pj == null) return false; CurseForgeModFile modfile = new CurseForgeModFile(pj, modID, -1, false); ForgeSvcFile file = getLastestFile("1.12.2", modfile); + if (file != null) { try { - print("downloading " + pj.getName()); - HttpHelper.readFileFromUrlToFolder(pj.getSlug(),file.getDownloadUrl(modID), "mods"); - println("download finished"); + String downloadUrl = file.getDownloadUrl(modID); + + Log.i("DB", "checking " + pj.getName()); + integrityChecker.checkAndDelete(new File("mods/" + HttpHelper.getFileNameFromURL(downloadUrl)), + pj.getSlug()); + // on remplace les espace par des tiret pour eviter toute confusion + Log.i("DB", "downloading " + pj.getName().replace(" ", "-")); + HttpHelper.readFileFromUrlToFolder(downloadUrl, "mods"); + Log.i("DB", "download finished"); return true; } catch (MalformedURLException e) { e.printStackTrace(); } } else { - System.out.println("no file in server"); + Log.e("DB", "no file in server"); } return false; } @@ -162,23 +191,21 @@ public class Database { String jsonProject; try { jsonProject = HttpHelper.readStringFromUrl(projectUrl); - if(jsonProject.equals("error"))return null; + if (jsonProject.equals("error")) + return null; } catch (MalformedURLException e) { e.printStackTrace(); return null; } - + ForgeSvcEntry entry = gson.fromJson(jsonProject, ForgeSvcEntry.class); for (ForgeSvcFile f : entry.getFiles()) { - if (f.getGameVersion().equals(minecraftVer)) { - return f; + if (f.getGameVersion().equals(minecraftVer)) { + return f; } } return null; } - - - } diff --git a/downloader/Downloader.java b/downloader/Downloader.java deleted file mode 100644 index 522a025..0000000 --- a/downloader/Downloader.java +++ /dev/null @@ -1,44 +0,0 @@ -package downloader; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.net.URISyntaxException; - -public class Downloader { - public static void main(String[] args) throws FileNotFoundException, IOException, URISyntaxException { - System.out.println("hello"); - - Database db = new Database(); - db.updateDatabase(); - - File f = new File("modpack.txt"); - BufferedReader in = new BufferedReader(new FileReader(f)); - while (in.ready()) { - String line = in.readLine(); - if (!line.startsWith("//")) { - if (line.split("/").length >= 5 && ! line.startsWith("direct=")) { - if (!db.fetchMod(line)) { - System.err.println("error could not get " + line); - } - }else { - if(line.startsWith("direct@")) - { - char[] chars = new char[line.length()-line.indexOf("@")-"direct=".length()+1]; - line.getChars("direct=".length(), line.indexOf("@"), chars, 0); - String slug = new String(chars); - - String[] url = line.replace("direct=*@", "").split("/"); - System.out.println("downloading "+ url[url.length -1 ]); - HttpHelper.readFileFromUrlToFolder(slug,line.replace("direct@", ""),"mods"); - System.out.println("done"); - } - } - } - } - in.close(); - } - -} diff --git a/downloader/HttpHelper.java b/downloader/HttpHelper.java index 46c524c..f448b79 100644 --- a/downloader/HttpHelper.java +++ b/downloader/HttpHelper.java @@ -10,11 +10,9 @@ import java.net.MalformedURLException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.zip.ZipFile; public class HttpHelper { - public static String[] fileList; public static String readStringFromUrl(String urlToFetch) throws MalformedURLException { URL url = new URL(urlToFetch); @@ -47,38 +45,36 @@ public class HttpHelper { return str; } - public static File readFileFromUrl(String slug, String urlToFetch) throws MalformedURLException { + public static File readFileFromUrl(String urlToFetch) throws MalformedURLException { String[] url = urlToFetch.split("/"); if (url.length == 0) throw new MalformedURLException(); - return readFileFromUrl(slug, urlToFetch, url[url.length - 1]); + return readFileFromUrl(urlToFetch, url[url.length - 1]); } - public static File readFileFromUrl(String slug, String urlToFetch, String filename) throws MalformedURLException { - return readFileFromUrl(slug, urlToFetch, new File(filename)); + public static File readFileFromUrl(String urlToFetch, String filename) throws MalformedURLException { + return readFileFromUrl(urlToFetch, new File(filename)); } - static File readFileFromUrlToFolder(String slug, String urlToFetch, String folder) throws MalformedURLException { + public static File readFileFromUrlToFolder(String urlToFetch, String folder) throws MalformedURLException { try { Files.createDirectories(Paths.get(folder)); } catch (IOException e) { - System.out.println("could not create folder"); + Log.e("HTTPHelper","could not create folder"); return null; } if (!folder.endsWith("/")) folder += "/"; - String[] url = urlToFetch.split("/"); - if (url.length == 0) - throw new MalformedURLException(); - String name = url[url.length - 1]; + String name = getFileNameFromURL(urlToFetch); + if(name == null)throw new MalformedURLException(); - return readFileFromUrl(slug, urlToFetch, new File(folder + name)); + return readFileFromUrl(urlToFetch, new File(folder + name)); } - private static File readFileFromUrl(String slug, String urlToFetch, File destination) throws MalformedURLException { + private static File readFileFromUrl(String urlToFetch, File destination) throws MalformedURLException { URL url = new URL(urlToFetch); HttpURLConnection con; try { @@ -90,16 +86,13 @@ public class HttpHelper { File downloadingFile = destination; - if (slug != null) - checkAndDelete(downloadingFile, slug); - if (downloadingFile.exists()) { return downloadingFile; } else { try { downloadingFile.createNewFile(); } catch (IOException e) { - System.out.println("permission error could not write file"); + Log.e("HTTPHelper","permission error could not write file"); System.exit(1); } } @@ -113,7 +106,7 @@ public class HttpHelper { } try (InputStream in = con.getInputStream()) { - int totalRead = 0; + //int totalRead = 0; // on lit par tranche de 4mo byte[] buffer = new byte[4096]; @@ -121,9 +114,10 @@ public class HttpHelper { int bytesRead = in.read(buffer); if (bytesRead < 0) break; - totalRead += bytesRead; + //totalRead += bytesRead; downloadingWriter.write(buffer, 0, bytesRead); - System.out.println("\r" + totalRead); + //TODO afficher cette valeur si on a un ui + //System.out.println("\r" + totalRead); } downloadingWriter.close(); return downloadingFile; @@ -132,31 +126,14 @@ public class HttpHelper { return null; } - - // suprime les versions obselettes - private static void checkAndDelete(File fForgeSvcFile, String slug) { - if (fileList == null) { - fileList = new File("mods").list(); - } - - if (fForgeSvcFile.exists()) { - try { - //si sa balance une exception sa veut dire que le jar n'est pas lisible - new ZipFile(fForgeSvcFile).close(); - } catch (IOException e) { - fForgeSvcFile.delete(); - return; - } - - for (String s : fileList) { - if (s.contains(slug)) { - if (!fForgeSvcFile.getName().contains(s)) { - new File(s).delete(); - } - return; - } - } - - } + + public static String getFileNameFromURL(String url) + { + String[] urlsplit = url.split("/"); + if (urlsplit.length == 0) + return null; + return urlsplit[urlsplit.length - 1]; } + + } diff --git a/downloader/IntegrityChecker.java b/downloader/IntegrityChecker.java new file mode 100644 index 0000000..bbb4464 --- /dev/null +++ b/downloader/IntegrityChecker.java @@ -0,0 +1,39 @@ +package downloader; + +import java.io.File; +import java.io.IOException; +import java.util.zip.ZipFile; + +public class IntegrityChecker { + public static String[] fileList; + + public IntegrityChecker() + { + fileList = new File("mods").list(); + } + + public void checkAndDelete(File modToDownload, String slug) { + if (modToDownload.exists()) { + try { + //si sa balance une exception sa veut dire que le jar n'est pas lisible + new ZipFile(modToDownload).close(); + } catch (IOException e) { + Log.i("integrityChecker","Corruption detected redownloading"); + modToDownload.delete(); + return; + } + + for (String s : fileList) { + if (s.contains(slug)) { + if (!modToDownload.getName().contains(s)) { + new File(s).delete(); + Log.i("integrityChecker","Corruption detected redownloading"); + } + return; + } + } + + } + } + +} diff --git a/downloader/Log.java b/downloader/Log.java new file mode 100644 index 0000000..e7f5100 --- /dev/null +++ b/downloader/Log.java @@ -0,0 +1,12 @@ +package downloader; + +public class Log { + public static void e(String who,String msg) + { + System.err.println(who + " : "+msg); + } + public static void i(String who,String msg) + { + System.out.println(who +" : "+msg); + } +} diff --git a/downloader/Main.java b/downloader/Main.java new file mode 100644 index 0000000..a8d5276 --- /dev/null +++ b/downloader/Main.java @@ -0,0 +1,60 @@ +package downloader; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.net.URISyntaxException; + +public class Main { + public static void main(String[] args) throws FileNotFoundException, IOException, URISyntaxException { + + IntegrityChecker updateManager = new IntegrityChecker(); + Database db = new Database(); + db.updateDatabase(); + + File f; + if(args.length == 0)f = new File("modpack.txt"); + else f = new File(args[0]); + + BufferedReader in = new BufferedReader(new FileReader(f)); + while (in.ready()) { + String line = in.readLine(); + if (!isAComment(line)) { + if(line.startsWith("direct=")) + { + String filename = line.replaceFirst("direct=*@", ""); + + //le slug est une nom de mods qui se trouve dans le nom du fichier + String slug = extractSlugFromLink(line); + //on check si le mods est pas déja installé et si il est a jours + updateManager.checkAndDelete(new File("mods/"+filename), slug); + + String[] url = line.replaceFirst("direct=*@", "").split("/"); + Log.i("main","downloading "+ url[url.length -1 ]); + + HttpHelper.readFileFromUrlToFolder(filename,"mods"); + Log.i("main","done"); + }else { + if (line.split("/").length >= 5 && !db.fetchMod(line)) { + Log.e("main","error could not get " + line); + } + } + } + } + in.close(); + } + + private static String extractSlugFromLink(String line) { + char[] chars = new char[line.length()-line.indexOf("@")-"direct=".length()+1]; + line.getChars("direct=".length(), line.indexOf("@"), chars, 0); + return new String(chars); + } + + public static boolean isAComment(String line) + { + return line.startsWith("//"); + } + +} diff --git a/downloader/forgeSvc/ForgeSvcFile.java b/downloader/forgeSvc/ForgeSvcFile.java index 8985a54..c5653dd 100644 --- a/downloader/forgeSvc/ForgeSvcFile.java +++ b/downloader/forgeSvc/ForgeSvcFile.java @@ -26,12 +26,6 @@ public class ForgeSvcFile { } public String getDownloadUrl(int modID) throws MalformedURLException { - System.out.println("https://addons-ecs.forgesvc.net/api/v2/addon/"+modID+"/file/"+getProjectFileId()); return HttpHelper.readStringFromUrl("https://addons-ecs.forgesvc.net/api/v2/addon/"+modID+"/file/"+projectFileId+"/download-url"); } - - public String getMD5(int modID) - { - return "https://addons-ecs.forgesvc.net/api/v2/addon/"+modID+"/file/"+getProjectFileId(); - } }