Marc/ModsListDownloader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor + cache de la bdd

Browse Source
This commit is contained in:
marc barbier
2021-01-13 23:41:32 +01:00
parent 568e577173
commit 3d6224e3af
8 changed files with 205 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
downloader/DBConnector.java
+1 -1
View File
@@ -18,7 +18,7 @@ public class DBConnector {
try { try {
// db parameters // db parameters
String url = "jdbc:sqlite:" + dbFile; String url = "jdbc:sqlite:" + dbFile;
System.out.println(url); Log.i("DB",url);
// create a connection to the database // create a connection to the database
conn = DriverManager.getConnection(url); conn = DriverManager.getConnection(url);
downloader/Database.java
+69 -42
View File
@@ -1,10 +1,13 @@
package downloader; package downloader;
import java.io.BufferedInputStream; import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
@@ -22,70 +25,88 @@ public class Database {
String version; String version;
DBConnector connector; DBConnector connector;
Gson gson; Gson gson;
IntegrityChecker integrityChecker;
File dbVer;
public Database() { public Database() {
dbVer = new File("dbVersion");
if (dbVer.exists()) {
try {
BufferedReader bfr = new BufferedReader(new FileReader(dbVer));
this.version = bfr.readLine();
bfr.close();
// Todo save ver connector = new DBConnector();
this.version = ""; connector.connect(new File("database.dat").getAbsolutePath());
} catch (IOException | SQLException e) {
dbVer.delete();
version = "";
}
} else {
version = "";
}
gson = new Gson(); gson = new Gson();
integrityChecker = new IntegrityChecker();
} }
public void updateDatabase() { public void updateDatabase() {
try { try {
String dbVersion = HttpHelper.readStringFromUrl("http://files.mcdex.net/data/latest.v5"); String dbVersion = HttpHelper.readStringFromUrl("http://files.mcdex.net/data/latest.v5");
if (dbVersion.equals(version)) { if (dbVersion.equals(version)) {
println("alredy up to date"); Log.i("DB", "alredy up to date");
return; return;
} }
if (dbVersion.equals("error")) { if (dbVersion.equals("error")) {
println("an error ocured"); Log.i("DB", "an error ocured");
return; return;
} }
println("newer version found"); Log.i("DB", "newer version found");
if (!this.version.isEmpty()) { if (!this.version.isEmpty()) {
println("current=" + this.version + " remote=" + dbVersion); Log.i("DB", "current=" + this.version + " remote=" + dbVersion);
} else { } else {
println("new=" + dbVersion); Log.i("DB", "new=" + dbVersion);
} }
println("fetching=" + "http://files.mcdex.net/data/mcdex-v5-" + dbVersion + ".dat.bz2"); Log.i("DB", "fetching=" + "http://files.mcdex.net/data/mcdex-v5-" + dbVersion + ".dat.bz2");
File archive = HttpHelper.readFileFromUrl(null,"http://files.mcdex.net/data/mcdex-v5-" + dbVersion + ".dat.bz2"); File archive = HttpHelper.readFileFromUrl("http://files.mcdex.net/data/mcdex-v5-" + dbVersion + ".dat.bz2");
println("download finished"); Log.i("DB", "download finished");
println("decompressing db"); Log.i("DB", "decompressing db");
File decompressedDatabase; File decompressedDatabase;
try { try {
decompressedDatabase = decompressBz2(archive, "database.dat"); decompressedDatabase = decompressBz2(archive, "database.dat");
} catch (IOException e) { } catch (IOException e) {
println("cannot extract database"); Log.i("DB", "cannot extract database");
return; return;
} }
println("done"); Log.i("DB", "done");
println("loading the database"); Log.i("DB", "loading the database");
connector = new DBConnector(); connector = new DBConnector();
try { try {
connector.connect(decompressedDatabase.getAbsolutePath()); connector.connect(decompressedDatabase.getAbsolutePath());
} catch (SQLException e) { } catch (SQLException e) {
return; return;
} }
println("database loaded"); Log.i("DB", "database loaded");
this.version = dbVersion; this.version = dbVersion;
Log.i("DB", "saving db data");
dbVer.delete();
try {
dbVer.createNewFile();
PrintWriter p = new PrintWriter(dbVer);
p.print(version);
p.close();
} catch (Exception e) {
Log.e("DB", "failed to save db");
}
} catch (MalformedURLException e) { } catch (MalformedURLException e) {
println("database link is dead"); Log.i("DB", "database link is dead");
System.exit(1); System.exit(1);
} }
} }
private void print(String message) {
System.out.print("db : " + message);
}
private void println(String message) {
print(message + "\n");
}
private File decompressBz2(File inputFile, String outputFile) throws IOException { private File decompressBz2(File inputFile, String outputFile) throws IOException {
BZip2CompressorInputStream input = new BZip2CompressorInputStream( BZip2CompressorInputStream input = new BZip2CompressorInputStream(
new BufferedInputStream(new FileInputStream(inputFile))); new BufferedInputStream(new FileInputStream(inputFile)));
@@ -102,9 +123,9 @@ public class Database {
try { try {
if (rs.next()) { if (rs.next()) {
modID = rs.getInt("projectid"); modID = rs.getInt("projectid");
System.out.println("modid:" + modID); Log.i("DB", "modid:" + modID);
} else { } else {
System.out.println("not found " + slug); Log.e("DB", "not found " + slug);
return -1; return -1;
} }
} catch (SQLException e) { } catch (SQLException e) {
@@ -119,8 +140,6 @@ public class Database {
} }
private ProjectInfo getProjectInfo(int projectId) { private ProjectInfo getProjectInfo(int projectId) {
System.out.println(
"select slug, name, description from projects where projectid = " + projectId + " and type = 0");
ResultSet rs = connector.executeRequest( ResultSet rs = connector.executeRequest(
"select slug, name, description from projects where projectid = " + projectId + " and type = 0"); "select slug, name, description from projects where projectid = " + projectId + " and type = 0");
try { try {
@@ -135,24 +154,34 @@ public class Database {
public boolean fetchMod(String string) { public boolean fetchMod(String string) {
String name = string.split("/")[5]; String name = string.split("/")[5];
int modID = findModBySlug(name); int modID = findModBySlug(name);
if (modID == -1) if (modID == -1) {
Log.e("DB", "could not find mod id");
return false; return false;
}
ProjectInfo pj = getProjectInfo(modID); ProjectInfo pj = getProjectInfo(modID);
if (pj == null) if (pj == null)
return false; return false;
CurseForgeModFile modfile = new CurseForgeModFile(pj, modID, -1, false); CurseForgeModFile modfile = new CurseForgeModFile(pj, modID, -1, false);
ForgeSvcFile file = getLastestFile("1.12.2", modfile); ForgeSvcFile file = getLastestFile("1.12.2", modfile);
if (file != null) { if (file != null) {
try { try {
print("downloading " + pj.getName()); String downloadUrl = file.getDownloadUrl(modID);
HttpHelper.readFileFromUrlToFolder(pj.getSlug(),file.getDownloadUrl(modID), "mods");
println("download finished"); Log.i("DB", "checking " + pj.getName());
integrityChecker.checkAndDelete(new File("mods/" + HttpHelper.getFileNameFromURL(downloadUrl)),
pj.getSlug());
// on remplace les espace par des tiret pour eviter toute confusion
Log.i("DB", "downloading " + pj.getName().replace(" ", "-"));
HttpHelper.readFileFromUrlToFolder(downloadUrl, "mods");
Log.i("DB", "download finished");
return true; return true;
} catch (MalformedURLException e) { } catch (MalformedURLException e) {
e.printStackTrace(); e.printStackTrace();
} }
} else { } else {
System.out.println("no file in server"); Log.e("DB", "no file in server");
} }
return false; return false;
} }
@@ -162,23 +191,21 @@ public class Database {
String jsonProject; String jsonProject;
try { try {
jsonProject = HttpHelper.readStringFromUrl(projectUrl); jsonProject = HttpHelper.readStringFromUrl(projectUrl);
if(jsonProject.equals("error"))return null; if (jsonProject.equals("error"))
return null;
} catch (MalformedURLException e) { } catch (MalformedURLException e) {
e.printStackTrace(); e.printStackTrace();
return null; return null;
} }
ForgeSvcEntry entry = gson.fromJson(jsonProject, ForgeSvcEntry.class); ForgeSvcEntry entry = gson.fromJson(jsonProject, ForgeSvcEntry.class);
for (ForgeSvcFile f : entry.getFiles()) { for (ForgeSvcFile f : entry.getFiles()) {
if (f.getGameVersion().equals(minecraftVer)) { if (f.getGameVersion().equals(minecraftVer)) {
return f; return f;
} }
} }
return null; return null;
} }
} }
downloader/Downloader.java
-44
View File
@@ -1,44 +0,0 @@
package downloader;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.net.URISyntaxException;
public class Downloader {
public static void main(String[] args) throws FileNotFoundException, IOException, URISyntaxException {
System.out.println("hello");
Database db = new Database();
db.updateDatabase();
File f = new File("modpack.txt");
BufferedReader in = new BufferedReader(new FileReader(f));
while (in.ready()) {
String line = in.readLine();
if (!line.startsWith("//")) {
if (line.split("/").length >= 5 && ! line.startsWith("direct=")) {
if (!db.fetchMod(line)) {
System.err.println("error could not get " + line);
}
}else {
if(line.startsWith("direct@"))
{
char[] chars = new char[line.length()-line.indexOf("@")-"direct=".length()+1];
line.getChars("direct=".length(), line.indexOf("@"), chars, 0);
String slug = new String(chars);
String[] url = line.replace("direct=*@", "").split("/");
System.out.println("downloading "+ url[url.length -1 ]);
HttpHelper.readFileFromUrlToFolder(slug,line.replace("direct@", ""),"mods");
System.out.println("done");
}
}
}
}
in.close();
}
}
downloader/HttpHelper.java
+24 -47
View File
@@ -10,11 +10,9 @@ import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.zip.ZipFile;
public class HttpHelper { public class HttpHelper {
public static String[] fileList;
public static String readStringFromUrl(String urlToFetch) throws MalformedURLException { public static String readStringFromUrl(String urlToFetch) throws MalformedURLException {
URL url = new URL(urlToFetch); URL url = new URL(urlToFetch);
@@ -47,38 +45,36 @@ public class HttpHelper {
return str; return str;
} }
public static File readFileFromUrl(String slug, String urlToFetch) throws MalformedURLException { public static File readFileFromUrl(String urlToFetch) throws MalformedURLException {
String[] url = urlToFetch.split("/"); String[] url = urlToFetch.split("/");
if (url.length == 0) if (url.length == 0)
throw new MalformedURLException(); throw new MalformedURLException();
return readFileFromUrl(slug, urlToFetch, url[url.length - 1]); return readFileFromUrl(urlToFetch, url[url.length - 1]);
} }
public static File readFileFromUrl(String slug, String urlToFetch, String filename) throws MalformedURLException { public static File readFileFromUrl(String urlToFetch, String filename) throws MalformedURLException {
return readFileFromUrl(slug, urlToFetch, new File(filename)); return readFileFromUrl(urlToFetch, new File(filename));
} }
static File readFileFromUrlToFolder(String slug, String urlToFetch, String folder) throws MalformedURLException { public static File readFileFromUrlToFolder(String urlToFetch, String folder) throws MalformedURLException {
try { try {
Files.createDirectories(Paths.get(folder)); Files.createDirectories(Paths.get(folder));
} catch (IOException e) { } catch (IOException e) {
System.out.println("could not create folder"); Log.e("HTTPHelper","could not create folder");
return null; return null;
} }
if (!folder.endsWith("/")) if (!folder.endsWith("/"))
folder += "/"; folder += "/";
String[] url = urlToFetch.split("/"); String name = getFileNameFromURL(urlToFetch);
if (url.length == 0) if(name == null)throw new MalformedURLException();
throw new MalformedURLException();
String name = url[url.length - 1];
return readFileFromUrl(slug, urlToFetch, new File(folder + name)); return readFileFromUrl(urlToFetch, new File(folder + name));
} }
private static File readFileFromUrl(String slug, String urlToFetch, File destination) throws MalformedURLException { private static File readFileFromUrl(String urlToFetch, File destination) throws MalformedURLException {
URL url = new URL(urlToFetch); URL url = new URL(urlToFetch);
HttpURLConnection con; HttpURLConnection con;
try { try {
@@ -90,16 +86,13 @@ public class HttpHelper {
File downloadingFile = destination; File downloadingFile = destination;
if (slug != null)
checkAndDelete(downloadingFile, slug);
if (downloadingFile.exists()) { if (downloadingFile.exists()) {
return downloadingFile; return downloadingFile;
} else { } else {
try { try {
downloadingFile.createNewFile(); downloadingFile.createNewFile();
} catch (IOException e) { } catch (IOException e) {
System.out.println("permission error could not write file"); Log.e("HTTPHelper","permission error could not write file");
System.exit(1); System.exit(1);
} }
} }
@@ -113,7 +106,7 @@ public class HttpHelper {
} }
try (InputStream in = con.getInputStream()) { try (InputStream in = con.getInputStream()) {
int totalRead = 0; //int totalRead = 0;
// on lit par tranche de 4mo // on lit par tranche de 4mo
byte[] buffer = new byte[4096]; byte[] buffer = new byte[4096];
@@ -121,9 +114,10 @@ public class HttpHelper {
int bytesRead = in.read(buffer); int bytesRead = in.read(buffer);
if (bytesRead < 0) if (bytesRead < 0)
break; break;
totalRead += bytesRead; //totalRead += bytesRead;
downloadingWriter.write(buffer, 0, bytesRead); downloadingWriter.write(buffer, 0, bytesRead);
System.out.println("\r" + totalRead); //TODO afficher cette valeur si on a un ui
//System.out.println("\r" + totalRead);
} }
downloadingWriter.close(); downloadingWriter.close();
return downloadingFile; return downloadingFile;
@@ -132,31 +126,14 @@ public class HttpHelper {
return null; return null;
} }
// suprime les versions obselettes public static String getFileNameFromURL(String url)
private static void checkAndDelete(File fForgeSvcFile, String slug) { {
if (fileList == null) { String[] urlsplit = url.split("/");
fileList = new File("mods").list(); if (urlsplit.length == 0)
} return null;
return urlsplit[urlsplit.length - 1];
if (fForgeSvcFile.exists()) {
try {
//si sa balance une exception sa veut dire que le jar n'est pas lisible
new ZipFile(fForgeSvcFile).close();
} catch (IOException e) {
fForgeSvcFile.delete();
return;
}
for (String s : fileList) {
if (s.contains(slug)) {
if (!fForgeSvcFile.getName().contains(s)) {
new File(s).delete();
}
return;
}
}
}
} }
} }
downloader/IntegrityChecker.java
+39
View File
@@ -0,0 +1,39 @@
package downloader;
import java.io.File;
import java.io.IOException;
import java.util.zip.ZipFile;
public class IntegrityChecker {
public static String[] fileList;
public IntegrityChecker()
{
fileList = new File("mods").list();
}
public void checkAndDelete(File modToDownload, String slug) {
if (modToDownload.exists()) {
try {
//si sa balance une exception sa veut dire que le jar n'est pas lisible
new ZipFile(modToDownload).close();
} catch (IOException e) {
Log.i("integrityChecker","Corruption detected redownloading");
modToDownload.delete();
return;
}
for (String s : fileList) {
if (s.contains(slug)) {
if (!modToDownload.getName().contains(s)) {
new File(s).delete();
Log.i("integrityChecker","Corruption detected redownloading");
}
return;
}
}
}
}
}
downloader/Log.java
+12
View File
@@ -0,0 +1,12 @@
package downloader;
public class Log {
public static void e(String who,String msg)
{
System.err.println(who + " : "+msg);
}
public static void i(String who,String msg)
{
System.out.println(who +" : "+msg);
}
}
downloader/Main.java
+60
View File
@@ -0,0 +1,60 @@
package downloader;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.net.URISyntaxException;
public class Main {
public static void main(String[] args) throws FileNotFoundException, IOException, URISyntaxException {
IntegrityChecker updateManager = new IntegrityChecker();
Database db = new Database();
db.updateDatabase();
File f;
if(args.length == 0)f = new File("modpack.txt");
else f = new File(args[0]);
BufferedReader in = new BufferedReader(new FileReader(f));
while (in.ready()) {
String line = in.readLine();
if (!isAComment(line)) {
if(line.startsWith("direct="))
{
String filename = line.replaceFirst("direct=*@", "");
//le slug est une nom de mods qui se trouve dans le nom du fichier
String slug = extractSlugFromLink(line);
//on check si le mods est pas déja installé et si il est a jours
updateManager.checkAndDelete(new File("mods/"+filename), slug);
String[] url = line.replaceFirst("direct=*@", "").split("/");
Log.i("main","downloading "+ url[url.length -1 ]);
HttpHelper.readFileFromUrlToFolder(filename,"mods");
Log.i("main","done");
}else {
if (line.split("/").length >= 5 && !db.fetchMod(line)) {
Log.e("main","error could not get " + line);
}
}
}
}
in.close();
}
private static String extractSlugFromLink(String line) {
char[] chars = new char[line.length()-line.indexOf("@")-"direct=".length()+1];
line.getChars("direct=".length(), line.indexOf("@"), chars, 0);
return new String(chars);
}
public static boolean isAComment(String line)
{
return line.startsWith("//");
}
}
downloader/forgeSvc/ForgeSvcFile.java
-6
View File
@@ -26,12 +26,6 @@ public class ForgeSvcFile {
} }
public String getDownloadUrl(int modID) throws MalformedURLException { public String getDownloadUrl(int modID) throws MalformedURLException {
System.out.println("https://addons-ecs.forgesvc.net/api/v2/addon/"+modID+"/file/"+getProjectFileId());
return HttpHelper.readStringFromUrl("https://addons-ecs.forgesvc.net/api/v2/addon/"+modID+"/file/"+projectFileId+"/download-url"); return HttpHelper.readStringFromUrl("https://addons-ecs.forgesvc.net/api/v2/addon/"+modID+"/file/"+projectFileId+"/download-url");
} }
public String getMD5(int modID)
{
return "https://addons-ecs.forgesvc.net/api/v2/addon/"+modID+"/file/"+getProjectFileId();
}
} }