From 915712e636e6a609d8dfe7556196b854e808aecd Mon Sep 17 00:00:00 2001 From: Stephan Richter Date: Wed, 20 Apr 2022 17:45:11 +0200 Subject: [PATCH] first version with working archive --- doc/data structure.dia | 133 +++++++++----- pom.xml | 2 +- .../de/srsoftware/widerhall/Constants.java | 1 + .../java/de/srsoftware/widerhall/Util.java | 4 + .../srsoftware/widerhall/data/Database.java | 3 +- .../widerhall/data/MailingList.java | 38 +--- .../de/srsoftware/widerhall/data/Post.java | 169 ++++++++++++++++++ .../de/srsoftware/widerhall/web/Rest.java | 19 ++ .../java/de/srsoftware/widerhall/web/Web.java | 34 +++- static/templates/archive.st | 26 +++ static/templates/inspect.st | 4 + static/templates/js.st | 22 ++- static/templates/post.st | 34 ++++ 13 files changed, 409 insertions(+), 80 deletions(-) create mode 100644 src/main/java/de/srsoftware/widerhall/data/Post.java create mode 100644 static/templates/archive.st create mode 100644 static/templates/post.st diff --git a/doc/data structure.dia b/doc/data structure.dia index e887b44..97d23ed 100644 --- a/doc/data structure.dia +++ b/doc/data structure.dia @@ -1051,13 +1051,13 @@ - + - + @@ -1080,7 +1080,7 @@ - + @@ -1096,13 +1096,13 @@ - + - + @@ -1125,7 +1125,7 @@ - + @@ -1141,13 +1141,13 @@ - + - + @@ -1161,7 +1161,7 @@ - #From# + #FromAddr# @@ -1170,7 +1170,7 @@ - + @@ -1186,13 +1186,13 @@ - + - + @@ -1215,7 +1215,7 @@ - + @@ -1228,13 +1228,13 @@ - + - + @@ -1267,16 +1267,16 @@ - + - + - + - + @@ -1299,7 +1299,7 @@ - + @@ -1312,16 +1312,16 @@ - + - + - + - + @@ -1344,7 +1344,7 @@ - + @@ -1357,16 +1357,16 @@ - + - + - + - + @@ -1389,7 +1389,7 @@ - + @@ -1402,16 +1402,16 @@ - + - + - + - + @@ -1425,7 +1425,7 @@ - #Content# + #File# @@ -1434,7 +1434,7 @@ - + @@ -1447,16 +1447,16 @@ - + - + - - + + - + @@ -1482,13 +1482,13 @@ - + - + - + @@ -1695,5 +1695,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + #FromAddr# + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 99031ba..3b62480 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.example Widerhall - 0.1.4 + 0.2.1 diff --git a/src/main/java/de/srsoftware/widerhall/Constants.java b/src/main/java/de/srsoftware/widerhall/Constants.java index be0229a..b84620f 100644 --- a/src/main/java/de/srsoftware/widerhall/Constants.java +++ b/src/main/java/de/srsoftware/widerhall/Constants.java @@ -12,6 +12,7 @@ public class Constants { public static final String IMAPS = "imaps"; public static final String INBOX = "inbox"; public static final String INDEX = "index"; + public static final String ID = "id"; public static final String INT = "INT"; public static final String LIST = "list"; public static final String LOCATIONS = "locations"; diff --git a/src/main/java/de/srsoftware/widerhall/Util.java b/src/main/java/de/srsoftware/widerhall/Util.java index b3781e4..3f64f67 100644 --- a/src/main/java/de/srsoftware/widerhall/Util.java +++ b/src/main/java/de/srsoftware/widerhall/Util.java @@ -158,4 +158,8 @@ public class Util { return null; } + + public static String dropEmail(String tx) { + return tx.replaceAll( "[.\\-\\w]+@[.\\-\\w]+", "[email_removed]"); + } } diff --git a/src/main/java/de/srsoftware/widerhall/data/Database.java b/src/main/java/de/srsoftware/widerhall/data/Database.java index 5e7c89d..1f85370 100644 --- a/src/main/java/de/srsoftware/widerhall/data/Database.java +++ b/src/main/java/de/srsoftware/widerhall/data/Database.java @@ -236,7 +236,7 @@ public class Database { /** * add a where condition in the form of … WHERE [key] in ([value]) * @param key - * @param values + * @param value * @return */ private Request where(String key, Object value) { @@ -260,6 +260,7 @@ public class Database { if (!tableExists(User.TABLE_NAME)) User.createTable(); if (!tableExists(MailingList.TABLE_NAME)) MailingList.createTable(); if (!tableExists(ListMember.TABLE_NAME)) ListMember.createTable(); + if (!tableExists(Post.TABLE_NAME)) Post.createTable(); return this; } diff --git a/src/main/java/de/srsoftware/widerhall/data/MailingList.java b/src/main/java/de/srsoftware/widerhall/data/MailingList.java index fc450e7..d7e753a 100644 --- a/src/main/java/de/srsoftware/widerhall/data/MailingList.java +++ b/src/main/java/de/srsoftware/widerhall/data/MailingList.java @@ -48,9 +48,10 @@ public class MailingList implements MessageHandler { public static final int STATE_HIDE_RECEIVERS = 16; public static final int STATE_REPLY_TO_LIST = 32; public static final int STATE_OPEN = 64; + public static final int STATE_PUBLIC_ARCHIVE = 128; private static final int VISIBLE = 1; private static final int HIDDEN = 0; - private static final int DEFAULT_STATE = STATE_PENDING|STATE_HIDE_RECEIVERS; + private static final int DEFAULT_STATE = STATE_PENDING|STATE_HIDE_RECEIVERS|STATE_PUBLIC_ARCHIVE; private static final String RETAINED_FOLDER = "retained"; private final String name; private final String email; @@ -82,6 +83,10 @@ public class MailingList implements MessageHandler { this.imap = new ImapClient(imapHost,imapPort,imapUser,imapPass,inbox); } + public MailingList archive(boolean enabled) throws SQLException { + return setFlag(STATE_PUBLIC_ARCHIVE,enabled); + } + /** * create a new ML object int the database * @param email @@ -336,15 +341,10 @@ public class MailingList implements MessageHandler { return; } } - storeMessage(message); + if (hasState(STATE_PUBLIC_ARCHIVE)) storeMessage(message); forward(message); } - - - - - public MailingList open(boolean open) throws SQLException { return setFlag(STATE_OPEN,open); } @@ -492,6 +492,7 @@ public class MailingList implements MessageHandler { if (hasState(STATE_HIDE_RECEIVERS)) map.put("hide_receivers",HIDDEN); if (hasState(STATE_REPLY_TO_LIST)) map.put("reply_to_list",HIDDEN); if (hasState(STATE_OPEN)) map.put("open",VISIBLE); + if (hasState(STATE_PUBLIC_ARCHIVE)) map.put("archive",VISIBLE); return map; } @@ -571,28 +572,7 @@ public class MailingList implements MessageHandler { } private void storeMessage(Message message) { - try { - var id = message.getHeader("Message-ID")[0].replace("<","").replace(">",""); - var addr = ((InternetAddress)message.getFrom()[0]); - var fromEmail = addr.getAddress(); - var fromName = addr.getPersonal(); - if (fromName == null || fromName.isBlank()) fromName = fromEmail.split("@")[0]+"@xxxxxx"; - var subject = message.getSubject(); - var text = Util.getText(message); - JSONObject json = new JSONObject(); - json.put("id",id); - json.put("from",Map.of(EMAIL,fromEmail,NAME,fromName)); - json.put(SUBJECT,subject); - json.put(TEXT,text); - - File file = new File("/tmp/"+id+".json"); - try (var fw = new FileWriter(file)) { - json.writeJSONString(fw); - fw.flush(); - } - } catch (MessagingException | IOException e) { - e.printStackTrace(); - } + Post.create(this,message); } diff --git a/src/main/java/de/srsoftware/widerhall/data/Post.java b/src/main/java/de/srsoftware/widerhall/data/Post.java new file mode 100644 index 0000000..75eeeda --- /dev/null +++ b/src/main/java/de/srsoftware/widerhall/data/Post.java @@ -0,0 +1,169 @@ +package de.srsoftware.widerhall.data; + +import de.srsoftware.widerhall.Util; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.internet.InternetAddress; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +import static de.srsoftware.widerhall.Constants.*; +import static de.srsoftware.widerhall.Constants.VARCHAR; + +public class Post { + public static final Logger LOG = LoggerFactory.getLogger(Post.class); + public static final String TABLE_NAME = "Posts"; + private static final String FROM_ADDR = "from_addr"; + private static final String FROM_NAME = "from_name"; + private static final String PARENT = "parent"; + private static final String LONG = "LONG"; + private static final String DATE = "date"; + private static final String FILE = "file"; + private static HashMap cache = new HashMap<>(); + + private String id, fromAddr, fromName, subject, filename; + private MailingList list; + private Post parent; + private Long timestamp; + + public Post(String id, MailingList list, String fromAddr, String fromName, String subject, Long timestamp){ + this.id = id; + this.list = list; + this.fromAddr = fromAddr; + this.fromName = fromName; + this.subject = subject; + this.timestamp = timestamp; + this.filename = generateFilename(); + } + + + public static Post create(MailingList list, Message message){ + try { + var id = message.getHeader("Message-ID")[0].replace("<", "").replace(">", ""); + var addr = ((InternetAddress) message.getFrom()[0]); + var fromEmail = addr.getAddress(); + var fromName = addr.getPersonal(); + if (fromName == null || fromName.isBlank()) fromName = fromEmail.split("@")[0] + "@xxxxxx"; + var subject = message.getSubject(); + var text = Util.getText(message); + var time = message.getSentDate().getTime(); + + Post post = new Post(id,list,fromEmail,fromName,subject,time); + Files.writeString(post.file().toPath(),text, StandardCharsets.UTF_8); + return post.save(); + } catch (MessagingException | IOException | SQLException e) { + LOG.warn("Failed to create post from {}",message); + } + return null; + } + + /** + * create posts table + * @throws SQLException + */ + public static void createTable() throws SQLException { + var sql = new StringBuilder() + .append("CREATE TABLE ").append(TABLE_NAME) + .append(" (") + .append(ID).append(" ").append(VARCHAR).append(" NOT NULL PRIMARY KEY, ") + .append(LIST).append(" ").append(VARCHAR).append(", ") + .append(FROM_ADDR).append(" ").append(VARCHAR).append(", ") + .append(FROM_NAME).append(" ").append(VARCHAR).append(", ") + .append(PARENT).append(" ").append(VARCHAR).append(", ") + .append(SUBJECT).append(" ").append(VARCHAR).append(", ") + .append(DATE).append(" ").append(LONG).append(", ") + .append(FILE).append(" ").append(VARCHAR) + + .append(");"); + Database.open().query(sql).compile().run(); + } + + + public File file(){ + return new File(filename); + } + + public static HashSet find(MailingList list) throws SQLException { + var rs = Database.open().select(TABLE_NAME).where(LIST,list.email()).compile().exec(); + try { + var result = new HashSet(); + while (rs.next()) result.add(Post.from(rs)); + return result; + } finally { + rs.close(); + } + } + + private static Post from(ResultSet rs) { + try { + var id = rs.getString(ID); + var post = cache.get(id); + if (post == null) { + var list = MailingList.load(rs.getString(LIST)); + post = new Post(id, list, rs.getString(FROM_ADDR), rs.getString(FROM_NAME), rs.getString(SUBJECT), rs.getLong(DATE)); + cache.put(id,post); + } + return post; + } catch (SQLException e){ + LOG.debug("Failed to load Post from database!",e); + } + return null; + } + + + private String generateFilename() { + return "/tmp/"+id+".json"; + } + + public String id() { + return id; + } + + public static Post load(String id) throws SQLException { + var rs = Database.open().select(TABLE_NAME).where(ID,id).compile().exec(); + try { + if (rs.next()) return Post.from(rs); + } finally { + rs.close(); + } + return null; + } + + public Map map() { + return Map.of(ID,id, + LIST,list.email(), + FROM_ADDR,fromAddr, + FROM_NAME,fromName, + SUBJECT,subject, + DATE,timestamp, + FILE,filename); + } + + public Map safeMap() { + return Map.of(ID,id, + LIST,list.name(), + FROM_NAME,fromName, + SUBJECT,Util.dropEmail(subject), + DATE,timestamp); + } + + private Post save() throws SQLException { + Database.open().insertInto(TABLE_NAME).values(map()).compile().run(); + return this; + } + + public long timestamp(){ + return timestamp; + } + +} diff --git a/src/main/java/de/srsoftware/widerhall/web/Rest.java b/src/main/java/de/srsoftware/widerhall/web/Rest.java index b90367a..8c8a87d 100644 --- a/src/main/java/de/srsoftware/widerhall/web/Rest.java +++ b/src/main/java/de/srsoftware/widerhall/web/Rest.java @@ -3,6 +3,7 @@ package de.srsoftware.widerhall.web; import de.srsoftware.widerhall.Util; import de.srsoftware.widerhall.data.ListMember; import de.srsoftware.widerhall.data.MailingList; +import de.srsoftware.widerhall.data.Post; import de.srsoftware.widerhall.data.User; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -27,6 +28,7 @@ import static de.srsoftware.widerhall.Util.t; public class Rest extends HttpServlet { private static final Logger LOG = LoggerFactory.getLogger(Rest.class); + private static final String LIST_ARCHIVE = "list/archive"; private static final String LIST_DISABLE = "list/disable"; private static final String LIST_EDITABLE = "list/editable"; private static final String LIST_DETAIL = "list/detail"; @@ -127,6 +129,9 @@ public class Rest extends HttpServlet { } } else { switch (path) { + case LIST_ARCHIVE: + json.put("archive",archive(req)); + break; case LIST_SUBSCRIBABLE: json.put("lists", MailingList.subscribable().stream().map(MailingList::minimalMap).toList()); break; @@ -143,6 +148,19 @@ public class Rest extends HttpServlet { } } + private Map archive(HttpServletRequest req) { + var list = Util.getMailingList(req); + if (list != null){ + try { + return Post.find(list).stream().collect(Collectors.toMap(Post::timestamp,Post::safeMap)); + } catch (SQLException e) { + e.printStackTrace(); + } + } + LOG.debug("list: {}",list.email()); + return Map.of(); + } + public String handlePost(HttpServletRequest req, HttpServletResponse resp){ var user = Util.getUser(req); @@ -223,6 +241,7 @@ public class Rest extends HttpServlet { if (list.hasState(MailingList.STATE_HIDE_RECEIVERS)) map.put("hide_receivers",true); if (list.hasState(MailingList.STATE_REPLY_TO_LIST)) map.put("reply_to_list",true); if (list.hasState(MailingList.STATE_OPEN)) map.put("open",true); + if (list.hasState(MailingList.STATE_PUBLIC_ARCHIVE)) map.put("archive",true); return map; } diff --git a/src/main/java/de/srsoftware/widerhall/web/Web.java b/src/main/java/de/srsoftware/widerhall/web/Web.java index 516b548..698d81d 100644 --- a/src/main/java/de/srsoftware/widerhall/web/Web.java +++ b/src/main/java/de/srsoftware/widerhall/web/Web.java @@ -3,6 +3,7 @@ package de.srsoftware.widerhall.web; import de.srsoftware.widerhall.Util; import de.srsoftware.widerhall.data.ListMember; import de.srsoftware.widerhall.data.MailingList; +import de.srsoftware.widerhall.data.Post; import de.srsoftware.widerhall.data.User; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -12,6 +13,7 @@ import javax.mail.MessagingException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import java.nio.file.Files; import java.security.InvalidKeyException; import java.sql.SQLException; import java.util.HashMap; @@ -26,9 +28,11 @@ public class Web extends TemplateServlet { private static final String CONFIRM = "confirm"; private static final Logger LOG = LoggerFactory.getLogger(Web.class); private static final String ADMIN = "admin"; + private static final String ARCHIVE = "archive"; private static final String INSPECT = "inspect"; private static final String LOGIN = "login"; private static final String LOGOUT = "logout"; + private static final String POST = "post"; private static final String REGISTER = "register"; private static final String RELOAD = "reload"; private static final String SUBSCRIBE = "subscribe"; @@ -130,6 +134,12 @@ public class Web extends TemplateServlet { } } + private String archive(HttpServletRequest req, HttpServletResponse resp) { + var domain = req.getParameter(DOMAIN); + var prefix = req.getParameter(PREFIX); + return loadTemplate(ARCHIVE,Map.of(DOMAIN,domain,PREFIX,prefix),resp); + } + private String confirm(HttpServletRequest req, HttpServletResponse resp) { try { var token = req.getParameter(TOKEN); @@ -181,8 +191,12 @@ public class Web extends TemplateServlet { var list = MailingList.load(listEmail); if (list != null) data.put(LIST,list.minimalMap()); switch (path){ + case ARCHIVE: + return archive(req,resp); case CONFIRM: return confirm(req,resp); + case POST: + return post(req,resp); case RELOAD: loadTemplates(); data.put(NOTES,t("Templates have been reloaded")); @@ -223,8 +237,6 @@ public class Web extends TemplateServlet { return redirectTo(LOGIN,resp); } - - private String handleLogin(HttpServletRequest req, HttpServletResponse resp) { var email = req.getParameter("email"); var pass = req.getParameter("pass"); @@ -293,7 +305,8 @@ public class Web extends TemplateServlet { .forwardAttached(Util.getCheckbox(req, "forward_attached")) .hideReceivers(Util.getCheckbox(req, "hide_receivers")) .replyToList(Util.getCheckbox(req, "reply_to_list")) - .open(Util.getCheckbox(req,"open")); + .open(Util.getCheckbox(req,"open")) + .archive(Util.getCheckbox(req,"archive")); data.put(NOTES,t("Sucessfully updated MailingList!")); } catch (SQLException e){ LOG.warn("Failed to update MailingList:",e); @@ -304,6 +317,21 @@ public class Web extends TemplateServlet { return loadTemplate(INSPECT,data,resp); } + private String post(HttpServletRequest req, HttpServletResponse resp) { + var id = req.getParameter(ID); + if (id == null) return t("Could not find email with id!"); + try { + var post = Post.load(id); + var map = new HashMap(); + map.putAll(post.safeMap()); + String content = Files.readString(post.file().toPath()); + map.put("text",Util.dropEmail(content)); + return loadTemplate("post",map,resp); + } catch (SQLException | IOException e) { + LOG.debug("Failed to load post from file!",e); + return t("Failed to load post from file!"); + } + } private String redirectTo(String page, HttpServletResponse resp) { try { diff --git a/static/templates/archive.st b/static/templates/archive.st new file mode 100644 index 0000000..4c462b1 --- /dev/null +++ b/static/templates/archive.st @@ -0,0 +1,26 @@ + + + + + + + + + + «navigation()» + «userinfo()» + «messages()» +

Widerhall List Archive

+ + + + + + +
DateFromSubject
+ «footer()» + + + \ No newline at end of file diff --git a/static/templates/inspect.st b/static/templates/inspect.st index 071fc44..5162fbf 100644 --- a/static/templates/inspect.st +++ b/static/templates/inspect.st @@ -35,6 +35,10 @@ Hide receivers (using BCC) + diff --git a/static/templates/js.st b/static/templates/js.st index 5910ec2..cc06e1d 100644 --- a/static/templates/js.st +++ b/static/templates/js.st @@ -27,6 +27,11 @@ function hideList(listEmail){ $.post('/api/list/hide',{list:listEmail},showListResult,'json'); } +function loadArchive(domain,prefix){ + let listEmail = prefix+'@'+domain; + $.get('/api/list/archive?list='+listEmail,showListArchive,'json'); +} + function loadListDetail(listEmail){ $.post('/api/list/detail',{list:listEmail},showListDetail,'json'); } @@ -55,13 +60,26 @@ function showList(listEmail){ $.post('/api/list/show',{list:listEmail},showListResult,'json'); } +function showListArchive(data){ + for (let time in data.archive){ + let post = data.archive[time]; + let row = $(''); + var url = 'post?id='+post.id; + $('').html(''+new Date(post.date)+'').appendTo(row); + $('').html(''+post.from_name+'').appendTo(row); + $('').html(''+post.subject+'').appendTo(row); + row.appendTo($('#archive')); + console.log(post); + } +} + function showListDetail(data){ - console.log(data); if (data.forward_from) $('input[name="forward_from"]').prop('checked',true); if (data.forward_attached) $('input[name="forward_attached"]').prop('checked',true); if (data.hide_receivers) $('input[name="hide_receivers"]').prop('checked',true); if (data.reply_to_list) $('input[name="reply_to_list"]').prop('checked',true); if (data.open) $('input[name="open"]').prop('checked',true); + if (data.archive) $('input[name="archive"]').prop('checked',true); } function showListOfEditableLists(data){ @@ -112,7 +130,7 @@ function showListList(data){ for (let i in data.lists){ let list = data.lists[i]; let row = $(''); - $('').text(list.name).appendTo(row); + $('').html(''+list.name+'').appendTo(row); $('',{class:'right'}).text(list.email.prefix).appendTo(row); $('',{class:'right'}).text('@').appendTo(row); $('').text(list.email.domain).appendTo(row); diff --git a/static/templates/post.st b/static/templates/post.st new file mode 100644 index 0000000..7da6a49 --- /dev/null +++ b/static/templates/post.st @@ -0,0 +1,34 @@ + + + + + + + + + + «navigation()» + «userinfo()» + «messages()» +

Widerhall Archive:

+ + + + + + + + + + + + + + + + + +
Date«data.date»
From«data.from_name»
Subject«data.subject»
Content
«data.text»
+ «footer()» + + \ No newline at end of file