new features:

- database now ignoring duplicates isntead of failing
- automatic loading of importer plugins
- automatic tagging

Signed-off-by: Stephan Richter <s.richter@srsoftware.de>

de.srsoftware.cal.app/build.gradle.kts

@@ -1,14 +1,19 @@
 description = "OpenCloudCal : Application"
 
 dependencies {
+    implementation(project(":de.srsoftware.cal.api"))
+    implementation(project(":de.srsoftware.cal.base"))
     implementation(project(":de.srsoftware.cal.db"))
     implementation(project(":de.srsoftware.cal.importer"))
     implementation(project(":de.srsoftware.cal.web"))
 
     implementation("de.srsoftware:configuration.api:1.0.0")
     implementation("de.srsoftware:configuration.json:1.0.0")
+
     implementation("de.srsoftware:tools.http:1.2.2")
-    implementation("de.srsoftware:tools.logging:1.0.2")
+    implementation("de.srsoftware:tools.logging:1.0.3")
-    implementation("de.srsoftware:tools.web:1.3.8")
+    implementation("de.srsoftware:tools.plugin:1.0.1")
+    implementation("de.srsoftware:tools.util:1.3.0")
+    implementation("de.srsoftware:tools.web:1.3.9")
     implementation("com.mysql:mysql-connector-j:9.1.0")
 }

de.srsoftware.cal.app/src/main/java/de/srsoftware/cal/app/Application.java

@@ -5,6 +5,7 @@ import static java.lang.System.Logger.Level.*;
 
 import com.sun.net.httpserver.HttpServer;
 import de.srsoftware.cal.ApiEndpoint;
+import de.srsoftware.cal.BaseImporter;
 import de.srsoftware.cal.IndexHandler;
 import de.srsoftware.cal.StaticHandler;
 import de.srsoftware.cal.db.Database;
@@ -12,9 +13,12 @@ import de.srsoftware.cal.db.MariaDB;
 import de.srsoftware.configuration.Configuration;
 import de.srsoftware.configuration.JsonConfig;
 import de.srsoftware.tools.ColorLogger;
+import de.srsoftware.tools.plugin.JarWatchdog;
+import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.sql.SQLException;
+import java.time.Duration;
 import java.util.Optional;
 
 /**
@@ -48,7 +52,7 @@ public class Application {
 	 */
 	public static void main(String[] args) throws IOException, SQLException {
 		LOG.log(INFO, "Starting application…");
-		if (LOG instanceof ColorLogger colorLogger) colorLogger.setLogLevel(TRACE);
+		if (LOG instanceof ColorLogger colorLogger) colorLogger.setLogLevel(DEBUG);
 		JsonConfig	 jsonConfig  = new JsonConfig("OpenCloudCal");
 		Optional<String> staticPath  = jsonConfig.get("opencloudcal.static");
 		var	 db          = connect(jsonConfig);
@@ -59,11 +63,19 @@ public class Application {
 		new ApiEndpoint(db).bindPath(("/api")).on(server);
 
 		server.start();
-		scheduleImports();
-		// TODO: allow list start time as URL param
-	}
 
-	private static void scheduleImports() {
+		var jarDir = new File("/tmp/jars");
+		var autoImport = new AutoImporter(db);
+		new JarWatchdog()
+				.setContext(BaseImporter.class.getClassLoader())
+				.addDirectory(jarDir)
+				.frequency(Duration.ofSeconds(30))
+				.addListener(autoImport)
+				.afterScan(autoImport)
+				.start();
 
+		// TODO: add importers
+		// TODO: add coordinates to importers
 	}
+
 }

de.srsoftware.cal.app/src/main/java/de/srsoftware/cal/app/AutoImporter.java

@@ -0,0 +1,300 @@
+/* © SRSoftware 2024 */
+package de.srsoftware.cal.app;
+
+import static java.lang.System.Logger.Level.*;
+import static java.util.Map.entry;
+
+import de.srsoftware.cal.BaseAppointment;
+import de.srsoftware.cal.api.Appointment;
+import de.srsoftware.cal.api.Importer;
+import de.srsoftware.cal.db.Database;
+import de.srsoftware.tools.plugin.ClassListener;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.*;
+
+
+public class AutoImporter implements Runnable, ClassListener {
+	private static final System.Logger LOG = System.getLogger(AutoImporter.class.getSimpleName());
+	private static final Map<String, String> DESCRIPTION_TAGS = Map.<String, String>ofEntries(
+			entry("50s","50er"),
+			entry("50er","50er"),
+			entry("60s","60er"),
+			entry("60er","60er"),
+			entry("70s","70er"),
+			entry("70er","70er"),
+			entry("80s","80er"),
+			entry("80er","80er"),
+			entry("90s","90er"),
+			entry("90er","90er"),
+			entry("ac dc","Metal"),
+			entry("alternative rock","AlternativeRock"),
+			entry("alrauna","Alrauna"),
+			entry("ausstellung","Ausstellung"),
+			entry("bad taste","BadTaste"),
+			entry("balkan","Balkan"),
+			entry("black channel","gothic"),
+			entry("black metal","Black Metal"),
+			entry("bat cave","BatCave"),
+			entry("schwarzmetall","Black Metal"),
+			entry("blues","Blues"),
+			entry("brettspiel","Brettspiele"),
+			entry("charts","Charts"),
+			entry("circus","Zirkus"),
+			entry("comedy","Comedy"),
+			entry("cosmic dawn","CosmicDawn"),
+			entry("country","Country"),
+			entry("cover band","Konzert"),
+			entry("dancehall","Dancehall"),
+			entry("dark ambient","DarkAmbient"),
+			entry("dark electro","DarkElectro"),
+			entry("dark elektro","DarkElectro"),
+			entry("dark metal","DarkMetal"),
+			entry("dark side","Gothic"),
+			entry("dark wave","DarkWave"),
+			entry("death metal","DeathMetal"),
+			entry("demokratie","Demokratie"),
+			entry("depeche mode","DepecheMode"),
+			entry("deutsch rock","Deutschrock"),
+			entry("dia show","DiaShow"),
+			entry("disco","Disco"),
+			entry("disko","Disco"),
+			entry(" dj ","DJ"),
+			entry(" d n b","DnB"),
+			entry("doom","Doom"),
+			entry("drum and bas","DnB"),
+			entry("drschz bäm","Jazz"),
+			entry(" ebm ","EBM"),
+			entry("electro pop","ElektroPop"),
+			entry("elektro pop","ElektroPop"),
+			entry("fasching","Karneval"),
+			entry("film","Kino"),
+			entry("folk rock","FolkRock"),
+			entry("fsu jena","FSUJena"),
+			entry("funk","Funk"),
+			entry("fußball","Fußball"),
+			entry("fussball","Fußball"),
+			entry("future pop","FuturePop"),
+			entry("goth rock","GothRock"),
+			entry("gothic rock","GothRock"),
+			entry("gothic","Gothic"),
+			entry("grind core","Grindcore"),
+			entry("halloween","Halloween"),
+			entry("helloween","Halloween"),
+			entry("hard rock","HardRock"),
+			entry("heavy metal","HeavyMetal"),
+			entry("hip hop","HipHop"),
+			entry("house music","House"),
+			entry("house musik","House"),
+			entry("humor","Comedy"),
+			entry("humpa","Humppa"),
+			entry("humppa","Humppa"),
+			entry("industrial","Industrial"),
+			entry("industrial metal","IndustrialMetal"),
+			entry("jam session","JamSession"),
+			entry("jazz","Jazz"),
+			entry("karneval","Karneval"),
+			entry("kammer musik","Kammermusik"),
+			entry("kino","Kino"),
+			entry("kirsche & co","konzert"),
+			entry("knorkator","Metal"),
+			entry("konzert","Konzert"),
+			entry("land markt","Landmarkt"),
+			entry("lese bühne","Lesung"),
+			entry("liedermach","Liedermacher"),
+			entry("linux","Linux"),
+			entry("markt","Markt"),
+			entry("med club","MedClub"),
+			entry("metal","Metal"),
+			entry("modellbahn","Modellbahn"),
+			entry("modell eisenbahn","Modellbahn"),
+			entry(" ndh ","NDH"),
+			entry("open air","OpenAir"),
+			entry("open mic","OpenMic"),
+			entry("pagan metal","PaganMetal"),
+			entry("party","Party"),
+			entry("polka","Polka"),
+			entry("power metal","PowerMetal"),
+			entry("progressiv","Progressive"),
+			entry("progressive metal","ProgressiveMetal"),
+			entry("psychedeli","Psychedelic"),
+			entry("punk ","Punk"),
+			entry("punk rock","PunkRock"),
+			entry("punks ","Punk"),
+			entry("rap ","Rap"),
+			entry(" rave ","Rave"),
+			entry(" rnb","RnB"),
+			entry("rock","Rock"),
+			entry("rock n roll","RockNRoll"),
+			entry("rollen spiel","Rollenspiel"),
+			entry("schatten tanz","SchwarzesJena"),
+			entry("schwarzer tanz","Gothic"),
+			entry("seminar","Seminar"),
+			entry("silvester","Silvester"),
+			entry("songwrit","Liedermacher"),
+			entry("southern rock","SouthernRock"),
+			entry("stoner","Stoner"),
+			entry("software","Software"),
+			entry("symphonic metal","SymphonicMetal"),
+			entry("syntension","ProgressiveMetal"),
+			entry("synthie pop","SynthiePop"),
+			entry("tanz","Tanzen"),
+			entry("techno","Techno"),
+			entry("windows","Windows"),
+			entry("volleyball","Volleyball"),
+			entry("universität","Universität"),
+			entry("viking metal","WikingMetal"),
+			entry("workshop","Workshop"),
+			entry("zirkus","Zirkus")
+	);
+	private static final Map<String,String> TITLE_TAGS = Map.of("festival","Festival");
+
+	private static final Map<String,Set<String>> TAG_EXPANSION = Map.ofEntries(
+			entry("AlternativeRock",Set.of("Rock")),
+			entry("Alrauna",Set.of("FolkMetal","Jena","SchwarzesJena")),
+			entry("BatCave",Set.of("Gothic","SchwarzesJena")),
+			entry("BlackMetal",Set.of("Metal")),
+			entry("DarkAmbient",Set.of("SchwarzesJena")),
+			entry("DarkElectro",Set.of("SchwarzesJena")),
+			entry("DarkMetal",Set.of("SchwarzesJena")),
+			entry("DarkWave",Set.of("SchwarzesJena")),
+			entry("DeathMetal",Set.of("Metal","SchwarzesJena")),
+			entry("Demokratie",Set.of("Politik")),
+			entry("DepecheMode",Set.of("SchwarzesJena","Wave")),
+			entry("Doom",Set.of("SchwarzesJena")),
+			entry("EBM",Set.of("SchwarzesJena")),
+			entry("Festival",Set.of("Konzert")),
+			entry("FolkRock",Set.of("Folk","Rock","SchwarzesJena")),
+			entry("FolkMetal",Set.of("Folk","Metal","SchwarzesJena")),
+			entry("FSUJena",Set.of("Universität","Jena")),
+			entry("Fußball",Set.of("Sport")),
+			entry("Gothic",Set.of("SchwarzesJena")),
+			entry("GothicMetal",Set.of("Gothic","Metal","SchwarzesJena")),
+			entry("GothMetal",Set.of("Gothic\",\"Metal\",\"SchwarzesJena")),
+			entry("GothicRock",Set.of("Gothic","Rock","SchwarzesJena")),
+			entry("GothRock",Set.of("Gothic\",\"Rock\",\"SchwarzesJena")),
+			entry("HardRock",Set.of("Rock","SchwarzesJena")),
+			entry("HeavyMetal",Set.of("Metal","SchwarzesJena")),
+			entry("Industrial",Set.of("SchwarzesJena")),
+			entry("IndustrialMetal",Set.of("Industrial","Metal","SchwarzesJena")),
+			entry("Metal",Set.of("SchwarzesJena")),
+			entry("OpenAir",Set.of("Festival","Konzert")),
+			entry("Liedermacher",Set.of("Konzert")),
+			entry("NDH",Set.of("SchwarzesJena")),
+			entry("PowerMetal",Set.of("Metal","SchwarzesJena")),
+			entry("ProgressiveMetal",Set.of("Progressive","Metal","SchwarzesJena")),
+			entry("PunkRock",Set.of("Punk","Rock")),
+			entry("Rap",Set.of("HipHop")),
+			entry("RockNRoll",Set.of("Rock")),
+			entry("Stoner",Set.of("Rock")),
+			entry("SymphonicMetal",Set.of("Symphonic","Metal","SchwarzesJena")),
+			entry("VikinMetal",Set.of("Pagan","Metal","SchwarzesJena")),
+			entry("volleyball",Set.of("Sport")),
+			entry("Wave",Set.of("SchwarzesJena"))
+	);
+
+	private final Set<Importer> importers = new HashSet<>();
+	private final Database db;
+	private LocalDateTime lastImport = null;
+
+	public AutoImporter(Database db) {
+		this.db = db;
+	}
+
+
+	@Override
+	public void run() {
+		var now = LocalDateTime.now();
+		if (lastImport == null || Duration.between(lastImport,now).minusHours(6).isPositive()) {
+			for (var importer : importers) {
+				try {
+					importer.fetch().map(AutoImporter::autoAddTags).forEach(this::saveOrUpdate);
+				} catch (IOException e) {
+					LOG.log(WARNING, "{0}.fetch() failed", importer, e);
+				}
+			}
+			lastImport = LocalDateTime.now();
+		}
+	}
+
+	private static String normalize(String wild){
+		if (wild == null) return "";
+		return wild.replaceAll("<[^>]+>","").replaceAll("[^\\wÄäÖöÜüß]+"," ").toLowerCase();
+	}
+
+	private static Appointment autoAddTags(Appointment appointment) {
+		LOG.log(DEBUG,"tags before auto detection: {0}",appointment.tags());
+		if (!(appointment instanceof BaseAppointment baseAppointment)) return appointment;
+		var additionalTags = new HashSet<String>();
+		var description = normalize(appointment.description());
+		var title = normalize(appointment.title());
+		for (var entry : DESCRIPTION_TAGS.entrySet()){
+			var key = entry.getKey();
+			if (title.contains(key) || description.contains(key)) {
+				additionalTags.add(entry.getValue());
+				continue;
+			}
+			if (key.contains(" ")) {
+				// keep spaces at start and end, remove all others
+				key = key.charAt(0) + key.substring(1, key.length() - 1).replace(" ", "") + key.charAt(key.length() - 1);
+				if (title.contains(key) || description.contains(key)) additionalTags.add(entry.getValue());
+			}
+		}
+		var more = new HashSet<String>();
+		for (var entry : TAG_EXPANSION.entrySet()){
+			if (additionalTags.contains(entry.getKey())) more.addAll(entry.getValue());
+		}
+		baseAppointment.tags(additionalTags).tags(more);
+		LOG.log(DEBUG,"tags after auto detection: {0}",baseAppointment.tags());
+		return baseAppointment;
+	}
+
+	private void saveOrUpdate(Appointment appointment) {
+		var location = appointment.location();
+		if (location.isEmpty()){
+			LOG.log(WARNING,"Skipping %s, since this event has no location!");
+			return;
+		}
+		var res = db.loadEvent(location.get(),appointment.start());
+		var loaded = res.optional();
+		if (loaded.isPresent()){
+			res = db.update(appointment.clone(loaded.get().id()));
+		} else {
+			res = db.add(appointment);
+		}
+		var saved = res.optional();
+		if (saved.isPresent()){
+			if (loaded.isPresent()) {
+				LOG.log(INFO, "UPDATED: {0}.", saved.get());
+			} else {
+				LOG.log(INFO, "CREATED: {0}.", saved.get());
+			}
+		} else {
+			LOG.log(WARNING,"Failed to save {0}!",appointment);
+		}
+
+	}
+
+	@Override
+	public void classAdded(Class<?> aClass) {
+		if (Importer.class.isAssignableFrom(aClass)) try {
+			var instance = aClass.getDeclaredConstructor().newInstance();
+			importers.add((Importer) instance);
+			LOG.log(INFO,"Added {0} to the list of importers. Will be used soon…",instance);
+			lastImport = null;
+		} catch (InvocationTargetException | InstantiationException | IllegalAccessException | NoSuchMethodException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	@Override
+	public void classRemoved(Class<?> aClass) {
+		var removing = new HashSet<Importer>();
+		for (var importer : importers){
+			if (importer.getClass().isAssignableFrom(aClass)) removing.add(importer);
+		}
+		importers.removeAll(removing);
+	}
+}

de.srsoftware.cal.base/build.gradle.kts

@@ -5,6 +5,6 @@ dependencies {
 
     implementation("de.srsoftware:tools.optionals:1.0.0")
     implementation("de.srsoftware:tools.util:1.3.0")
-    implementation("de.srsoftware:tools.web:1.3.8")
+    implementation("de.srsoftware:tools.web:1.3.9")
     implementation("org.json:json:20240303")
 }

de.srsoftware.cal.base/src/main/java/de/srsoftware/cal/BaseImporter.java

@@ -46,6 +46,7 @@ public abstract class BaseImporter implements Importer {
 		    .flatMap(tag -> tag.find(TagFilter.ofType("img")).stream())
 		    .map(tag -> tag.get("src"))
 		    .filter(Objects::nonNull)
+			.map(url -> url.contains("://") ? url : baseUrl()+url)
 		    .map(Payload::of)
 		    .map(BaseImporter::url)
 		    .map(BaseImporter::toAttachment)
@@ -59,9 +60,11 @@ public abstract class BaseImporter implements Importer {
 	}
 
 	protected Result<String> extractDescription(Tag eventTag) {
-		Result<Tag> titleTag = extractDescriptionTag(eventTag);
+		Result<Tag> descriptionTag = extractDescriptionTag(eventTag);
-		if (titleTag.optional().isEmpty()) return transform(titleTag);
+		if (descriptionTag.optional().isEmpty()) return transform(descriptionTag);
-		var inner = titleTag.optional().flatMap(tag -> tag.inner(2));
+		Tag tag = descriptionTag.optional().get();
+		tag.find(t -> t.is("iframe")).forEach(Tag::remove);
+		var inner = tag.inner(2);
 		if (inner.isPresent()) return Payload.of(inner.get());
 		return error("No description found");
 	}
@@ -105,6 +108,7 @@ public abstract class BaseImporter implements Importer {
 		var event = new BaseAppointment(id, title, description, start, end, location)  //
 			.add(extractAttachments(eventTag))
 			.addLinks(extractLinks(eventTag))
+			.addLinks(eventPage)
 			.tags(extractTags(eventTag));
 
 		extractCoords(eventTag).optional().ifPresent(event::coords);
@@ -184,13 +188,13 @@ public abstract class BaseImporter implements Importer {
 		@Override
 		public Stream<Appointment> fetch() {
 			var	       url    = Payload.of(programURL());
-			Stream<Result<String>> stream = url(url)
+			Stream<Result<String>> urls = url(url)
 					    .map(this::open)  //
 					    .map(this::preload)
 					    .map(this::parseXML)
 					    .map(this::extractEventUrls)
 					    .stream();
-			return stream  //
+			return urls  //
 			    .map(BaseImporter::url)
 			    .map(this::loadEvent)
 			    .peek(e -> {

de.srsoftware.cal.db/build.gradle.kts

@@ -4,7 +4,7 @@ dependencies {
     implementation(project(":de.srsoftware.cal.api"))
     implementation(project(":de.srsoftware.cal.base"))
 
-    implementation("de.srsoftware:tools.jdbc:1.1.3")
+    implementation("de.srsoftware:tools.jdbc:1.1.4")
     implementation("de.srsoftware:tools.optionals:1.0.0")
     implementation("de.srsoftware:tools.util:1.3.0")
 }

de.srsoftware.cal.db/src/main/java/de/srsoftware/cal/db/MariaDB.java

@@ -78,7 +78,7 @@ public class MariaDB implements Database {
 					if (assignQuery == null) assignQuery = insertInto(APPOINTMENT_ATTACHMENTS, AID, UID, MIME);
 					if (urlId.isPresent()) assignQuery.values(saved.id(), urlId.get(), attachment.mime());
 				}
-				if (assignQuery != null) assignQuery.execute(connection);
+				if (assignQuery != null) assignQuery.ignoreDuplicates().execute(connection);
 			}
 
 			{  // link to links
@@ -89,7 +89,7 @@ public class MariaDB implements Database {
 					if (assignQuery == null) assignQuery = insertInto(APPOINTMENT_URLS, AID, UID, DESCRIPTION);
 					if (urlId.isPresent()) assignQuery.values(saved.id(), urlId.get(), link.desciption());
 				}
-				if (assignQuery != null) assignQuery.execute(connection);
+				if (assignQuery != null) assignQuery.ignoreDuplicates().execute(connection);
 			}
 
 			{
@@ -100,7 +100,7 @@ public class MariaDB implements Database {
 					if (assignQuery == null) assignQuery = insertInto(APPOINTMENT_TAGS, AID, TID);
 					if (tagId.isPresent()) assignQuery.values(saved.id(), tagId.get());
 				}
-				if (assignQuery != null) assignQuery.execute(connection);
+				if (assignQuery != null) assignQuery.ignoreDuplicates().execute(connection);
 			}
 			return Payload.of(saved);
 		} catch (SQLException e) {

de.srsoftware.cal.importer/build.gradle.kts

@@ -5,5 +5,5 @@ dependencies {
     implementation(project(":de.srsoftware.cal.base"))
     implementation("de.srsoftware:tools.optionals:1.0.0")
     implementation("de.srsoftware:tools.util:1.3.0")
-    implementation("de.srsoftware:tools.web:1.3.8")
+    implementation("de.srsoftware:tools.web:1.3.9")
 }

de.srsoftware.cal.importer/src/main/java/de/srsoftware/cal/importer/jena/Kassablanca.java

@@ -90,7 +90,7 @@ public class Kassablanca extends BaseImporter {
 
 	@Override
 	protected List<String> extractTags(Tag eventTag) {
-		return List.of("Kassablanca");
+		return List.of("Kassablanca","Jena");
 	}
 
 	@Override

de.srsoftware.cal.web/src/main/resources/event.html

@@ -9,7 +9,6 @@
         <script src="/static/script/leaflet.js"></script>
     </head>
     <body class="event">
-        <nav />
         <span id="buttons">
             <button title="create a copy of this event" onclick="window.top.location.href = location.href.replace('/static/event','/static/edit').replace('id=','copy=')">⧉ copy</button>
             <button title="edit the details of this event" onclick="window.top.location.href = location.href.replace('/static/event','/static/edit')">✍ edit</button>