Browse Source
- database now ignoring duplicates isntead of failing - automatic loading of importer plugins - automatic tagging Signed-off-by: Stephan Richter <s.richter@srsoftware.de>main
10 changed files with 340 additions and 20 deletions
@ -1,14 +1,19 @@
@@ -1,14 +1,19 @@
|
||||
description = "OpenCloudCal : Application" |
||||
|
||||
dependencies { |
||||
implementation(project(":de.srsoftware.cal.api")) |
||||
implementation(project(":de.srsoftware.cal.base")) |
||||
implementation(project(":de.srsoftware.cal.db")) |
||||
implementation(project(":de.srsoftware.cal.importer")) |
||||
implementation(project(":de.srsoftware.cal.web")) |
||||
|
||||
implementation("de.srsoftware:configuration.api:1.0.0") |
||||
implementation("de.srsoftware:configuration.json:1.0.0") |
||||
|
||||
implementation("de.srsoftware:tools.http:1.2.2") |
||||
implementation("de.srsoftware:tools.logging:1.0.2") |
||||
implementation("de.srsoftware:tools.web:1.3.8") |
||||
implementation("de.srsoftware:tools.logging:1.0.3") |
||||
implementation("de.srsoftware:tools.plugin:1.0.1") |
||||
implementation("de.srsoftware:tools.util:1.3.0") |
||||
implementation("de.srsoftware:tools.web:1.3.9") |
||||
implementation("com.mysql:mysql-connector-j:9.1.0") |
||||
} |
||||
|
||||
@ -0,0 +1,300 @@
@@ -0,0 +1,300 @@
|
||||
/* © SRSoftware 2024 */ |
||||
package de.srsoftware.cal.app; |
||||
|
||||
import static java.lang.System.Logger.Level.*; |
||||
import static java.util.Map.entry; |
||||
|
||||
import de.srsoftware.cal.BaseAppointment; |
||||
import de.srsoftware.cal.api.Appointment; |
||||
import de.srsoftware.cal.api.Importer; |
||||
import de.srsoftware.cal.db.Database; |
||||
import de.srsoftware.tools.plugin.ClassListener; |
||||
import java.io.IOException; |
||||
import java.lang.reflect.InvocationTargetException; |
||||
import java.time.Duration; |
||||
import java.time.LocalDateTime; |
||||
import java.util.*; |
||||
|
||||
|
||||
public class AutoImporter implements Runnable, ClassListener { |
||||
private static final System.Logger LOG = System.getLogger(AutoImporter.class.getSimpleName()); |
||||
private static final Map<String, String> DESCRIPTION_TAGS = Map.<String, String>ofEntries( |
||||
entry("50s","50er"), |
||||
entry("50er","50er"), |
||||
entry("60s","60er"), |
||||
entry("60er","60er"), |
||||
entry("70s","70er"), |
||||
entry("70er","70er"), |
||||
entry("80s","80er"), |
||||
entry("80er","80er"), |
||||
entry("90s","90er"), |
||||
entry("90er","90er"), |
||||
entry("ac dc","Metal"), |
||||
entry("alternative rock","AlternativeRock"), |
||||
entry("alrauna","Alrauna"), |
||||
entry("ausstellung","Ausstellung"), |
||||
entry("bad taste","BadTaste"), |
||||
entry("balkan","Balkan"), |
||||
entry("black channel","gothic"), |
||||
entry("black metal","Black Metal"), |
||||
entry("bat cave","BatCave"), |
||||
entry("schwarzmetall","Black Metal"), |
||||
entry("blues","Blues"), |
||||
entry("brettspiel","Brettspiele"), |
||||
entry("charts","Charts"), |
||||
entry("circus","Zirkus"), |
||||
entry("comedy","Comedy"), |
||||
entry("cosmic dawn","CosmicDawn"), |
||||
entry("country","Country"), |
||||
entry("cover band","Konzert"), |
||||
entry("dancehall","Dancehall"), |
||||
entry("dark ambient","DarkAmbient"), |
||||
entry("dark electro","DarkElectro"), |
||||
entry("dark elektro","DarkElectro"), |
||||
entry("dark metal","DarkMetal"), |
||||
entry("dark side","Gothic"), |
||||
entry("dark wave","DarkWave"), |
||||
entry("death metal","DeathMetal"), |
||||
entry("demokratie","Demokratie"), |
||||
entry("depeche mode","DepecheMode"), |
||||
entry("deutsch rock","Deutschrock"), |
||||
entry("dia show","DiaShow"), |
||||
entry("disco","Disco"), |
||||
entry("disko","Disco"), |
||||
entry(" dj ","DJ"), |
||||
entry(" d n b","DnB"), |
||||
entry("doom","Doom"), |
||||
entry("drum and bas","DnB"), |
||||
entry("drschz bäm","Jazz"), |
||||
entry(" ebm ","EBM"), |
||||
entry("electro pop","ElektroPop"), |
||||
entry("elektro pop","ElektroPop"), |
||||
entry("fasching","Karneval"), |
||||
entry("film","Kino"), |
||||
entry("folk rock","FolkRock"), |
||||
entry("fsu jena","FSUJena"), |
||||
entry("funk","Funk"), |
||||
entry("fußball","Fußball"), |
||||
entry("fussball","Fußball"), |
||||
entry("future pop","FuturePop"), |
||||
entry("goth rock","GothRock"), |
||||
entry("gothic rock","GothRock"), |
||||
entry("gothic","Gothic"), |
||||
entry("grind core","Grindcore"), |
||||
entry("halloween","Halloween"), |
||||
entry("helloween","Halloween"), |
||||
entry("hard rock","HardRock"), |
||||
entry("heavy metal","HeavyMetal"), |
||||
entry("hip hop","HipHop"), |
||||
entry("house music","House"), |
||||
entry("house musik","House"), |
||||
entry("humor","Comedy"), |
||||
entry("humpa","Humppa"), |
||||
entry("humppa","Humppa"), |
||||
entry("industrial","Industrial"), |
||||
entry("industrial metal","IndustrialMetal"), |
||||
entry("jam session","JamSession"), |
||||
entry("jazz","Jazz"), |
||||
entry("karneval","Karneval"), |
||||
entry("kammer musik","Kammermusik"), |
||||
entry("kino","Kino"), |
||||
entry("kirsche & co","konzert"), |
||||
entry("knorkator","Metal"), |
||||
entry("konzert","Konzert"), |
||||
entry("land markt","Landmarkt"), |
||||
entry("lese bühne","Lesung"), |
||||
entry("liedermach","Liedermacher"), |
||||
entry("linux","Linux"), |
||||
entry("markt","Markt"), |
||||
entry("med club","MedClub"), |
||||
entry("metal","Metal"), |
||||
entry("modellbahn","Modellbahn"), |
||||
entry("modell eisenbahn","Modellbahn"), |
||||
entry(" ndh ","NDH"), |
||||
entry("open air","OpenAir"), |
||||
entry("open mic","OpenMic"), |
||||
entry("pagan metal","PaganMetal"), |
||||
entry("party","Party"), |
||||
entry("polka","Polka"), |
||||
entry("power metal","PowerMetal"), |
||||
entry("progressiv","Progressive"), |
||||
entry("progressive metal","ProgressiveMetal"), |
||||
entry("psychedeli","Psychedelic"), |
||||
entry("punk ","Punk"), |
||||
entry("punk rock","PunkRock"), |
||||
entry("punks ","Punk"), |
||||
entry("rap ","Rap"), |
||||
entry(" rave ","Rave"), |
||||
entry(" rnb","RnB"), |
||||
entry("rock","Rock"), |
||||
entry("rock n roll","RockNRoll"), |
||||
entry("rollen spiel","Rollenspiel"), |
||||
entry("schatten tanz","SchwarzesJena"), |
||||
entry("schwarzer tanz","Gothic"), |
||||
entry("seminar","Seminar"), |
||||
entry("silvester","Silvester"), |
||||
entry("songwrit","Liedermacher"), |
||||
entry("southern rock","SouthernRock"), |
||||
entry("stoner","Stoner"), |
||||
entry("software","Software"), |
||||
entry("symphonic metal","SymphonicMetal"), |
||||
entry("syntension","ProgressiveMetal"), |
||||
entry("synthie pop","SynthiePop"), |
||||
entry("tanz","Tanzen"), |
||||
entry("techno","Techno"), |
||||
entry("windows","Windows"), |
||||
entry("volleyball","Volleyball"), |
||||
entry("universität","Universität"), |
||||
entry("viking metal","WikingMetal"), |
||||
entry("workshop","Workshop"), |
||||
entry("zirkus","Zirkus") |
||||
); |
||||
private static final Map<String,String> TITLE_TAGS = Map.of("festival","Festival"); |
||||
|
||||
private static final Map<String,Set<String>> TAG_EXPANSION = Map.ofEntries( |
||||
entry("AlternativeRock",Set.of("Rock")), |
||||
entry("Alrauna",Set.of("FolkMetal","Jena","SchwarzesJena")), |
||||
entry("BatCave",Set.of("Gothic","SchwarzesJena")), |
||||
entry("BlackMetal",Set.of("Metal")), |
||||
entry("DarkAmbient",Set.of("SchwarzesJena")), |
||||
entry("DarkElectro",Set.of("SchwarzesJena")), |
||||
entry("DarkMetal",Set.of("SchwarzesJena")), |
||||
entry("DarkWave",Set.of("SchwarzesJena")), |
||||
entry("DeathMetal",Set.of("Metal","SchwarzesJena")), |
||||
entry("Demokratie",Set.of("Politik")), |
||||
entry("DepecheMode",Set.of("SchwarzesJena","Wave")), |
||||
entry("Doom",Set.of("SchwarzesJena")), |
||||
entry("EBM",Set.of("SchwarzesJena")), |
||||
entry("Festival",Set.of("Konzert")), |
||||
entry("FolkRock",Set.of("Folk","Rock","SchwarzesJena")), |
||||
entry("FolkMetal",Set.of("Folk","Metal","SchwarzesJena")), |
||||
entry("FSUJena",Set.of("Universität","Jena")), |
||||
entry("Fußball",Set.of("Sport")), |
||||
entry("Gothic",Set.of("SchwarzesJena")), |
||||
entry("GothicMetal",Set.of("Gothic","Metal","SchwarzesJena")), |
||||
entry("GothMetal",Set.of("Gothic\",\"Metal\",\"SchwarzesJena")), |
||||
entry("GothicRock",Set.of("Gothic","Rock","SchwarzesJena")), |
||||
entry("GothRock",Set.of("Gothic\",\"Rock\",\"SchwarzesJena")), |
||||
entry("HardRock",Set.of("Rock","SchwarzesJena")), |
||||
entry("HeavyMetal",Set.of("Metal","SchwarzesJena")), |
||||
entry("Industrial",Set.of("SchwarzesJena")), |
||||
entry("IndustrialMetal",Set.of("Industrial","Metal","SchwarzesJena")), |
||||
entry("Metal",Set.of("SchwarzesJena")), |
||||
entry("OpenAir",Set.of("Festival","Konzert")), |
||||
entry("Liedermacher",Set.of("Konzert")), |
||||
entry("NDH",Set.of("SchwarzesJena")), |
||||
entry("PowerMetal",Set.of("Metal","SchwarzesJena")), |
||||
entry("ProgressiveMetal",Set.of("Progressive","Metal","SchwarzesJena")), |
||||
entry("PunkRock",Set.of("Punk","Rock")), |
||||
entry("Rap",Set.of("HipHop")), |
||||
entry("RockNRoll",Set.of("Rock")), |
||||
entry("Stoner",Set.of("Rock")), |
||||
entry("SymphonicMetal",Set.of("Symphonic","Metal","SchwarzesJena")), |
||||
entry("VikinMetal",Set.of("Pagan","Metal","SchwarzesJena")), |
||||
entry("volleyball",Set.of("Sport")), |
||||
entry("Wave",Set.of("SchwarzesJena")) |
||||
); |
||||
|
||||
private final Set<Importer> importers = new HashSet<>(); |
||||
private final Database db; |
||||
private LocalDateTime lastImport = null; |
||||
|
||||
public AutoImporter(Database db) { |
||||
this.db = db; |
||||
} |
||||
|
||||
|
||||
@Override |
||||
public void run() { |
||||
var now = LocalDateTime.now(); |
||||
if (lastImport == null || Duration.between(lastImport,now).minusHours(6).isPositive()) { |
||||
for (var importer : importers) { |
||||
try { |
||||
importer.fetch().map(AutoImporter::autoAddTags).forEach(this::saveOrUpdate); |
||||
} catch (IOException e) { |
||||
LOG.log(WARNING, "{0}.fetch() failed", importer, e); |
||||
} |
||||
} |
||||
lastImport = LocalDateTime.now(); |
||||
} |
||||
} |
||||
|
||||
private static String normalize(String wild){ |
||||
if (wild == null) return ""; |
||||
return wild.replaceAll("<[^>]+>","").replaceAll("[^\\wÄäÖöÜüß]+"," ").toLowerCase(); |
||||
} |
||||
|
||||
private static Appointment autoAddTags(Appointment appointment) { |
||||
LOG.log(DEBUG,"tags before auto detection: {0}",appointment.tags()); |
||||
if (!(appointment instanceof BaseAppointment baseAppointment)) return appointment; |
||||
var additionalTags = new HashSet<String>(); |
||||
var description = normalize(appointment.description()); |
||||
var title = normalize(appointment.title()); |
||||
for (var entry : DESCRIPTION_TAGS.entrySet()){ |
||||
var key = entry.getKey(); |
||||
if (title.contains(key) || description.contains(key)) { |
||||
additionalTags.add(entry.getValue()); |
||||
continue; |
||||
} |
||||
if (key.contains(" ")) { |
||||
// keep spaces at start and end, remove all others
|
||||
key = key.charAt(0) + key.substring(1, key.length() - 1).replace(" ", "") + key.charAt(key.length() - 1); |
||||
if (title.contains(key) || description.contains(key)) additionalTags.add(entry.getValue()); |
||||
} |
||||
} |
||||
var more = new HashSet<String>(); |
||||
for (var entry : TAG_EXPANSION.entrySet()){ |
||||
if (additionalTags.contains(entry.getKey())) more.addAll(entry.getValue()); |
||||
} |
||||
baseAppointment.tags(additionalTags).tags(more); |
||||
LOG.log(DEBUG,"tags after auto detection: {0}",baseAppointment.tags()); |
||||
return baseAppointment; |
||||
} |
||||
|
||||
private void saveOrUpdate(Appointment appointment) { |
||||
var location = appointment.location(); |
||||
if (location.isEmpty()){ |
||||
LOG.log(WARNING,"Skipping %s, since this event has no location!"); |
||||
return; |
||||
} |
||||
var res = db.loadEvent(location.get(),appointment.start()); |
||||
var loaded = res.optional(); |
||||
if (loaded.isPresent()){ |
||||
res = db.update(appointment.clone(loaded.get().id())); |
||||
} else { |
||||
res = db.add(appointment); |
||||
} |
||||
var saved = res.optional(); |
||||
if (saved.isPresent()){ |
||||
if (loaded.isPresent()) { |
||||
LOG.log(INFO, "UPDATED: {0}.", saved.get()); |
||||
} else { |
||||
LOG.log(INFO, "CREATED: {0}.", saved.get()); |
||||
} |
||||
} else { |
||||
LOG.log(WARNING,"Failed to save {0}!",appointment); |
||||
} |
||||
|
||||
} |
||||
|
||||
@Override |
||||
public void classAdded(Class<?> aClass) { |
||||
if (Importer.class.isAssignableFrom(aClass)) try { |
||||
var instance = aClass.getDeclaredConstructor().newInstance(); |
||||
importers.add((Importer) instance); |
||||
LOG.log(INFO,"Added {0} to the list of importers. Will be used soon…",instance); |
||||
lastImport = null; |
||||
} catch (InvocationTargetException | InstantiationException | IllegalAccessException | NoSuchMethodException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public void classRemoved(Class<?> aClass) { |
||||
var removing = new HashSet<Importer>(); |
||||
for (var importer : importers){ |
||||
if (importer.getClass().isAssignableFrom(aClass)) removing.add(importer); |
||||
} |
||||
importers.removeAll(removing); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue