minor tweak to Wotufa importer, added AtParty importer

Signed-off-by: Stephan Richter <s.richter@srsoftware.de>

de.srsoftware.cal.app/build.gradle.kts

@@ -11,7 +11,7 @@ dependencies {
     implementation("de.srsoftware:configuration.json:1.0.0")
 
     implementation("de.srsoftware:tools.http:1.2.2")
-    implementation("de.srsoftware:tools.logging:1.0.3")
+    implementation("de.srsoftware:tools.logging:1.0.5")
     implementation("de.srsoftware:tools.plugin:1.0.1")
     implementation("de.srsoftware:tools.util:1.3.0")
     implementation("de.srsoftware:tools.web:1.3.14")

de.srsoftware.cal.app/src/main/java/de/srsoftware/cal/app/Application.java

@@ -52,7 +52,9 @@ public class Application {
 	 */
 	public static void main(String[] args) throws IOException, SQLException {
 		LOG.log(INFO, "Starting application…");
-		if (LOG instanceof ColorLogger colorLogger) colorLogger.setLogLevel(DEBUG);
+		ColorLogger.setRootLogLevel(DEBUG);
+		ColorLogger.setLogLevel("jdk.event.security",INFO);
+		ColorLogger.setLogLevel("sun.net.www.protocol.http.HttpURLConnection",INFO);
 		JsonConfig	 jsonConfig  = new JsonConfig("OpenCloudCal");
 		Optional<String> staticPath  = jsonConfig.get("opencloudcal.static");
 		var	 db          = connect(jsonConfig);

de.srsoftware.cal.app/src/main/java/de/srsoftware/cal/app/AutoImporter.java

@@ -209,7 +209,7 @@ public class AutoImporter implements Runnable, ClassListener {
 		if (lastImport == null || Duration.between(lastImport,now).minusHours(6).isPositive()) {
 			for (var importer : importers) {
 				try {
-					importer.fetch().map(AutoImporter::autoAddTags).forEach(this::saveOrUpdate);
+					importer.fetch().map(AutoImporter::autoAddTags).forEach(event -> saveOrUpdate(event,importer));
 				} catch (IOException e) {
 					LOG.log(WARNING, "{0}.fetch() failed", importer, e);
 				}
@@ -251,7 +251,7 @@ public class AutoImporter implements Runnable, ClassListener {
 		return baseAppointment;
 	}
 
-	private void saveOrUpdate(Appointment appointment) {
+	private void saveOrUpdate(Appointment appointment, Importer importer) {
 		var location = appointment.location();
 		if (location.isEmpty()){
 			LOG.log(WARNING,"Skipping %s, since this event has no location!");
@@ -267,12 +267,12 @@ public class AutoImporter implements Runnable, ClassListener {
 		var saved = res.optional();
 		if (saved.isPresent()){
 			if (loaded.isPresent()) {
-				LOG.log(INFO, "UPDATED: {0}.", saved.get());
+				LOG.log(INFO, "UPDATED: {0} from {1}.", saved.get(),importer.getClass().getSimpleName());
 			} else {
-				LOG.log(INFO, "CREATED: {0}.", saved.get());
+				LOG.log(INFO, "CREATED: {0} from {1}.", saved.get(),importer.getClass().getSimpleName());
 			}
 		} else {
-			LOG.log(WARNING,"Failed to save {0}!",appointment);
+			LOG.log(WARNING,"Failed to save {0} from {1}!",appointment,importer.getClass().getSimpleName());
 		}
 
 	}

de.srsoftware.cal.base/src/main/java/de/srsoftware/cal/Util.java

@@ -38,8 +38,9 @@ public class Util {
 	public static final String VCALENDAR = "VCALENDAR";
 
 	public static final Pattern GERMAN_DATE_PATTERN = Pattern.compile("^\\D*(\\d\\d?)\\.(\\d\\d?)\\.(\\d{4})\\D");
+	public static final Pattern GERMAN_DATE_PATTERN_LONG = Pattern.compile("(\\d\\d?)\\.\\s*(\\w+)\\s+(\\d{4})\\D");
 	public static final Pattern GERMAN_TIME_PATTERN = Pattern.compile("(\\d\\d?):(\\d\\d?)(:(\\d\\d?))?\\D");
-	private static final Pattern BG_IMAGE_URL = Pattern.compile("background(-image)?:\\surl\\(([^)]+)\\)");
+	private static final Pattern BG_IMAGE_URL = Pattern.compile("background(-image)?:\\surl\\('?([^)]+)'?\\)");
 	private static final System.Logger LOG    = System.getLogger(Util.class.getSimpleName());
 
 	private Util(){}
@@ -110,6 +111,19 @@ public class Util {
 		return error("Failed to find date");
 	}
 
+	public static Result<LocalDate> parseLongGermanDate(String s){
+		var match = GERMAN_DATE_PATTERN_LONG.matcher(" "+s+" ");
+		if (match.find()){
+			var day = Integer.parseInt(match.group(1));
+			var res = toNumericMonth(match.group(2));
+			var month = res.optional();
+			if (month.isEmpty()) return transform(res);
+			var year = Integer.parseInt(match.group(3));
+			return Payload.of(LocalDate.of(year,month.get(),day));
+		}
+		return error("Failed to find date");
+	}
+
 	public static Result<LocalTime> parseGermanTime(String s){
 		var match = GERMAN_TIME_PATTERN.matcher(s);
 		if (match.find()){

de.srsoftware.cal.importer/src/main/java/de/srsoftware/cal/importer/gera/AtParty.java

@@ -0,0 +1,229 @@
+package de.srsoftware.cal.importer.gera;
+
+import de.srsoftware.cal.BaseImporter;
+import de.srsoftware.cal.Util;
+import de.srsoftware.cal.api.Attachment;
+import de.srsoftware.cal.api.Coords;
+import de.srsoftware.tools.Payload;
+import de.srsoftware.tools.Result;
+import de.srsoftware.tools.Tag;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.NoSuchAlgorithmException;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Predicate;
+
+import static de.srsoftware.cal.Util.*;
+import static de.srsoftware.tools.Error.error;
+import static de.srsoftware.tools.Optionals.emptyIfNull;
+import static de.srsoftware.tools.Result.transform;
+import static de.srsoftware.tools.Tag.CLASS;
+import static de.srsoftware.tools.Tag.HREF;
+import static de.srsoftware.tools.TagFilter.*;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Map.of;
+
+public class AtParty extends BaseImporter {
+	private static final Coords COORDS_HOFWIESENPARK = new Coords(50.87802, 12.07067);
+	private static final Coords COORDS_QUI = new Coords(50.87932, 12.07853);
+	private static final Coords COORDS_MUSIC_HALL = new Coords(50.87135, 12.07504);
+	private static final Coords COORDS_HBF = new Coords(50.88335, 12.07767);
+
+	public AtParty() throws NoSuchAlgorithmException {
+		super();
+	}
+
+	@Override
+	protected String baseUrl() {
+		return "https://www.at-party.de";
+	}
+
+	@Override
+	public String description() {
+		return "Importer for Events listed at at-party.de";
+	}
+
+	@Override
+	protected List<Attachment> extractAttachments(Tag eventTag) {
+		var combined = new HashSet<>(super.extractAttachments(eventTag));
+		eventTag.find(attributeHas(CLASS,"slider-area")).stream()
+				.map(tag -> extractBackgroundImage(tag,baseUrl()))
+				//.map(AtParty::dropQuery)
+				.map(Util::url)
+				.map(Util::toAttachment)
+				.map(Result::optional)
+				.flatMap(Optional::stream)
+				.forEach(combined::add);
+		return List.copyOf(combined);
+	}
+
+	private static Result<String> dropQuery(Result<String> s) {
+		var opt = s.optional();
+		if (opt.isEmpty()) return s;
+		return Payload.of(opt.get().replaceAll("\\?.*",""));
+	}
+
+	@Override
+	protected Predicate<Tag> extractAttachmentsFilter() {
+		return attributeHas(CLASS,"event-details");
+	}
+
+	@Override
+	protected Predicate<Tag> extractDescriptionFilter() {
+		return attributeHas(CLASS,"event-details");
+	}
+
+	@Override
+	protected Result<Coords> extractCoords(Tag eventTag) {
+		var res = super.extractLocation(eventTag);
+		var opt = res.optional();
+		if (opt.isEmpty()) return transform(res);
+		var loc = opt.get();
+		var lower = loc.toLowerCase();
+		if (lower.contains("seven")) return Payload.of(COORDS_HBF);
+		if (lower.contains("music hall")) return Payload.of(COORDS_MUSIC_HALL);
+		if (lower.contains("partyhaus")) return Payload.of(COORDS_QUI);
+		if (lower.contains("qui")) return Payload.of(COORDS_QUI);
+		if (lower.contains("hofwiesenpark")) return Payload.of(COORDS_HOFWIESENPARK);
+		return error("unknown location: %s",loc);
+	}
+
+	@Override
+	protected Predicate<Tag> extractEndDateFilter() {
+		return null;
+	}
+
+	@Override
+	protected Predicate<Tag> extractEndTimeFilter() {
+		return null;
+	}
+
+	@Override
+	protected Predicate<Tag> extractEventTagFilter() {
+		return attributeEquals(CLASS,"main-content");
+	}
+
+	@Override
+	protected Result<List<String>> extractEventUrls(Result<Tag> programPage) {
+		var opt = programPage.optional();
+		if (opt.isEmpty()) return transform(programPage);
+		var list = opt.get().find(IS_ANCHOR)
+				.stream().map(tag -> tag.get(HREF))
+				.filter(Objects::nonNull)
+				.filter(link -> link.contains("/events/"))
+				.map(url -> url.contains("://")?url:baseUrl()+url)
+				.distinct()
+				.toList();
+		return Payload.of(list);
+	}
+
+	@Override
+	protected Predicate<Tag> extractLinksFilter() {
+		return attributeHas(CLASS,"event-details");
+	}
+
+	@Override
+	protected Result<String> extractLocation(Tag eventTag) {
+		var res = super.extractLocation(eventTag);
+		var opt = res.optional();
+		if (opt.isEmpty()) return transform(res);
+		var loc = opt.get();
+		var lower = loc.toLowerCase();
+		if (lower.contains("seven")) return Payload.of("Club Seven, Bahnhofsplatz 6, 07545 Gera");
+		if (lower.contains("music hall")) return Payload.of("Music Hall Gera, Heinrichstraße 49, 07545 Gera");
+		if (lower.contains("partyhaus")) return Payload.of("Partyhaus, De-Smit-Str. 2, 07545 Gera");
+		if (lower.contains("qui")) return Payload.of("Club QUI Gera, De-Smit-Str. 2, 07545 Gera");
+		if (lower.contains("hofwiesenpark")) return Payload.of("Hofwiesenpark, 07545 Gera");
+		LOG.log(System.Logger.Level.ERROR,"unknown location: {0}",loc);
+		return Payload.of("Gera");
+	}
+
+
+	@Override
+	protected Predicate<Tag> extractLocationFilter() {
+		return ofType("h1");
+	}
+
+	@Override
+	protected Predicate<Tag> extractStartDateFilter() {
+		return attributeEquals(CLASS,"theme-gradient");
+	}
+
+	@Override
+	protected Predicate<Tag> extractStartTimeFilter() {
+		return ofType("h4").and(tag -> tag.strip().contains("Uhr"));
+	}
+
+	@Override
+	protected List<String> extractTags(Tag eventTag) {
+		return List.of("Gera");
+	}
+
+	@Override
+	protected Result<String> extractTitle(Tag eventTag) {
+		var res = super.extractTitle(eventTag);
+		var opt = res.optional();
+		return opt.isEmpty() ? res : Payload.of(opt.get().split("\\|")[0].trim());
+	}
+
+	@Override
+	protected Predicate<Tag> extractTitleFilter() {
+		return ofType("h1");
+	}
+
+	@Override
+	protected Result<LocalDate> parseEndDate(String string) {
+		return null;
+	}
+
+	@Override
+	protected Result<LocalTime> parseEndTime(String string) {
+		return null;
+	}
+
+	@Override
+	protected Result<LocalDate> parseStartDate(String string) {
+		return parseLongGermanDate(string);
+	}
+
+	@Override
+	protected Result<LocalTime> parseStartTime(String string) {
+		return parseGermanTime(string);
+	}
+
+	/**
+	 * Die ATParty-Seite haut kaputte meta-tags mit raus.
+	 * Also schneiden wir den kompletten header ab...
+	 * @param inputStream eingehender InputStream, verpackt in Result
+	 * @return ausgehender InputStream, verpackt in Result
+	 */
+	@Override
+	protected Result<InputStream> preload(Result<InputStream> inputStream) {
+		var opt = inputStream.optional();
+		if (opt.isEmpty()) return transform(inputStream);
+		try {
+			var input = opt.get();
+			var bos = new ByteArrayOutputStream();
+			input.transferTo(bos);
+			input.close();
+			String code = bos.toString(UTF_8);
+			var pos = code.indexOf("<body");
+			return Payload.of(new ByteArrayInputStream(code.substring(pos).getBytes(UTF_8)));
+		} catch (IOException e) {
+			return error(e, "Failed to buffer data from %s", inputStream);
+		}
+	}
+
+	@Override
+	protected String programURL() {
+		return baseUrl()+"/veranstaltungen";
+	}
+}

de.srsoftware.cal.importer/src/main/java/de/srsoftware/cal/importer/neustadt/Wotufa.java

@@ -97,7 +97,9 @@ public class Wotufa extends BaseImporter {
 		Result<Tag> locationTag = extractLocationTag(eventTag);
 		if (locationTag.optional().isEmpty()) return transform(locationTag);
 		var loc = locationTag.optional().get().strip();
-		return Payload.of("Neustadt an der Orla".equals(loc) ? DEFAULT_LOCATION : loc);
+
+		var wotufa = loc.equals("Neustadt an der Orla") || loc.toLowerCase().contains("wotufa");
+		return Payload.of(wotufa ? DEFAULT_LOCATION : loc);
 	}
 
 	@Override