SRSoftware/OpenCloudCal
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

started another importer overhaul

Browse Source 
Signed-off-by: Stephan Richter <s.richter@srsoftware.de>
This commit is contained in:
StephanRichter
2025-01-01 16:11:30 +01:00
parent d9e5475962
commit 3c7313fc01
9 changed files with 527 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files
de.srsoftware.cal.base/build.gradle.kts
+1 -1
View File
@@ -5,6 +5,6 @@ dependencies {
implementation("de.srsoftware:tools.optionals:1.0.0")
implementation("de.srsoftware:tools.util:1.3.0")
implementation("de.srsoftware:tools.web:1.3.9")
implementation("de.srsoftware:tools.web:1.3.10")
implementation("org.json:json:20240303")
}
de.srsoftware.cal.base/src/main/java/de/srsoftware/cal/BaseImporter.java
+264 -183
View File
@@ -1,8 +1,11 @@
/* © SRSoftware 2024 */
package de.srsoftware.cal;
import static de.srsoftware.cal.Util.combine;
import static de.srsoftware.tools.Error.error;
import static de.srsoftware.tools.Result.transform;
import static de.srsoftware.tools.Tag.HREF;
import static de.srsoftware.tools.TagFilter.*;
import static java.lang.System.Logger.Level.WARNING;
import de.srsoftware.cal.api.*;
@@ -10,17 +13,16 @@ import de.srsoftware.tools.*;
import de.srsoftware.tools.Error;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.*;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.time.LocalTime;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Predicate;
import java.util.stream.Stream;
public abstract class BaseImporter implements Importer {
@@ -35,15 +37,13 @@ public abstract class BaseImporter implements Importer {
protected abstract String baseUrl();
@Override
public String description() {
return "abstract base class to create other importers on";
}
public abstract String description();
protected List<Attachment> extractAttachments(Tag eventTag) {
return extractAttachmentsTag(eventTag) //
.optional()
.stream()
.flatMap(tag -> tag.find(TagFilter.ofType("img")).stream())
.flatMap(tag -> tag.find(IS_IMAGE).stream())
.map(tag -> tag.get("src"))
.filter(Objects::nonNull)
.map(url -> url.contains("://") ? url : baseUrl()+url)
@@ -55,35 +55,77 @@ public abstract class BaseImporter implements Importer {
.toList();
}
protected abstract Predicate<Tag> extractAttachmentsFilter();
protected Result<Tag> extractAttachmentsTag(Tag eventTag) {
return extractDescriptionTag(eventTag);
var list = eventTag.find(extractAttachmentsFilter());
if (list.isEmpty()) return error("Failed to find attachments tag");
return Payload.of(list.getFirst());
}
protected Result<String> extractDescription(Tag eventTag) {
Result<Tag> descriptionTag = extractDescriptionTag(eventTag);
if (descriptionTag.optional().isEmpty()) return transform(descriptionTag);
Tag tag = descriptionTag.optional().get();
tag.find(t -> t.is("iframe")).forEach(Tag::remove);
var inner = tag.inner(2);
if (inner.isPresent()) return Payload.of(inner.get());
return error("No description found");
return inner.isPresent() ? Payload.of(inner.get()) : error("No description found");
}
protected abstract Result<Tag> extractDescriptionTag(Tag eventTag);
protected abstract Predicate<Tag> extractDescriptionFilter();
protected Result<Coords> extractCoords(Tag eventTag) {
return error("not implemented");
protected Result<Tag> extractDescriptionTag(Tag eventTag){
var list = eventTag.find(extractDescriptionFilter());
if (list.isEmpty()) return error("Failed to find attachments tag");
return Payload.of(list.getFirst());
}
protected abstract Result<Coords> extractCoords(Tag eventTag);
protected Result<LocalDateTime> extractEnd(Tag eventTag) {
Result<Tag> endTag = extractEndTag(eventTag);
if (endTag.optional().isEmpty()) return transform(endTag);
return parseEndDate(endTag.optional().get().toString(0));
Result<LocalDate> date = extractEndDate(eventTag);
Result<LocalTime> time = extractEndTime(eventTag);
return combine(date,time);
}
protected abstract Result<Tag> extractEndTag(Tag eventTag);
protected Result<LocalDate> extractEndDate(Tag eventTag) {
Result<Tag> endDateTag = extractEndDateTag(eventTag);
var opt = endDateTag.optional();
if (opt.isEmpty()) return transform(endDateTag);
return parseEndDate(opt.get().strip());
}
private Result<Tag> extractEndDateTag(Tag eventTag) {
var list = eventTag.find(extractEndDateFilter());
if (list.isEmpty()) return error("Failed to find end date tag");
return Payload.of(list.getFirst());
}
protected abstract Predicate<Tag> extractEndDateFilter();
protected Result<LocalTime> extractEndTime(Tag eventTag) {
Result<Tag> endTimeTag = extractEndTimeTag(eventTag);
var opt = endTimeTag.optional();
if (opt.isEmpty()) return transform(endTimeTag);
return parseEndTime(opt.get().strip());
}
private Result<Tag> extractEndTimeTag(Tag eventTag) {
var list = eventTag.find(extractEndTimeFilter());
if (list.isEmpty()) return error("Failed to find end time tag");
return Payload.of(list.getFirst());
}
protected abstract Predicate<Tag> extractEndTimeFilter();
protected Result<Appointment> extractEvent(Result<Tag> domResult, Link eventPage) {
var opt = domResult.optional();
if (opt.isEmpty()) return transform(domResult);
var eventTag = opt.get();
protected Result<Appointment> extractEvent(Tag eventTag, Link eventPage) {
long id = 0;
var titleResult = extractTitle(eventTag);
@@ -116,196 +158,235 @@ public abstract class BaseImporter implements Importer {
return Payload.of(event);
}
private Result<Appointment> extractEvent(Result<Tag> domResult, Link eventPage) {
return switch (domResult) {
case Payload<Tag> payload -> extractEvent(payload.get(), eventPage);
case Error<Tag> err -> err.transform();
default -> invalidParameter(domResult);
};
}
/**
* Die ist der Tag auf der Seite, der alle weiteren Event-Daten umfasst.
* Im Prinzip kann der Page-Tag auch direkt weitergereicht werden, dann
* sind die weiteren Suchen aber Umfangreicher.
* Besser ist es daher, den Tag näher einzugrenzen
* @param pageResult
* @return
*/
protected Result<Tag> extractEventTag(Result<Tag> pageResult){
var opt = pageResult.optional();
if (opt.isEmpty()) return transform(pageResult);
var list = opt.get().find(extractEventTagFilter());
if (list.isEmpty()) return error("Failed to find event tag");
return Payload.of(list.getFirst());
}
protected abstract Result<Tag> extractEventTag(Result<Tag> pageResult);
protected abstract Predicate<Tag> extractEventTagFilter();
protected abstract Result<List<String>> extractEventUrls(Result<Tag> programPage);
protected List<Link> extractLinks(Tag appointmentTag) {
var links = new ArrayList<Link>();
protected abstract Result<List<String>> extractEventUrls(Result<Tag> programPage);
extractLinksTag(appointmentTag) //
.map(this::extractLinkAnchors)
.optional()
.stream()
.flatMap(List::stream)
.forEach(anchor -> {
var href = anchor.get("href");
if (href == null) return;
if (!href.contains("://")) href = baseUrl() + href;
var text = anchor.inner(0).orElse(href);
Payload //
.of(href)
.map(BaseImporter::url)
.map(url -> link(url, text))
.optional()
.ifPresent(links::add);
});
return links;
}
public abstract Result<List<Tag>> extractLinkAnchors(Result<Tag> tagResult);
protected List<Link> extractLinks(Tag appointmentTag) {
var tag = extractLinksTag(appointmentTag);
var opt = tag.optional();
if (opt.isEmpty()) return List.of();
Tag linksTag = opt.get();
return linksTag.find(IS_ANCHOR).stream()
.map(anchor -> {
var href = anchor.get(HREF);
if (href == null) return null;
if (!href.contains("://")) href = baseUrl()+href;
var txt = anchor.strip();
return BaseImporter.url(Payload.of(href)).optional().map(url -> new Link(url,txt)).orElse(null);
})
.filter(Objects::nonNull)
.toList();
}
protected Result<Tag> extractLinksTag(Tag eventTag) {
return extractDescriptionTag(eventTag);
}
var list = eventTag.find(extractLinksFilter());
if (list.isEmpty()) return error("Failed to find links tag");
return Payload.of(list.getFirst());
}
protected Result<String> extractLocation(Tag eventTag) {
protected abstract Predicate<Tag> extractLinksFilter();
protected Result<String> extractLocation(Tag eventTag) {
Result<Tag> locationTag = extractLocationTag(eventTag);
if (locationTag.optional().isEmpty()) return transform(locationTag);
return Payload.of(locationTag.optional().get().toString(2));
}
protected abstract Result<Tag> extractLocationTag(Tag eventTag);
protected Result<Tag> extractLocationTag(Tag eventTag){
var list = eventTag.find(extractLocationFilter());
if (list.isEmpty()) return error("Failed to find location tag");
return Payload.of(list.getFirst());
}
protected Result<LocalDateTime> extractStart(Tag eventTag) {
Result<Tag> startTag = extractStartTag(eventTag);
if (startTag.optional().isEmpty()) return transform(startTag);
return parseStartDate(startTag.optional().get().strip());
}
protected abstract Predicate<Tag> extractLocationFilter();
protected abstract Result<Tag> extractStartTag(Tag eventTag);
protected abstract List<String> extractTags(Tag eventTag);
protected Result<LocalDateTime> extractStart(Tag eventTag) {
Result<LocalDate> date = extractStartDate(eventTag);
Result<LocalTime> time = extractStartTime(eventTag);
return combine(date,time);
}
protected Result<LocalDate> extractStartDate(Tag eventTag) {
Result<Tag> startDateTag = extractStartDateTag(eventTag);
var opt = startDateTag.optional();
if (opt.isEmpty()) return transform(startDateTag);
return parseStartDate(opt.get().strip());
}
private Result<Tag> extractStartDateTag(Tag eventTag) {
var list = eventTag.find(extractStartDateFilter());
if (list.isEmpty()) return error("Failed to find start date tag");
return Payload.of(list.getFirst());
}
protected abstract Predicate<Tag> extractStartDateFilter();
protected Result<LocalTime> extractStartTime(Tag eventTag) {
Result<Tag> startTimeTag = extractStartTimeTag(eventTag);
var opt = startTimeTag.optional();
if (opt.isEmpty()) return transform(startTimeTag);
return parseStartTime(opt.get().strip());
}
private Result<Tag> extractStartTimeTag(Tag eventTag) {
var list = eventTag.find(extractStartTimeFilter());
if (list.isEmpty()) return error("Failed to find start time tag");
return Payload.of(list.getFirst());
}
protected abstract Predicate<Tag> extractStartTimeFilter();
protected abstract List<String> extractTags(Tag eventTag);
protected Result<String> extractTitle(Tag eventTag) {
Result<Tag> titleTag = extractTitleTag(eventTag);
if (titleTag.optional().isEmpty()) return transform(titleTag);
var inner = titleTag.optional().flatMap(tag -> tag.inner(2));
return inner.isPresent() ? Payload.of(inner.get()) :
error("No title found");
return inner.isPresent() ? Payload.of(inner.get()) : error("No title found");
}
protected Result<Tag> extractTitleTag(Tag eventTag){
var list = eventTag.find(extractTitleFilter());
if (list.isEmpty()) return error("Failed to find title tag");
return Payload.of(list.getFirst());
}
protected abstract Predicate<Tag> extractTitleFilter();
@Override
public Stream<Appointment> fetch() {
var url = Payload.of(programURL());
Stream<Result<String>> urls = url(url)
.map(this::open) //
.map(this::preload)
.map(this::parseXML)
.map(this::extractEventUrls)
.stream();
return urls //
.map(BaseImporter::url)
.map(this::loadEvent)
.peek(e -> {
if (e instanceof Error<Appointment> err) System.err.println(err);
})
.flatMap(result -> result.optional().stream());
}
protected static <T> Result<T> invalidParameter(Result<?> result) {
return error("Invalid parameter: %s", result.getClass().getSimpleName());
}
protected static Result<Link> link(Result<URL> url, String text) {
var opt = url.optional();
return opt.isEmpty() ? transform(url) : Payload.of(new Link(opt.get(), text));
}
protected Result<Appointment> loadEvent(Result<URL> urlResult) {
var link = urlResult //
.optional()
.map(url -> new Link(url, "Event-Seite"))
.orElse(null);
return urlResult //
.map(this::open)
.map(this::preload)
.map(this::parseXML)
.map(this::extractEventTag)
.map(tagResult -> extractEvent(tagResult, link));
}
protected Result<InputStream> open(Result<URL> url) {
var opt = url.optional();
if (opt.isEmpty()) return transform(url);
try {
var conn = (HttpURLConnection) opt.get().openConnection();
conn.setRequestProperty("Accept","*/*");
conn.setRequestProperty("Host",opt.get().getHost());
conn.setRequestProperty("User-Agent","OpenCloudCal/0.1");
return Payload.of(conn.getInputStream());
} catch (IOException e) {
return error(e, "Failed to open %s", url, e);
}
protected abstract Result<Tag> extractTitleTag(Tag eventTag);
}
@Override
public Stream<Appointment> fetch() {
var url = Payload.of(programURL());
Stream<Result<String>> urls = url(url)
.map(this::open) //
.map(this::preload)
.map(this::parseXML)
.map(this::extractEventUrls)
.stream();
return urls //
.map(BaseImporter::url)
.map(this::loadEvent)
.peek(e -> {
if (e instanceof Error<Appointment> err) System.err.println(err);
})
.flatMap(result -> result.optional().stream());
}
protected abstract Result<LocalDate> parseEndDate(String string);
protected abstract Result<LocalTime> parseEndTime(String string);
protected static <T> Result<T> invalidParameter(Result<?> result) {
return error("Invalid parameter: %s", result.getClass().getSimpleName());
}
protected abstract Result<LocalDate> parseStartDate(String string);
protected abstract Result<LocalTime> parseStartTime(String string);
protected static Result<Link> link(Result<URL> url, String text) {
var opt = url.optional();
if (opt.isEmpty()) return transform(url);
return Payload.of(new Link(opt.get(), text));
}
protected Result<Appointment> loadEvent(Result<URL> urlResult) {
var link = urlResult //
.optional()
.map(url -> new Link(url, "Event-Seite"))
.orElse(null);
return urlResult //
.map(this::open)
.map(this::preload)
.map(this::parseXML)
.map(this::extractEventTag)
.map(tagResult -> extractEvent(tagResult, link));
}
protected Result<InputStream> open(Result<URL> url) {
switch (url) {
case Payload<URL> payload:
try {
return Payload.of(payload.get().openConnection().getInputStream());
} catch (IOException e) {
return error(e, "Failed to open %s", payload, e);
}
case Error<URL> error:
return error.transform();
default:
return invalidParameter(url);
}
}
protected abstract Result<LocalDateTime> parseEndDate(String string);
protected abstract Result<LocalDateTime> parseStartDate(String string);
protected Result<Tag> parseXML(Result<InputStream> inputStream) {
return switch (inputStream) {
case Payload<InputStream> payload -> XMLParser.parse(payload.get());
case Error<InputStream> error -> error.transform();
default -> invalidParameter(inputStream);
};
protected Result<Tag> parseXML(Result<InputStream> inputStream) {
var opt = inputStream.optional();
return opt.isEmpty() ? transform((inputStream)) : XMLParser.parse(opt.get());
}
protected Result<InputStream> preload(Result<InputStream> inputStream) {
switch (inputStream) {
case Payload<InputStream> payload:
try {
return Payload.of(XMLParser.preload(payload.get()));
} catch (IOException e) {
return error(e, "Failed to buffer data from %s", payload);
}
case Error<InputStream> error:
return error.transform();
default:
return invalidParameter(inputStream);
}
}
protected abstract String programURL();
protected static Result<Attachment> toAttachment(Result<URL> urlResult) {
var opt = urlResult.optional();
if (opt.isEmpty()) return transform(urlResult);
try {
var mime = opt.get().openConnection().getContentType();
return Payload.of(new Attachment(opt.get(), mime));
} catch (Exception e) {
LOG.log(WARNING, "Failed to read mime type of {0}", opt.get());
return error("Failed to read mime type of %s", opt.get());
}
}
protected static Result<Integer> toNumericMonth(String month) {
month = month.toLowerCase();
if (month.startsWith("ja")) return Payload.of(1);
if (month.startsWith("f")) return Payload.of(2);
if ("may".equals(month) || "mai".equals(month)) return Payload.of(5);
if (month.startsWith("m")) return Payload.of(3);
if (month.startsWith("ap")) return Payload.of(4);
if (month.startsWith("jun")) return Payload.of(6);
if (month.startsWith("jul")) return Payload.of(7);
if (month.startsWith("au")) return Payload.of(8);
if (month.startsWith("s")) return Payload.of(9);
if (month.startsWith("o")) return Payload.of(10);
if (month.startsWith("n")) return Payload.of(11);
if (month.startsWith("d")) return Payload.of(12);
return error("Failed to recognize \"%s\" as a month!", month);
}
protected static Result<URL> url(Result<String> urlResult) {
if (urlResult.optional().isEmpty()) return transform(urlResult);
var url = urlResult.optional().get();
try {
return Payload.of(new URI(url).toURL());
} catch (MalformedURLException | URISyntaxException e) {
return error(e, "Failed to create URL of %s", url);
}
var opt = inputStream.optional();
if (opt.isEmpty()) return transform(inputStream);
try {
return Payload.of(XMLParser.preload(opt.get()));
} catch (IOException e) {
return error(e, "Failed to buffer data from %s", inputStream);
}
}
protected abstract String programURL();
protected static Result<Attachment> toAttachment(Result<URL> urlResult) {
var opt = urlResult.optional();
if (opt.isEmpty()) return transform(urlResult);
try {
var mime = opt.get().openConnection().getContentType();
return Payload.of(new Attachment(opt.get(), mime));
} catch (Exception e) {
LOG.log(WARNING, "Failed to read mime type of {0}", opt.get());
return error("Failed to read mime type of %s", opt.get());
}
}
protected static Result<Integer> toNumericMonth(String month) {
month = month.toLowerCase();
if (month.startsWith("ja")) return Payload.of(1);
if (month.startsWith("f")) return Payload.of(2);
if ("may".equals(month) || "mai".equals(month)) return Payload.of(5);
if (month.startsWith("m")) return Payload.of(3);
if (month.startsWith("ap")) return Payload.of(4);
if (month.startsWith("jun")) return Payload.of(6);
if (month.startsWith("jul")) return Payload.of(7);
if (month.startsWith("au")) return Payload.of(8);
if (month.startsWith("s")) return Payload.of(9);
if (month.startsWith("o")) return Payload.of(10);
if (month.startsWith("n")) return Payload.of(11);
if (month.startsWith("d")) return Payload.of(12);
return error("Failed to recognize \"%s\" as a month!", month);
}
protected static Result<URL> url(Result<String> urlResult) {
if (urlResult.optional().isEmpty()) return transform(urlResult);
var url = urlResult.optional().get();
try {
return Payload.of(new URI(url).toURL());
} catch (MalformedURLException | URISyntaxException e) {
return error(e, "Failed to create URL of %s", url);
}
}
}
de.srsoftware.cal.base/src/main/java/de/srsoftware/cal/Util.java
+37
View File
@@ -2,12 +2,17 @@
package de.srsoftware.cal;
import static de.srsoftware.tools.Error.error;
import static de.srsoftware.tools.Result.transform;
import de.srsoftware.cal.api.Coords;
import de.srsoftware.tools.Payload;
import de.srsoftware.tools.Result;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.regex.Pattern;
public class Util {
public static final String BEGIN = "BEGIN";
@@ -26,8 +31,18 @@ public class Util {
public static final String VEVENT = "VEVENT";
public static final String VCALENDAR = "VCALENDAR";
public static final Pattern GERMAN_DATE_PATTERN = Pattern.compile("\\D(\\d\\d?)\\.(\\d\\d?)\\.(\\d{4})\\D");
public static final Pattern GERMAN_TIME_PATTERN = Pattern.compile("\\D(\\d\\d?):(\\d\\d?)(:(\\d\\d?))?\\D");
private Util(){}
public static Result<LocalDateTime> combine(Result<LocalDate> date, Result<LocalTime> time) {
if (date.optional().isEmpty())return transform(date);
if (time.optional().isEmpty())return transform(time);
return Payload.of(LocalDateTime.of(date.optional().get(),time.optional().get()));
}
/**
* formats a content line as defined in <a href="https://datatracker.ietf.org/doc/html/rfc5545#section-3.1">iCalendar spec</a>
* @param key the content line key
@@ -76,6 +91,28 @@ public class Util {
.replace(":","/");
}
public static Result<LocalDate> parseGermanDate(String s){
var match = GERMAN_DATE_PATTERN.matcher(s);
if (match.find()){
var day = Integer.parseInt(match.group(1));
var month = Integer.parseInt(match.group(2));
var year = Integer.parseInt(match.group(3));
return Payload.of(LocalDate.of(year,month,day));
}
return error("Failed to find date");
}
public static Result<LocalTime> parseGermanTime(String s){
var match = GERMAN_TIME_PATTERN.matcher(s);
if (match.find()){
var hour = Integer.parseInt(match.group(1));
var minute = Integer.parseInt(match.group(2));
var sec = match.group(4);
var second = sec == null ? 0 : Integer.parseInt(sec);
return Payload.of(LocalTime.of(hour,minute,second));
}
return error("Failed to find date");
}
/**
* wraps a text (list of vevents in a vcalendar, as described in th <a href="https://datatracker.ietf.org/doc/html/rfc5545#section-3.4">iCalendar spec</a>