SRSoftware/OpenCloudCal
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

finished importer for Rosenkeller

Browse Source 
Signed-off-by: Stephan Richter <s.richter@srsoftware.de>
This commit is contained in:
Stephan Richter
2024-12-20 21:56:42 +01:00
parent d7967b2d95
commit a96fb8cb2d
7 changed files with 234 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
de.srsoftware.cal.api/src/main/java/de/srsoftware/cal/api/Attachment.java
View File

@@ -4,18 +4,9 @@ package de.srsoftware.cal.api;
import java.net.URL;
/**
* attachments may provide additional information about an appointment
* an attachment for appointments
* @param url the URL of the attached document
* @param mime the mime type of the attached document
*/
public interface Attachment {
/**
* the mime type of the attached document
* @return a mime type
*/
String mime();
/**
* the URL of the attached document
* @return the attachment URL
*/
URL url();
public record Attachment(URL url, String mime) {
}

5
de.srsoftware.cal.api/src/main/java/de/srsoftware/cal/api/Importer.java
View File

@@ -1,7 +1,6 @@
/* © SRSoftware 2024 */
package de.srsoftware.cal.api;
import de.srsoftware.tools.Result;
import java.io.IOException;
import java.util.stream.Stream;
@@ -17,7 +16,9 @@ public interface Importer {
/**
* get the list of appointments from the source associated with this importer
*
* @return a list of appointments
* @throws IOException if there is an IOException
*/
Stream<Result<Appointment>> fetch() throws IOException;
Stream<Appointment> fetch() throws IOException;
}

15
de.srsoftware.cal.api/src/main/java/de/srsoftware/cal/api/Link.java
View File

@@ -5,17 +5,8 @@ import java.net.URL;
/**
* Links are additional content that may be added to appointments
* @param url the URL of the link
* @param desciption some information about the target of the link
*/
public interface Link {
/**
* some information about the target of the link
* @return descriptive text
*/
String description();
/**
* the URL of the link
* @return the link`s URL
*/
URL url();
public record Link(URL url, String desciption) {
}

25
de.srsoftware.cal.app/src/main/java/de/srsoftware/cal/app/Application.java
View File

@@ -1,23 +1,22 @@
/* © SRSoftware 2024 */
package de.srsoftware.cal.app;
import de.srsoftware.cal.api.Appointment;
import de.srsoftware.cal.importer.JenaRosenkeller;
import de.srsoftware.tools.Payload;
import java.io.IOException;
/**
* Test application
*/
public class Application {
public static void main(String[] args) throws IOException {
private Application() {
}
/**
* sandbox
* @param args default
*/
public static void main(String[] args) {
var rosenkeller = new JenaRosenkeller();
var appointments = rosenkeller.fetch();
appointments.forEach(res -> {
System.out.printf("class: %s%n", res.getClass());
if (res instanceof Payload<Appointment> payload) {
System.out.printf("payload: %s%n", payload.get().getClass());
System.out.println(payload.get());
} else {
System.err.println(res);
}
});
appointments.forEach(System.err::println);
}
}

2
de.srsoftware.cal.importer/build.gradle.kts
View File

@@ -3,6 +3,6 @@ description = "OpenCloudCal : Importers"
dependencies {
implementation(project(":de.srsoftware.cal.api"))
implementation("de.srsoftware:tools.optionals:1.0.0")
implementation("de.srsoftware:tools.util:1.1.1")
implementation("de.srsoftware:tools.util:1.1.2")
implementation("de.srsoftware:tools.web:1.3.2")
}

138
de.srsoftware.cal.importer/src/main/java/de/srsoftware/cal/importer/BaseAppointment.java
View File

@@ -1,17 +1,116 @@
/* © SRSoftware 2024 */
package de.srsoftware.cal.importer;
import static java.util.Optional.empty;
import static de.srsoftware.tools.Optionals.nullable;
import de.srsoftware.cal.api.Appointment;
import de.srsoftware.cal.api.Attachment;
import de.srsoftware.cal.api.Coords;
import de.srsoftware.cal.api.Link;
import java.time.LocalDateTime;
import java.util.Optional;
import java.util.Set;
import java.util.*;
/**
* basic class for Appointments
*/
public class BaseAppointment implements Appointment {
private final long id;
private final String title, description;
private final LocalDateTime end, start;
private Coords coords = null;
private final Set<Attachment> attachments = new HashSet<>();
private final Set<String> tags = new HashSet<>();
private final Set<Link> links = new HashSet<>();
private final String location;
/**
* create a new appointment
* @param id set the id
* @param title set the title
* @param description set the description
* @param start set the start date
* @param end set the end date
* @param location set the location
*/
public BaseAppointment(long id, String title, String description, LocalDateTime start, LocalDateTime end, String location) {
this.description = description;
this.end = end;
this.id = id;
this.location = location;
this.start = start;
this.title = title;
}
/**
* adds attachments
* @param newAttachments the attachments to add to the appointment
* @return the appointment
*/
public BaseAppointment add(Attachment... newAttachments) {
Collections.addAll(attachments, newAttachments);
return this;
}
/**
* adds attachments
* @param newAttachments the attachments to add to the appointment
* @return the appointment
*/
public BaseAppointment add(Collection<Attachment> newAttachments) {
attachments.addAll(newAttachments);
return this;
}
/**
* adds links
* @param newLinks the links to add to the appointment
* @return the appointment
*/
public BaseAppointment addLinks(Link... newLinks) {
Collections.addAll(links, newLinks);
return this;
}
/**
* adds links
* @param newLinks the links to add to the appointment
* @return the appointment
*/
public BaseAppointment addLinks(Collection<Link> newLinks) {
links.addAll(newLinks);
return this;
}
/**
* adds tag
* @param newTags the tag to add to the appointment
* @return the appointment
*/
public BaseAppointment tag(String... newTags) {
Collections.addAll(tags, newTags);
return this;
}
/**
* adds tag
* @param newTags the tag to add to the appointment
* @return the appointment
*/
public BaseAppointment tag(Collection<String> newTags) {
tags.addAll(newTags);
return this;
}
/**
* set the coordinates of the attachments
* @param newCoords the coordinates to apply
* @return the appointment
*/
public BaseAppointment coords(Coords newCoords) {
coords = newCoords;
return this;
}
public abstract class BaseAppointment implements Appointment {
@Override
public Set<Attachment> attachments() {
return Set.of();
@@ -19,12 +118,32 @@ public abstract class BaseAppointment implements Appointment {
@Override
public Optional<Coords> coords() {
return empty();
return nullable(coords);
}
@Override
public String description() {
return description;
}
@Override
public Optional<LocalDateTime> end() {
return empty();
return nullable(end);
}
@Override
public long id() {
return id;
}
@Override
public String location() {
return location;
}
@Override
public LocalDateTime start() {
return start;
}
@Override
@@ -32,9 +151,14 @@ public abstract class BaseAppointment implements Appointment {
return Set.of();
}
@Override
public String title() {
return title;
}
@Override
public String toString() {
return "%s (%s)".formatted(title(), BaseAppointment.class.getSimpleName());
return "%s (%s)".formatted(title, start);
}
@Override

106
de.srsoftware.cal.importer/src/main/java/de/srsoftware/cal/importer/JenaRosenkeller.java
View File

@@ -16,12 +16,18 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.regex.Pattern;
import java.util.stream.Stream;
/**
* Importer für Events vom Rosenkeller Jena
*/
public class JenaRosenkeller implements Importer {
private static final String BASE_URL = "https://rosenkeller.org";
private static final String APPOINTMENT_TAG_ID = "tribe-events-content";
private static final Coords DEFAULT_COORDS = new Coords(50.9294, 11.585);
private static final String DEFAULT_LOCATION = "Rosenkeller, Johannisstr. 13, 07743 Jena";
@@ -33,8 +39,8 @@ public class JenaRosenkeller implements Importer {
}
@Override
public Stream<Result<Appointment>> fetch() throws IOException {
var url = Payload.of("https://rosenkeller.org/de/programm");
public Stream<Appointment> fetch() {
var url = Payload.of(BASE_URL + "/de/programm");
Stream<Result<String>> stream = url(url)
.map(JenaRosenkeller::open) //
.map(JenaRosenkeller::preload)
@@ -43,11 +49,18 @@ public class JenaRosenkeller implements Importer {
.stream();
return stream //
.map(JenaRosenkeller::url)
.map(JenaRosenkeller::loadEvent)
.flatMap(result -> result.optional().stream());
}
private static Result<Appointment> loadEvent(Result<URL> urlResult) {
var link = urlResult.optional().map(url -> new Link(url, "Event-Seite")).orElse(null);
return urlResult //
.map(JenaRosenkeller::open)
.map(JenaRosenkeller::preload)
.map(JenaRosenkeller::parse)
.map(JenaRosenkeller::getEventDiv)
.map(JenaRosenkeller::loadEvent);
.map(tagResult -> parseEvent(tagResult, link));
}
private static Result<Tag> getEventDiv(Result<Tag> pageResult) {
@@ -119,14 +132,20 @@ public class JenaRosenkeller implements Importer {
}
}
private static Result<Appointment> loadEvent(Result<Tag> domResult) {
private static Result<Appointment> parseEvent(Result<Tag> domResult, Link eventPage) {
switch (domResult) {
case Payload<Tag> payload:
var appointmentTag = payload.get();
var title = extractTitle(appointmentTag);
if (title.isEmpty()) return Error.format("No title found at %s", eventPage.url());
var description = extractDescription(appointmentTag);
if (description.isEmpty()) return Error.format("No description found at %s", eventPage.url());
var start = extractStart(appointmentTag);
return Error.of("Could not find appointment title");
if (start.isEmpty()) return Error.format("No start date/time found at %s", eventPage.url());
var links = extractLinks(appointmentTag);
var attachments = extractAttachments(appointmentTag);
var appointment = new BaseAppointment(0, title.get(), description.get(), start.get(), null, DEFAULT_LOCATION).addLinks(links).add(attachments);
return Payload.of(appointment);
case Error<Tag> err:
return err.transform();
default:
@@ -134,6 +153,60 @@ public class JenaRosenkeller implements Importer {
}
}
private static List<Attachment> extractAttachments(Tag appointmentTag) {
return appointmentTag //
.find(ofType("img"))
.stream()
.map(tag -> tag.get("src"))
.filter(Objects::nonNull)
.map(Payload::of)
.map(JenaRosenkeller::url)
.map(JenaRosenkeller::toAttachment)
.map(Result::optional)
.flatMap(Optional::stream)
.toList();
}
private static Result<Attachment> toAttachment(Result<URL> urlResult) {
switch (urlResult) {
case Payload<URL> payload:
try {
var mime = payload.get().openConnection().getContentType();
return Payload.of(new Attachment(payload.get(), mime));
} catch (Exception e) {
return Error.format("Failed to read mime type of %s", payload);
}
case Error<URL> err:
return err.transform();
default:
return Error.format("Invalid parameter: %s", urlResult.getClass().getSimpleName());
}
}
private static List<Link> extractLinks(Tag appointmentTag) {
var links = new ArrayList<Link>();
appointmentTag //
.find(attributeStartsWith("id", "post-"))
.stream()
.flatMap(tag -> tag.find(ofType("a")).stream())
.forEach(anchor -> {
var href = anchor.get("href");
if (href == null) return;
if (!href.contains("://")) href = BASE_URL + "href";
var text = anchor.inner(0).orElse(href);
Payload.of(href).map(JenaRosenkeller::url).optional().map(url -> new Link(url, text)).ifPresent(links::add);
});
return links;
}
private static Result<Link> toLink(Result<URL> urlResult, Optional<String> description) {
return switch (urlResult) {
case Payload<URL> payload -> Payload.of(new Link(payload.get(),description.orElse(payload.toString())));
case Error<URL> err -> err.transform();
default -> Error.format("Invalid parameter: %s", urlResult.getClass().getSimpleName());
};
}
private static Optional<LocalDateTime> extractStart(Tag appointmentTag) {
return appointmentTag.find(attributeEquals("class", "tribe-event-date-start")).stream().flatMap(tag -> tag.inner(0).stream()).flatMap(txt -> toDateTime(txt).stream()).findAny();
}
@@ -185,29 +258,6 @@ public class JenaRosenkeller implements Importer {
.findAny();
}
private static Optional<Appointment> nope(URL url) {
try {
var input = url.openConnection().getInputStream();
input = XMLParser.preload(input);
var result = XMLParser.parse(input);
input.close();
if (result instanceof Payload<Tag> payload) {
var tag = payload.get();
tag.find(attributeEndsWith("class", "single-event-title")) //
.stream()
.map(Tag::children)
.filter(not(List::isEmpty))
.map(List::getFirst)
.map(Tag::toString)
.forEach(System.out::println);
}
return empty();
} catch (IOException e) {
e.printStackTrace();
return empty();
}
}
private static Result<URL> url(Result<String> urls) {
switch (urls) {
case Payload<String> payload: