SRSoftware/OpenCloudCal
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

transformed JenaRosenkeller importer to use Result-driven pipeline

Browse Source 
Signed-off-by: Stephan Richter <s.richter@srsoftware.de>
This commit is contained in:
Stephan Richter
2024-12-19 16:23:30 +01:00
parent a17b322001
commit ed8cef3738
8 changed files with 272 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
de.srsoftware.cal.api/build.gradle.kts
View File

@@ -1,4 +1,5 @@
description = "OpenCloudCal : API"
dependencies {
implementation("de.srsoftware:tools.util:1.1.1")
}

7
de.srsoftware.cal.api/src/main/java/de/srsoftware/cal/api/Appointment.java
View File

@@ -60,6 +60,13 @@ public interface Appointment {
*/
Set<String> tags();
/**
* the title of the appointment
* @return the title
*/
String title();
/**
* set of Links that point to related information
* @return set of links

5
de.srsoftware.cal.api/src/main/java/de/srsoftware/cal/api/Importer.java
View File

@@ -1,8 +1,9 @@
/* © SRSoftware 2024 */
package de.srsoftware.cal.api;
import de.srsoftware.tools.Result;
import java.io.IOException;
import java.util.List;
import java.util.stream.Stream;
/**
* Instances of this class can be used to read Appointments from various sources
@@ -18,5 +19,5 @@ public interface Importer {
* get the list of appointments from the source associated with this importer
* @return a list of appointments
*/
List<Appointment> fetch() throws IOException;
Stream<Result<Appointment>> fetch() throws IOException;
}

1
de.srsoftware.cal.app/build.gradle.kts
View File

@@ -3,4 +3,5 @@ description = "OpenCloudCal : Application"
dependencies {
implementation(project(":de.srsoftware.cal.api"))
implementation(project(":de.srsoftware.cal.importer"))
implementation("de.srsoftware:tools.util:1.1.1")
}

15
de.srsoftware.cal.app/src/main/java/de/srsoftware/cal/app/Application.java
View File

@@ -1,12 +1,23 @@
/* © SRSoftware 2024 */
package de.srsoftware.cal.app;
import de.srsoftware.cal.api.Appointment;
import de.srsoftware.cal.importer.JenaRosenkeller;
import de.srsoftware.tools.Payload;
import java.io.IOException;
public class Application {
public static void main(String[] args) throws IOException {
var rosenkeller = new JenaRosenkeller();
rosenkeller.fetch();
var rosenkeller = new JenaRosenkeller();
var appointments = rosenkeller.fetch();
appointments.forEach(res -> {
System.out.printf("class: %s%n", res.getClass());
if (res instanceof Payload<Appointment> payload) {
System.out.printf("payload: %s%n", payload.get().getClass());
System.out.println(payload.get());
} else {
System.err.println(res);
}
});
}
}

4
de.srsoftware.cal.importer/build.gradle.kts
View File

@@ -3,6 +3,6 @@ description = "OpenCloudCal : Importers"
dependencies {
implementation(project(":de.srsoftware.cal.api"))
implementation("de.srsoftware:tools.optionals:1.0.0")
implementation("de.srsoftware:tools.util:1.1.0")
implementation("de.srsoftware:tools.web:1.2.1")
implementation("de.srsoftware:tools.util:1.1.1")
implementation("de.srsoftware:tools.web:1.2.2")
}

214
de.srsoftware.cal.importer/src/main/java/de/srsoftware/cal/importer/JenaRosenkeller.java
View File

@@ -1,59 +1,192 @@
/* © SRSoftware 2024 */
package de.srsoftware.cal.importer;
import static de.srsoftware.tools.Optionals.nullable;
import static java.util.Optional.empty;
import static java.util.function.Predicate.not;
import de.srsoftware.cal.api.Appointment;
import de.srsoftware.cal.api.Importer;
import de.srsoftware.cal.api.*;
import de.srsoftware.tools.Error;
import de.srsoftware.tools.Payload;
import de.srsoftware.tools.Result;
import de.srsoftware.tools.Tag;
import de.srsoftware.tools.XMLParser;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Stream;
public class JenaRosenkeller implements Importer {
private static class EmptyAppointment implements Appointment {
@Override
public Set<Attachment> attachments() {
return Set.of();
}
@Override
public Optional<Coords> coords() {
return empty();
}
@Override
public String description() {
return null;
}
@Override
public Optional<LocalDateTime> end() {
return empty();
}
@Override
public long id() {
return 0;
}
@Override
public String location() {
return null;
}
@Override
public LocalDateTime start() {
return null;
}
@Override
public Set<String> tags() {
return Set.of();
}
@Override
public String title() {
return null;
}
@Override
public String toString() {
return "%s (%s)".formatted(title(), EmptyAppointment.class.getSimpleName());
}
@Override
public Set<Link> urls() {
return Set.of();
}
}
@Override
public String description() {
return "Events von der Seite rosenkeller.org importieren";
}
@Override
public List<Appointment> fetch() throws IOException {
var url = "https://rosenkeller.org/de/programm";
var listUrl = url(url);
if (listUrl.isEmpty()) throw new RuntimeException("Failed to fetch %s".formatted(url));
var input = listUrl.get().openConnection().getInputStream();
input = XMLParser.preload(input);
var result = XMLParser.parse(input);
input.close();
List<Appointment> appointments = nullable(result) //
.filter(o -> o instanceof Payload<Tag>)
.map(o -> (Payload<Tag>)o)
.map(Payload::get)
.stream()
.flatMap(JenaRosenkeller::findEventUrls)
.map(JenaRosenkeller::url)
.filter(Optional::isPresent)
.map(Optional::get)
.map(JenaRosenkeller::loadEvent)
.filter(Optional::isPresent)
.map(Optional::get)
.toList();
System.out.println(appointments);
return appointments;
public Stream<Result<Appointment>> fetch() throws IOException {
var url = Payload.of("https://rosenkeller.org/de/programm");
Stream<Result<String>> stream = url(url)
.map(JenaRosenkeller::open) //
.map(JenaRosenkeller::preload)
.map(JenaRosenkeller::parse)
.map(JenaRosenkeller::findEventUrls)
.stream();
return stream //
.map(JenaRosenkeller::url)
.map(JenaRosenkeller::open)
.map(JenaRosenkeller::preload)
.map(JenaRosenkeller::parse)
.map(JenaRosenkeller::loadEvent);
}
private static Optional<Appointment> loadEvent(URL url) {
private static Result<List<String>> findEventUrls(Result<Tag> tagResult) {
return switch (tagResult) {
case Payload<Tag> payload -> {
List<String> urls = payload // find tag with event-id
.get()
.find("id", val -> val.startsWith("event-"))
.stream()
.map(t -> t.find("class", "ect-event-url"::equals))
.flatMap(List::stream)
.map(t -> t.get("href"))
.toList();
yield Payload.of(urls);
}
case Error<Tag> error -> error.transform();
default -> Error.format("Invalid parameter: %s", tagResult.getClass().getSimpleName());
};
}
private static Result<Tag> parse(Result<InputStream> inputStream) {
return switch (inputStream) {
case Payload<InputStream> payload -> XMLParser.parse(payload.get());
case Error<InputStream> error -> error.transform();
default -> Error.of("Invalid parameter: %s".formatted(inputStream.getClass().getSimpleName()));
};
}
private static Result<InputStream> preload(Result<InputStream> inputStream) {
switch (inputStream){
case Payload<InputStream> payload:
try {
return Payload.of(XMLParser.preload(payload.get()));
} catch (IOException e) {
return Error.of("Failed to buffer data from %s".formatted(payload), e);
}
case Error<InputStream> error:
return error.transform();
default:
return Error.format("Invalid parameter: %s", inputStream.getClass().getSimpleName());
}
}
private static Result<InputStream> open(Result<URL> url) {
switch (url) {
case Payload<URL> payload:
try {
return Payload.of(payload.get().openConnection().getInputStream());
} catch (IOException e) {
return Error.of("Failed to open %s".formatted(payload), e);
}
case Error<URL> error:
return error.transform();
default:
return Error.format("Invalid parameter: %s", url.getClass().getSimpleName());
}
}
private static Result<Appointment> loadEvent(Result<Tag> domResult) {
switch (domResult) {
case Payload<Tag> payload:
var tag = payload.get();
Optional<String> title = tag.find("class", s -> s.endsWith("single-event-title")) //
.stream()
.map(Tag::children)
.filter(not(List::isEmpty))
.map(List::getFirst)
.map(Tag::toString)
.findAny();
if (title.isPresent()) {
var appointment = new EmptyAppointment() {
@Override
public String title() {
return title.get();
}
};
return Payload.of(appointment);
}
return Error.of("Could not find appointment title");
case Error<Tag> err:
return err.transform();
default:
return Error.format("Invalid parameter: %s", domResult.getClass().getSimpleName());
}
}
private static Optional<Appointment> nope(URL url) {
try {
var input = url.openConnection().getInputStream();
input = XMLParser.preload(input);
@@ -76,16 +209,19 @@ public class JenaRosenkeller implements Importer {
}
}
private static Stream<String> findEventUrls(Tag tag) {
return tag.find("id", val -> val.startsWith("event-")).stream().map(t -> t.find("class", "ect-event-url" ::equals)).flatMap(List::stream).map(t -> t.get("href"));
}
private static Optional<URL> url(String url) {
try {
return Optional.of(new URI(url).toURL());
} catch (MalformedURLException | URISyntaxException e) {
e.printStackTrace();
return empty();
private static Result<URL> url(Result<String> urls) {
switch (urls) {
case Payload<String> payload:
var url = payload.get();
try {
return Payload.of(new URI(url).toURL());
} catch (MalformedURLException | URISyntaxException e) {
return Error.of("Failed to create URL of %s".formatted(url), e);
}
case Error<String> err:
return err.transform();
default:
return Error.format("Invalid parameter: %s", urls.getClass().getSimpleName());
}
}
}

94
doc/database.dia
View File

@@ -88,6 +88,7 @@
</dia:composite>
</dia:attribute>
</dia:diagramdata>
<dia:layer name="Hintergrund" visible="true" connectable="false"/>
<dia:layer name="Hintergrund" visible="true" connectable="true" active="true">
<dia:object type="Flowchart - Box" version="0" id="O0">
<dia:attribute name="obj_pos">
@@ -181,13 +182,13 @@
</dia:object>
<dia:object type="Flowchart - Box" version="0" id="O2">
<dia:attribute name="obj_pos">
<dia:point val="11,9"/>
<dia:point val="11,11"/>
</dia:attribute>
<dia:attribute name="obj_bb">
<dia:rectangle val="10.95,8.95;16.05,11.05"/>
<dia:rectangle val="10.95,10.95;16.05,13.05"/>
</dia:attribute>
<dia:attribute name="elem_corner">
<dia:point val="11,9"/>
<dia:point val="11,11"/>
</dia:attribute>
<dia:attribute name="elem_width">
<dia:real val="5"/>
@@ -213,7 +214,7 @@
<dia:real val="0.80000000000000004"/>
</dia:attribute>
<dia:attribute name="pos">
<dia:point val="13.5,10.195"/>
<dia:point val="13.5,12.195"/>
</dia:attribute>
<dia:attribute name="color">
<dia:color val="#000000ff"/>
@@ -226,13 +227,13 @@
</dia:object>
<dia:object type="Flowchart - Box" version="0" id="O3">
<dia:attribute name="obj_pos">
<dia:point val="11,11"/>
<dia:point val="11,13"/>
</dia:attribute>
<dia:attribute name="obj_bb">
<dia:rectangle val="10.95,10.95;16.05,13.05"/>
<dia:rectangle val="10.95,12.95;16.05,15.05"/>
</dia:attribute>
<dia:attribute name="elem_corner">
<dia:point val="11,11"/>
<dia:point val="11,13"/>
</dia:attribute>
<dia:attribute name="elem_width">
<dia:real val="5"/>
@@ -258,7 +259,7 @@
<dia:real val="0.80000000000000004"/>
</dia:attribute>
<dia:attribute name="pos">
<dia:point val="13.5,12.195"/>
<dia:point val="13.5,14.195"/>
</dia:attribute>
<dia:attribute name="color">
<dia:color val="#000000ff"/>
@@ -271,13 +272,13 @@
</dia:object>
<dia:object type="Flowchart - Box" version="0" id="O4">
<dia:attribute name="obj_pos">
<dia:point val="11,13"/>
<dia:point val="11,15"/>
</dia:attribute>
<dia:attribute name="obj_bb">
<dia:rectangle val="10.95,12.95;16.05,15.05"/>
<dia:rectangle val="10.95,14.95;16.05,17.05"/>
</dia:attribute>
<dia:attribute name="elem_corner">
<dia:point val="11,13"/>
<dia:point val="11,15"/>
</dia:attribute>
<dia:attribute name="elem_width">
<dia:real val="5"/>
@@ -303,7 +304,7 @@
<dia:real val="0.80000000000000004"/>
</dia:attribute>
<dia:attribute name="pos">
<dia:point val="13.5,14.195"/>
<dia:point val="13.5,16.195"/>
</dia:attribute>
<dia:attribute name="color">
<dia:color val="#000000ff"/>
@@ -316,13 +317,13 @@
</dia:object>
<dia:object type="Flowchart - Box" version="0" id="O5">
<dia:attribute name="obj_pos">
<dia:point val="11,15"/>
<dia:point val="11,17"/>
</dia:attribute>
<dia:attribute name="obj_bb">
<dia:rectangle val="10.95,14.95;16.05,17.05"/>
<dia:rectangle val="10.95,16.95;16.05,19.05"/>
</dia:attribute>
<dia:attribute name="elem_corner">
<dia:point val="11,15"/>
<dia:point val="11,17"/>
</dia:attribute>
<dia:attribute name="elem_width">
<dia:real val="5"/>
@@ -348,7 +349,7 @@
<dia:real val="0.80000000000000004"/>
</dia:attribute>
<dia:attribute name="pos">
<dia:point val="13.5,16.195"/>
<dia:point val="13.5,18.195"/>
</dia:attribute>
<dia:attribute name="color">
<dia:color val="#000000ff"/>
@@ -361,13 +362,13 @@
</dia:object>
<dia:object type="Flowchart - Box" version="0" id="O6">
<dia:attribute name="obj_pos">
<dia:point val="11,17"/>
<dia:point val="11,19"/>
</dia:attribute>
<dia:attribute name="obj_bb">
<dia:rectangle val="10.95,16.95;16.05,19.05"/>
<dia:rectangle val="10.95,18.95;16.05,21.05"/>
</dia:attribute>
<dia:attribute name="elem_corner">
<dia:point val="11,17"/>
<dia:point val="11,19"/>
</dia:attribute>
<dia:attribute name="elem_width">
<dia:real val="5"/>
@@ -393,7 +394,7 @@
<dia:real val="0.80000000000000004"/>
</dia:attribute>
<dia:attribute name="pos">
<dia:point val="13.5,18.195"/>
<dia:point val="13.5,20.195"/>
</dia:attribute>
<dia:attribute name="color">
<dia:color val="#000000ff"/>
@@ -1596,16 +1597,16 @@
</dia:object>
<dia:object type="Flowchart - Box" version="0" id="O35">
<dia:attribute name="obj_pos">
<dia:point val="12,27"/>
<dia:point val="11.9962,27"/>
</dia:attribute>
<dia:attribute name="obj_bb">
<dia:rectangle val="11.95,26.95;16.08,29.05"/>
<dia:rectangle val="11.9462,26.95;16.0837,29.05"/>
</dia:attribute>
<dia:attribute name="elem_corner">
<dia:point val="12,27"/>
<dia:point val="11.9962,27"/>
</dia:attribute>
<dia:attribute name="elem_width">
<dia:real val="4.0300000000000002"/>
<dia:real val="4.0374999999999996"/>
</dia:attribute>
<dia:attribute name="elem_height">
<dia:real val="2"/>
@@ -1684,5 +1685,50 @@
</dia:composite>
</dia:attribute>
</dia:object>
<dia:object type="Flowchart - Box" version="0" id="O37">
<dia:attribute name="obj_pos">
<dia:point val="11,9"/>
</dia:attribute>
<dia:attribute name="obj_bb">
<dia:rectangle val="10.95,8.95;16.05,11.05"/>
</dia:attribute>
<dia:attribute name="elem_corner">
<dia:point val="11,9"/>
</dia:attribute>
<dia:attribute name="elem_width">
<dia:real val="5"/>
</dia:attribute>
<dia:attribute name="elem_height">
<dia:real val="2"/>
</dia:attribute>
<dia:attribute name="show_background">
<dia:boolean val="true"/>
</dia:attribute>
<dia:attribute name="padding">
<dia:real val="0.5"/>
</dia:attribute>
<dia:attribute name="text">
<dia:composite type="text">
<dia:attribute name="string">
<dia:string>#title#</dia:string>
</dia:attribute>
<dia:attribute name="font">
<dia:font family="sans" style="0" name="Helvetica"/>
</dia:attribute>
<dia:attribute name="height">
<dia:real val="0.80000000000000004"/>
</dia:attribute>
<dia:attribute name="pos">
<dia:point val="13.5,10.195"/>
</dia:attribute>
<dia:attribute name="color">
<dia:color val="#000000ff"/>
</dia:attribute>
<dia:attribute name="alignment">
<dia:enum val="1"/>
</dia:attribute>
</dia:composite>
</dia:attribute>
</dia:object>
</dia:layer>
</dia:diagram>