9 changed files with 277 additions and 29 deletions
@ -0,0 +1,229 @@
@@ -0,0 +1,229 @@
|
||||
/* © SRSoftware 2024 */ |
||||
package de.srsoftware.cal.importer.erfurt; |
||||
|
||||
import static de.srsoftware.cal.Util.parseGermanTime; |
||||
import static de.srsoftware.tools.Error.error; |
||||
import static de.srsoftware.tools.Result.transform; |
||||
import static de.srsoftware.tools.Tag.*; |
||||
import static de.srsoftware.tools.TagFilter.*; |
||||
import static java.nio.charset.StandardCharsets.UTF_8; |
||||
|
||||
import de.srsoftware.cal.BaseImporter; |
||||
import de.srsoftware.cal.Util; |
||||
import de.srsoftware.cal.api.Attachment; |
||||
import de.srsoftware.cal.api.Coords; |
||||
import de.srsoftware.cal.api.Link; |
||||
import de.srsoftware.tools.Payload; |
||||
import de.srsoftware.tools.Result; |
||||
import de.srsoftware.tools.Tag; |
||||
import java.io.ByteArrayInputStream; |
||||
import java.io.ByteArrayOutputStream; |
||||
import java.io.IOException; |
||||
import java.io.InputStream; |
||||
import java.security.NoSuchAlgorithmException; |
||||
import java.time.LocalDate; |
||||
import java.time.LocalTime; |
||||
import java.util.HashSet; |
||||
import java.util.List; |
||||
import java.util.Objects; |
||||
import java.util.function.Predicate; |
||||
|
||||
public class GewerkschaftshausErfurt extends BaseImporter { |
||||
private static final String MUSEUMSKELLER = "Museumskeller, Juri-Gagarin-Ring 140a, 99084 Erfurt"; |
||||
private static final String HSD_LOC = "Gewerkschaftshaus, Juri-Gagarin-Ring 150, 99084 Erfurt"; |
||||
private static final Coords COORDS_MUSEUM = new Coords(50.98196, 11.03554); |
||||
private static final Coords COORDS_HSD = new Coords(50.98271, 11.0349); |
||||
|
||||
public GewerkschaftshausErfurt() throws NoSuchAlgorithmException { |
||||
super(); |
||||
} |
||||
|
||||
@Override |
||||
protected String baseUrl() { |
||||
return "https://hsd-erfurt.de"; |
||||
} |
||||
|
||||
@Override |
||||
public String description() { |
||||
return "Importer für Events des Gewerkschaftshaus` Erfurt"; |
||||
} |
||||
|
||||
@Override |
||||
protected List<Attachment> extractAttachments(Tag eventTag) { |
||||
return super.extractAttachments(eventTag).stream() |
||||
.filter(att -> !att.url().toString().contains("/img/social")) |
||||
.toList(); |
||||
} |
||||
|
||||
@Override |
||||
protected Predicate<Tag> extractAttachmentsFilter() { |
||||
return attributeHas(ID,"main-content"); |
||||
} |
||||
|
||||
@Override |
||||
protected Predicate<Tag> extractDescriptionFilter() { |
||||
return attributeHas(CLASS,"all-description"); |
||||
} |
||||
|
||||
@Override |
||||
protected Result<Coords> extractCoords(Tag eventTag) { |
||||
var res = super.extractLocation(eventTag); |
||||
if (res.optional().isEmpty()) return transform(res); |
||||
var location = res.optional().get(); |
||||
var lower = location.toLowerCase(); |
||||
if (lower.contains("museum")) return Payload.of(COORDS_MUSEUM); |
||||
if (lower.contains("keller")) return Payload.of(COORDS_MUSEUM); |
||||
if (lower.contains("hsd")) return Payload.of(COORDS_HSD); |
||||
if (lower.contains("haus")) return Payload.of(COORDS_HSD); |
||||
if (lower.contains("bier")) return Payload.of(COORDS_MUSEUM); |
||||
if (lower.contains("garten")) return Payload.of(COORDS_MUSEUM); |
||||
return error("unknown location: %s",location); |
||||
} |
||||
|
||||
@Override |
||||
protected Predicate<Tag> extractEndDateFilter() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
protected Predicate<Tag> extractEndTimeFilter() { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
protected Predicate<Tag> extractEventTagFilter() { |
||||
return attributeHas(ID,"main-content"); |
||||
} |
||||
|
||||
@Override |
||||
protected Result<List<String>> extractEventUrls(Result<Tag> programPage) { |
||||
var opt = programPage.optional(); |
||||
if (opt.isEmpty()) return transform(programPage); |
||||
var list = opt.get().find(attributeEquals(ID,"events-list")).stream() |
||||
.flatMap(tag -> tag.find(IS_ANCHOR).stream()) |
||||
.map(a -> a.get(HREF)) |
||||
.filter(Objects::nonNull) |
||||
.map(link -> link.contains("://") ? link : baseUrl()+link) |
||||
.filter(link -> link.contains("/events/")) |
||||
.distinct() |
||||
.toList(); |
||||
return Payload.of(list); |
||||
} |
||||
|
||||
@Override |
||||
protected List<Link> extractLinks(Tag appointmentTag) { |
||||
return super.extractLinks(appointmentTag).stream() |
||||
.filter(link -> !link.url().toString().contains("#")) |
||||
.toList(); |
||||
} |
||||
|
||||
@Override |
||||
protected Predicate<Tag> extractLinksFilter() { |
||||
return attributeHas(CLASS,"event-details"); |
||||
} |
||||
|
||||
@Override |
||||
protected Result<String> extractLocation(Tag eventTag) { |
||||
var res = super.extractLocation(eventTag); |
||||
if (res.optional().isEmpty()) return transform(res); |
||||
var location = res.optional().get(); |
||||
var lower = location.toLowerCase(); |
||||
if (lower.contains("museum")) return Payload.of(MUSEUMSKELLER); |
||||
if (lower.contains("keller")) return Payload.of(MUSEUMSKELLER); |
||||
if (lower.contains("hsd")) return Payload.of(HSD_LOC); |
||||
if (lower.contains("haus")) return Payload.of(HSD_LOC); |
||||
if (lower.contains("bier")) return Payload.of(MUSEUMSKELLER); |
||||
if (lower.contains("garten")) return Payload.of(MUSEUMSKELLER); |
||||
LOG.log(System.Logger.Level.WARNING, "unknown location: {0}",location); |
||||
return Payload.of(location); |
||||
} |
||||
|
||||
@Override |
||||
protected Predicate<Tag> extractLocationFilter() { |
||||
return attributeHas(CLASS,"event-time-place"); |
||||
} |
||||
|
||||
@Override |
||||
protected Predicate<Tag> extractStartDateFilter() { |
||||
return attributeHas(CLASS,"event-date"); |
||||
} |
||||
|
||||
@Override |
||||
protected Predicate<Tag> extractStartTimeFilter() { |
||||
return attributeHas(CLASS,"event-time-place"); |
||||
} |
||||
|
||||
@Override |
||||
protected List<String> extractTags(Tag eventTag) { |
||||
var tags = new HashSet<String>(); |
||||
tags.add("Erfurt"); |
||||
var res = super.extractLocation(eventTag); |
||||
if (res.optional().isPresent()) { |
||||
var location = res.optional().get(); |
||||
var lower = location.toLowerCase(); |
||||
if (lower.contains("museum")||lower.contains("keller")||lower.contains("bier")||lower.contains("garten")) tags.add("Museumskeller"); |
||||
if (lower.contains("hsd")||lower.contains("haus")) { |
||||
tags.add("HSD"); |
||||
tags.add("Gewerkschaftshaus"); |
||||
} |
||||
} |
||||
return List.copyOf(tags); |
||||
} |
||||
|
||||
@Override |
||||
protected Predicate<Tag> extractTitleFilter() { |
||||
return ofType("h2"); |
||||
} |
||||
|
||||
@Override |
||||
protected Result<LocalDate> parseEndDate(String string) { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
protected Result<LocalTime> parseEndTime(String string) { |
||||
return null; |
||||
} |
||||
|
||||
@Override |
||||
protected Result<LocalDate> parseStartDate(String string) { |
||||
return Util.parseGermanDateWithoutYear(string); |
||||
} |
||||
|
||||
@Override |
||||
protected Result<LocalTime> parseStartTime(String string) { |
||||
return parseGermanTime(string); |
||||
} |
||||
|
||||
/** |
||||
* Die HSD haut einen haufen invaliden Code mit raus |
||||
* Also schneiden wir den kompletten header ab... |
||||
* @param inputStream eingehender InputStream, verpackt in Result |
||||
* @return ausgehender InputStream, verpackt in Result |
||||
*/ |
||||
@Override |
||||
protected Result<InputStream> preload(Result<InputStream> inputStream) { |
||||
var opt = inputStream.optional(); |
||||
if (opt.isEmpty()) return transform(inputStream); |
||||
try { |
||||
var input = opt.get(); |
||||
var bos = new ByteArrayOutputStream(); |
||||
input.transferTo(bos); |
||||
input.close(); |
||||
|
||||
String code = bos.toString(UTF_8) |
||||
// mitigate <img alt="image description contains title that contains single quote (') breaks parser" />
|
||||
.replaceAll("(<img.*) alt=[\"][^\"]*[\"](.*>)","$1$2") |
||||
// mitigate <a script="anchor code contains title that contains unmatched single quote (') that breaks parser" />
|
||||
.replaceAll("(<a.*) onclick=[\"][^\"]*[\"](.*>)","$1$2"); |
||||
return Payload.of(new ByteArrayInputStream(code.getBytes(UTF_8))); |
||||
} catch (IOException e) { |
||||
return error(e, "Failed to buffer data from %s", inputStream); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
protected String programURL() { |
||||
return baseUrl(); |
||||
} |
||||
} |
||||
@ -0,0 +1,14 @@
@@ -0,0 +1,14 @@
|
||||
/* © SRSoftware 2024 */ |
||||
import static org.junit.jupiter.api.Assertions.assertEquals; |
||||
|
||||
import org.junit.jupiter.api.Test; |
||||
|
||||
public class RegexTest { |
||||
@Test |
||||
public void test(){ |
||||
var code = " <img id=\"nope\" alt=\"that's crap\" class=\"test\" >\n"; |
||||
code = code.replaceAll("(<img.*) alt=[\"][^\"]*[\"](.*>)","$1$2"); |
||||
assertEquals(" <img id=\"nope\" class=\"test\" >\n",code); |
||||
|
||||
} |
||||
} |
||||
Loading…
Reference in new issue