userAgent();
+
+ /** Time to wait for a connection to be established. */
+ @WithDefault("5s")
+ Duration connectTimeout();
+
+ /** Time to wait for SSL handshake to complete. */
+ @WithDefault("5s")
+ Duration sslHandshakeTimeout();
+
+ /** Time to wait between two packets before timeout. */
+ @WithDefault("10s")
+ Duration socketTimeout();
+
+ /** Time to wait for the full response to be received. */
+ @WithDefault("10s")
+ Duration responseTimeout();
+
+ /** Time to live for a connection in the pool. */
+ @WithDefault("30s")
+ Duration connectionTimeToLive();
+
+ /** Time between eviction runs for idle connections. */
+ @WithDefault("1m")
+ Duration idleConnectionsEvictionInterval();
+
+ /**
+ * If a feed is larger than this, it will be discarded to prevent memory issues while
+ * parsing the feed.
+ */
+ @WithDefault("5M")
+ MemorySize maxResponseSize();
+
+ /**
+ * Prevent access to local addresses to mitigate server-side request forgery (SSRF) attacks,
+ * which could potentially expose internal resources.
+ *
+ * You may want to enable this if you host a public instance of CommaFeed with
+ * registrations open.
+ */
+ @WithDefault("true")
+ boolean blockLocalAddresses();
+
+ /** HTTP client cache configuration */
+ @ConfigDocSection
+ HttpClientCache cache();
+ }
+
+ interface HttpClientCache {
+ /**
+ * Whether to enable the cache. This cache is used to avoid spamming feeds in short bursts
+ * (e.g. when subscribing to a feed for the first time or when clicking "fetch all my feeds
+ * now").
+ */
+ @WithDefault("true")
+ boolean enabled();
+
+ /** Maximum amount of memory the cache can use. */
+ @WithDefault("10M")
+ MemorySize maximumMemorySize();
+
+ /** Duration after which an entry is removed from the cache. */
+ @WithDefault("1m")
+ Duration expiration();
+ }
+
+ interface FeedRefresh {
+ /** Default amount of time CommaFeed will wait before refreshing a feed. */
+ @WithDefault("5m")
+ Duration interval();
+
+ /**
+ * Maximum amount of time CommaFeed will wait before refreshing a feed. This is used as an
+ * upper bound when:
+ *
+ *
+ * - an error occurs while refreshing a feed and we're backing off exponentially
+ *
- we receive a Cache-Control header from the feed
+ *
- we receive a Retry-After header from the feed
+ *
+ */
+ @WithDefault("4h")
+ Duration maxInterval();
+
+ /**
+ * If enabled, CommaFeed will calculate the next refresh time based on the feed's average
+ * time between entries and the time since the last entry was published. The interval will
+ * be sometimes between the default refresh interval (`commafeed.feed-refresh.interval`) and
+ * the maximum refresh interval (`commafeed.feed-refresh.max-interval`).
+ *
+ * See {@link FeedRefreshIntervalCalculator} for details.
+ */
+ @WithDefault("true")
+ boolean intervalEmpirical();
+
+ /** Feed refresh engine error handling settings. */
+ @ConfigDocSection
+ FeedRefreshErrorHandling errors();
+
+ /** Amount of http threads used to fetch feeds. */
+ @Min(1)
+ @WithDefault("3")
+ int httpThreads();
+
+ /** Amount of threads used to insert new entries in the database. */
+ @Min(1)
+ @WithDefault("1")
+ int databaseThreads();
+
+ /**
+ * Duration after which a user is considered inactive. Feeds for inactive users are not
+ * refreshed until they log in again.
+ *
+ *
0 to disable.
+ */
+ @WithDefault("0")
+ Duration userInactivityPeriod();
+
+ /**
+ * Duration after which the evaluation of a filtering expresion to mark an entry as read is
+ * considered to have timed out.
+ */
+ @WithDefault("500ms")
+ Duration filteringExpressionEvaluationTimeout();
+
+ /**
+ * Duration after which the "Fetch all my feeds now" action is available again after use to
+ * avoid spamming feeds.
+ */
+ @WithDefault("0")
+ Duration forceRefreshCooldownDuration();
+ }
+
+ interface PushNotifications {
+ /** Whether to enable push notifications to notify users of new entries in their feeds. */
+ @WithDefault("true")
+ boolean enabled();
+
+ /** Amount of threads used to send external notifications about new entries. */
+ @Min(1)
+ @WithDefault("5")
+ int threads();
+
+ /**
+ * Maximum amount of notifications that can be queued before new notifications are
+ * discarded.
+ */
+ @Min(1)
+ @WithDefault("100")
+ int queueCapacity();
+ }
+
+ interface FeedRefreshErrorHandling {
+ /** Number of retries before backoff is applied. */
+ @Min(0)
+ @WithDefault("3")
+ int retriesBeforeBackoff();
+
+ /**
+ * Duration to wait before retrying after an error. Will be multiplied by the number of
+ * errors since the last successful fetch.
+ */
+ @WithDefault("1h")
+ Duration backoffInterval();
+ }
+
+ interface Database {
+ /**
+ * Timeout applied to all database queries.
+ *
+ *
0 to disable.
+ */
+ @WithDefault("0")
+ Duration queryTimeout();
+
+ /** Database cleanup settings. */
+ @ConfigDocSection
+ Cleanup cleanup();
+
+ interface Cleanup {
+ /**
+ * Maximum age of feed entries in the database. Older entries will be deleted.
+ *
+ *
0 to disable.
+ */
+ @WithDefault("365d")
+ Duration entriesMaxAge();
+
+ /**
+ * Maximum age of feed entry statuses (read/unread) in the database. Older statuses will
+ * be deleted.
+ *
+ *
0 to disable.
+ */
+ @WithDefault("0")
+ Duration statusesMaxAge();
+
+ /**
+ * Maximum number of entries per feed to keep in the database.
+ *
+ *
0 to disable.
+ */
+ @WithDefault("500")
+ int maxFeedCapacity();
+
+ /**
+ * Limit the number of feeds a user can subscribe to.
+ *
+ *
0 to disable.
+ */
+ @WithDefault("0")
+ int maxFeedsPerUser();
+
+ /** Rows to delete per query while cleaning up old entries. */
+ @Positive
+ @WithDefault("100")
+ int batchSize();
+
+ /** Whether to keep starred entries when cleaning up old entries. */
+ @WithDefault("true")
+ boolean keepStarredEntries();
+
+ default Instant statusesInstantThreshold() {
+ return statusesMaxAge().toMillis() > 0
+ ? Instant.now().minus(statusesMaxAge())
+ : null;
+ }
+ }
+ }
+
+ interface Users {
+ /** Whether to let users create accounts for themselves. */
+ @WithDefault("false")
+ boolean allowRegistrations();
+
+ /** Minimum password length for user accounts. */
+ @WithDefault("4")
+ int minimumPasswordLength();
+
+ /** Whether an email address is required when creating a user account. */
+ @WithDefault("false")
+ boolean emailAddressRequired();
+
+ /** Whether to create a demo account the first time the app starts. */
+ @WithDefault("false")
+ boolean createDemoAccount();
+ }
+
+ interface Websocket {
+ /**
+ * Enable websocket connection so the server can notify web clients that there are new
+ * entries for feeds.
+ */
+ @WithDefault("true")
+ boolean enabled();
+
+ /**
+ * Interval at which the client will send a ping message on the websocket to keep the
+ * connection alive.
+ */
+ @WithDefault("15m")
+ Duration pingInterval();
+
+ /**
+ * If the websocket connection is disabled or the connection is lost, the client will reload
+ * the feed tree at this interval.
+ */
+ @WithDefault("30s")
+ Duration treeReloadInterval();
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/CommaFeedConstants.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/CommaFeedConstants.java
new file mode 100644
index 000000000..43778b4b5
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/CommaFeedConstants.java
@@ -0,0 +1,8 @@
+package com.commafeed;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class CommaFeedConstants {
+ public static final String USERNAME_DEMO = "demo";
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/CommaFeedProducers.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/CommaFeedProducers.java
new file mode 100644
index 000000000..1933463f8
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/CommaFeedProducers.java
@@ -0,0 +1,24 @@
+package com.commafeed;
+
+import com.codahale.metrics.MetricRegistry;
+
+import jakarta.enterprise.inject.Produces;
+import jakarta.inject.Singleton;
+
+import java.time.InstantSource;
+
+@Singleton
+public class CommaFeedProducers {
+
+ @Produces
+ @Singleton
+ public InstantSource instantSource() {
+ return InstantSource.system();
+ }
+
+ @Produces
+ @Singleton
+ public MetricRegistry metricRegistry() {
+ return new MetricRegistry();
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/CommaFeedVersion.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/CommaFeedVersion.java
new file mode 100644
index 000000000..fcc834014
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/CommaFeedVersion.java
@@ -0,0 +1,29 @@
+package com.commafeed;
+
+import jakarta.inject.Singleton;
+
+import lombok.Getter;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+@Singleton
+@Getter
+public class CommaFeedVersion {
+
+ private final String version;
+ private final String gitCommit;
+
+ public CommaFeedVersion() throws IOException {
+ Properties properties = new Properties();
+ try (InputStream stream = getClass().getResourceAsStream("/git.properties")) {
+ if (stream != null) {
+ properties.load(stream);
+ }
+ }
+
+ this.version = properties.getProperty("git.build.version", "unknown");
+ this.gitCommit = properties.getProperty("git.commit.id.abbrev", "unknown");
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/ExceptionMappers.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/ExceptionMappers.java
new file mode 100644
index 000000000..da6f9c4e1
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/ExceptionMappers.java
@@ -0,0 +1,58 @@
+package com.commafeed;
+
+import com.commafeed.security.CookieService;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+import io.quarkus.security.AuthenticationFailedException;
+import io.quarkus.security.UnauthorizedException;
+
+import jakarta.annotation.Priority;
+import jakarta.validation.ValidationException;
+import jakarta.ws.rs.core.NewCookie;
+import jakarta.ws.rs.ext.Provider;
+
+import lombok.RequiredArgsConstructor;
+
+import org.jboss.resteasy.reactive.RestResponse;
+import org.jboss.resteasy.reactive.RestResponse.ResponseBuilder;
+import org.jboss.resteasy.reactive.RestResponse.Status;
+import org.jboss.resteasy.reactive.server.ServerExceptionMapper;
+
+@RequiredArgsConstructor
+@Provider
+@Priority(1)
+public class ExceptionMappers {
+
+ private final CookieService cookieService;
+ private final CommaFeedConfiguration config;
+
+ @ServerExceptionMapper(UnauthorizedException.class)
+ public RestResponse unauthorized(UnauthorizedException e) {
+ return RestResponse.status(
+ Status.UNAUTHORIZED,
+ new UnauthorizedResponse(e.getMessage(), config.users().allowRegistrations()));
+ }
+
+ @ServerExceptionMapper(AuthenticationFailedException.class)
+ public RestResponse authenticationFailed(
+ AuthenticationFailedException e) {
+ NewCookie logoutCookie = cookieService.buildLogoutCookie();
+ return ResponseBuilder.create(Status.UNAUTHORIZED, new AuthenticationFailed(e.getMessage()))
+ .cookie(logoutCookie)
+ .build();
+ }
+
+ @ServerExceptionMapper(ValidationException.class)
+ public RestResponse validationFailed(ValidationException e) {
+ return RestResponse.status(Status.BAD_REQUEST, new ValidationFailed(e.getMessage()));
+ }
+
+ @RegisterForReflection
+ public record UnauthorizedResponse(String message, boolean allowRegistrations) {}
+
+ @RegisterForReflection
+ public record AuthenticationFailed(String message) {}
+
+ @RegisterForReflection
+ public record ValidationFailed(String message) {}
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/JacksonCustomizer.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/JacksonCustomizer.java
new file mode 100644
index 000000000..18e8f31a0
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/JacksonCustomizer.java
@@ -0,0 +1,30 @@
+package com.commafeed;
+
+import com.codahale.metrics.json.MetricsModule;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+
+import io.quarkus.jackson.ObjectMapperCustomizer;
+
+import jakarta.inject.Singleton;
+
+import java.util.concurrent.TimeUnit;
+
+@Singleton
+public class JacksonCustomizer implements ObjectMapperCustomizer {
+ @Override
+ public void customize(ObjectMapper objectMapper) {
+ objectMapper.registerModule(new JavaTimeModule());
+
+ // read and write instants as milliseconds instead of nanoseconds
+ objectMapper
+ .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true)
+ .configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false)
+ .configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false);
+
+ // add support for serializing metrics
+ objectMapper.registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/NativeImageClasses.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/NativeImageClasses.java
new file mode 100644
index 000000000..9b5d3fffd
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/NativeImageClasses.java
@@ -0,0 +1,349 @@
+package com.commafeed;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection(
+ targets = {
+ // metrics
+ MetricRegistry.class,
+ Meter.class,
+ Gauge.class,
+ Counter.class,
+ Timer.class,
+ Histogram.class,
+
+ // rome
+ java.util.Date.class,
+ com.rometools.opml.feed.synd.impl.TreeCategoryImpl.class,
+ com.rometools.rome.feed.synd.SyndFeedImpl.class,
+ com.rometools.rome.feed.module.DCSubjectImpl.class,
+ com.rometools.rome.feed.synd.SyndEntryImpl.class,
+ com.rometools.modules.psc.types.SimpleChapter.class,
+ com.rometools.rome.feed.synd.SyndCategoryImpl.class,
+ com.rometools.rome.feed.synd.SyndImageImpl.class,
+ com.rometools.rome.feed.synd.SyndContentImpl.class,
+ com.rometools.rome.feed.synd.SyndEnclosureImpl.class,
+
+ // rome cloneable
+ com.rometools.modules.activitystreams.types.Article.class,
+ com.rometools.modules.activitystreams.types.Audio.class,
+ com.rometools.modules.activitystreams.types.Bookmark.class,
+ com.rometools.modules.activitystreams.types.Comment.class,
+ com.rometools.modules.activitystreams.types.Event.class,
+ com.rometools.modules.activitystreams.types.File.class,
+ com.rometools.modules.activitystreams.types.Folder.class,
+ com.rometools.modules.activitystreams.types.List.class,
+ com.rometools.modules.activitystreams.types.Note.class,
+ com.rometools.modules.activitystreams.types.Person.class,
+ com.rometools.modules.activitystreams.types.Photo.class,
+ com.rometools.modules.activitystreams.types.PhotoAlbum.class,
+ com.rometools.modules.activitystreams.types.Place.class,
+ com.rometools.modules.activitystreams.types.Playlist.class,
+ com.rometools.modules.activitystreams.types.Product.class,
+ com.rometools.modules.activitystreams.types.Review.class,
+ com.rometools.modules.activitystreams.types.Service.class,
+ com.rometools.modules.activitystreams.types.Song.class,
+ com.rometools.modules.activitystreams.types.Status.class,
+ com.rometools.modules.base.types.DateTimeRange.class,
+ com.rometools.modules.base.types.FloatUnit.class,
+ com.rometools.modules.base.types.GenderEnumeration.class,
+ com.rometools.modules.base.types.IntUnit.class,
+ com.rometools.modules.base.types.PriceTypeEnumeration.class,
+ com.rometools.modules.base.types.ShippingType.class,
+ com.rometools.modules.base.types.ShortDate.class,
+ com.rometools.modules.base.types.Size.class,
+ com.rometools.modules.base.types.YearType.class,
+ com.rometools.modules.content.ContentItem.class,
+ com.rometools.modules.georss.GeoRSSPoint.class,
+ com.rometools.modules.georss.geometries.Envelope.class,
+ com.rometools.modules.georss.geometries.LineString.class,
+ com.rometools.modules.georss.geometries.LinearRing.class,
+ com.rometools.modules.georss.geometries.Point.class,
+ com.rometools.modules.georss.geometries.Polygon.class,
+ com.rometools.modules.georss.geometries.Position.class,
+ com.rometools.modules.georss.geometries.PositionList.class,
+ com.rometools.modules.mediarss.types.MediaGroup.class,
+ com.rometools.modules.mediarss.types.Metadata.class,
+ com.rometools.modules.mediarss.types.Thumbnail.class,
+ com.rometools.modules.opensearch.entity.OSQuery.class,
+ com.rometools.modules.photocast.types.PhotoDate.class,
+ com.rometools.modules.sle.types.DateValue.class,
+ com.rometools.modules.sle.types.Group.class,
+ com.rometools.modules.sle.types.NumberValue.class,
+ com.rometools.modules.sle.types.Sort.class,
+ com.rometools.modules.sle.types.StringValue.class,
+ com.rometools.modules.yahooweather.types.Astronomy.class,
+ com.rometools.modules.yahooweather.types.Atmosphere.class,
+ com.rometools.modules.yahooweather.types.Condition.class,
+ com.rometools.modules.yahooweather.types.Forecast.class,
+ com.rometools.modules.yahooweather.types.Location.class,
+ com.rometools.modules.yahooweather.types.Units.class,
+ com.rometools.modules.yahooweather.types.Wind.class,
+ com.rometools.opml.feed.opml.Attribute.class,
+ com.rometools.opml.feed.opml.Opml.class,
+ com.rometools.opml.feed.opml.Outline.class,
+ com.rometools.rome.feed.atom.Category.class,
+ com.rometools.rome.feed.atom.Content.class,
+ com.rometools.rome.feed.atom.Entry.class,
+ com.rometools.rome.feed.atom.Feed.class,
+ com.rometools.rome.feed.atom.Generator.class,
+ com.rometools.rome.feed.atom.Link.class,
+ com.rometools.rome.feed.atom.Person.class,
+ com.rometools.rome.feed.rss.Category.class,
+ com.rometools.rome.feed.rss.Channel.class,
+ com.rometools.rome.feed.rss.Cloud.class,
+ com.rometools.rome.feed.rss.Content.class,
+ com.rometools.rome.feed.rss.Description.class,
+ com.rometools.rome.feed.rss.Enclosure.class,
+ com.rometools.rome.feed.rss.Guid.class,
+ com.rometools.rome.feed.rss.Image.class,
+ com.rometools.rome.feed.rss.Item.class,
+ com.rometools.rome.feed.rss.Source.class,
+ com.rometools.rome.feed.rss.TextInput.class,
+ com.rometools.rome.feed.synd.SyndLinkImpl.class,
+ com.rometools.rome.feed.synd.SyndPersonImpl.class,
+ java.util.ArrayList.class,
+
+ // rome modules
+ com.rometools.modules.sse.modules.Conflict.class,
+ com.rometools.modules.sse.modules.Conflicts.class,
+ com.rometools.modules.cc.CreativeCommonsImpl.class,
+ com.rometools.modules.feedpress.modules.FeedpressModuleImpl.class,
+ com.rometools.modules.opensearch.impl.OpenSearchModuleImpl.class,
+ com.rometools.modules.sse.modules.Sharing.class,
+ com.rometools.modules.georss.SimpleModuleImpl.class,
+ com.rometools.modules.atom.modules.AtomLinkModuleImpl.class,
+ com.rometools.modules.itunes.EntryInformationImpl.class,
+ com.rometools.modules.sse.modules.Update.class,
+ com.rometools.modules.photocast.PhotocastModuleImpl.class,
+ com.rometools.modules.itunes.FeedInformationImpl.class,
+ com.rometools.modules.yahooweather.YWeatherModuleImpl.class,
+ com.rometools.modules.feedburner.FeedBurnerImpl.class,
+ com.rometools.modules.sse.modules.Related.class,
+ com.rometools.modules.fyyd.modules.FyydModuleImpl.class,
+ com.rometools.modules.psc.modules.PodloveSimpleChapterModuleImpl.class,
+ com.rometools.modules.thr.ThreadingModuleImpl.class,
+ com.rometools.modules.sse.modules.Sync.class,
+ com.rometools.modules.sle.SimpleListExtensionImpl.class,
+ com.rometools.modules.slash.SlashImpl.class,
+ com.rometools.modules.sse.modules.History.class,
+ com.rometools.modules.georss.GMLModuleImpl.class,
+ com.rometools.modules.base.CustomTagsImpl.class,
+ com.rometools.modules.base.GoogleBaseImpl.class,
+ com.rometools.modules.sle.SleEntryImpl.class,
+ com.rometools.modules.mediarss.MediaEntryModuleImpl.class,
+ com.rometools.modules.content.ContentModuleImpl.class,
+ com.rometools.modules.georss.W3CGeoModuleImpl.class,
+ com.rometools.rome.feed.module.DCModuleImpl.class,
+ com.rometools.modules.mediarss.MediaModuleImpl.class,
+ com.rometools.rome.feed.module.SyModuleImpl.class,
+
+ // extracted from all 3 rome.properties files of rome library
+ com.rometools.rome.io.impl.RSS090Parser.class,
+ com.rometools.rome.io.impl.RSS091NetscapeParser.class,
+ com.rometools.rome.io.impl.RSS091UserlandParser.class,
+ com.rometools.rome.io.impl.RSS092Parser.class,
+ com.rometools.rome.io.impl.RSS093Parser.class,
+ com.rometools.rome.io.impl.RSS094Parser.class,
+ com.rometools.rome.io.impl.RSS10Parser.class,
+ com.rometools.rome.io.impl.RSS20wNSParser.class,
+ com.rometools.rome.io.impl.RSS20Parser.class,
+ com.rometools.rome.io.impl.Atom10Parser.class,
+ com.rometools.rome.io.impl.Atom03Parser.class,
+ com.rometools.rome.io.impl.SyModuleParser.class,
+ com.rometools.rome.io.impl.DCModuleParser.class,
+ com.rometools.rome.io.impl.RSS090Generator.class,
+ com.rometools.rome.io.impl.RSS091NetscapeGenerator.class,
+ com.rometools.rome.io.impl.RSS091UserlandGenerator.class,
+ com.rometools.rome.io.impl.RSS092Generator.class,
+ com.rometools.rome.io.impl.RSS093Generator.class,
+ com.rometools.rome.io.impl.RSS094Generator.class,
+ com.rometools.rome.io.impl.RSS10Generator.class,
+ com.rometools.rome.io.impl.RSS20Generator.class,
+ com.rometools.rome.io.impl.Atom10Generator.class,
+ com.rometools.rome.io.impl.Atom03Generator.class,
+ com.rometools.rome.feed.synd.impl.ConverterForAtom10.class,
+ com.rometools.rome.feed.synd.impl.ConverterForAtom03.class,
+ com.rometools.rome.feed.synd.impl.ConverterForRSS090.class,
+ com.rometools.rome.feed.synd.impl.ConverterForRSS091Netscape.class,
+ com.rometools.rome.feed.synd.impl.ConverterForRSS091Userland.class,
+ com.rometools.rome.feed.synd.impl.ConverterForRSS092.class,
+ com.rometools.rome.feed.synd.impl.ConverterForRSS093.class,
+ com.rometools.rome.feed.synd.impl.ConverterForRSS094.class,
+ com.rometools.rome.feed.synd.impl.ConverterForRSS10.class,
+ com.rometools.rome.feed.synd.impl.ConverterForRSS20.class,
+ com.rometools.modules.mediarss.io.RSS20YahooParser.class,
+ com.rometools.modules.cc.io.ModuleParserRSS2.class,
+ com.rometools.modules.content.io.ContentModuleParser.class,
+ com.rometools.modules.itunes.io.ITunesParser.class,
+ com.rometools.modules.mediarss.io.MediaModuleParser.class,
+ com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
+ com.rometools.modules.georss.SimpleParser.class,
+ com.rometools.modules.georss.W3CGeoParser.class,
+ com.rometools.modules.photocast.io.Parser.class,
+ com.rometools.modules.mediarss.io.MediaModuleParser.class,
+ com.rometools.modules.atom.io.AtomModuleParser.class,
+ com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
+ com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
+ com.rometools.modules.sle.io.ModuleParser.class,
+ com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
+ com.rometools.modules.feedpress.io.FeedpressParser.class,
+ com.rometools.modules.fyyd.io.FyydParser.class,
+ com.rometools.modules.cc.io.ModuleParserRSS2.class,
+ com.rometools.modules.content.io.ContentModuleParser.class,
+ com.rometools.modules.itunes.io.ITunesParser.class,
+ com.rometools.modules.mediarss.io.MediaModuleParser.class,
+ com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
+ com.rometools.modules.georss.SimpleParser.class,
+ com.rometools.modules.georss.W3CGeoParser.class,
+ com.rometools.modules.photocast.io.Parser.class,
+ com.rometools.modules.mediarss.io.MediaModuleParser.class,
+ com.rometools.modules.atom.io.AtomModuleParser.class,
+ com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
+ com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
+ com.rometools.modules.sle.io.ModuleParser.class,
+ com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
+ com.rometools.modules.feedpress.io.FeedpressParser.class,
+ com.rometools.modules.fyyd.io.FyydParser.class,
+ com.rometools.modules.cc.io.ModuleParserRSS1.class,
+ com.rometools.modules.content.io.ContentModuleParser.class,
+ com.rometools.modules.cc.io.ModuleParserRSS2.class,
+ com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
+ com.rometools.modules.georss.SimpleParser.class,
+ com.rometools.modules.georss.W3CGeoParser.class,
+ com.rometools.modules.photocast.io.Parser.class,
+ com.rometools.modules.mediarss.io.MediaModuleParser.class,
+ com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
+ com.rometools.modules.cc.io.ModuleParserRSS2.class,
+ com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
+ com.rometools.modules.georss.SimpleParser.class,
+ com.rometools.modules.georss.W3CGeoParser.class,
+ com.rometools.modules.photocast.io.Parser.class,
+ com.rometools.modules.mediarss.io.MediaModuleParser.class,
+ com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
+ com.rometools.modules.feedpress.io.FeedpressParser.class,
+ com.rometools.modules.fyyd.io.FyydParser.class,
+ com.rometools.modules.cc.io.ModuleParserRSS2.class,
+ com.rometools.modules.base.io.GoogleBaseParser.class,
+ com.rometools.modules.content.io.ContentModuleParser.class,
+ com.rometools.modules.slash.io.SlashModuleParser.class,
+ com.rometools.modules.itunes.io.ITunesParser.class,
+ com.rometools.modules.mediarss.io.MediaModuleParser.class,
+ com.rometools.modules.atom.io.AtomModuleParser.class,
+ com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
+ com.rometools.modules.georss.SimpleParser.class,
+ com.rometools.modules.georss.W3CGeoParser.class,
+ com.rometools.modules.photocast.io.Parser.class,
+ com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
+ com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
+ com.rometools.modules.sle.io.ItemParser.class,
+ com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
+ com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
+ com.rometools.modules.cc.io.ModuleParserRSS1.class,
+ com.rometools.modules.base.io.GoogleBaseParser.class,
+ com.rometools.modules.base.io.CustomTagParser.class,
+ com.rometools.modules.content.io.ContentModuleParser.class,
+ com.rometools.modules.slash.io.SlashModuleParser.class,
+ com.rometools.modules.cc.io.ModuleParserRSS2.class,
+ com.rometools.modules.base.io.GoogleBaseParser.class,
+ com.rometools.modules.base.io.CustomTagParser.class,
+ com.rometools.modules.slash.io.SlashModuleParser.class,
+ com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
+ com.rometools.modules.georss.SimpleParser.class,
+ com.rometools.modules.georss.W3CGeoParser.class,
+ com.rometools.modules.photocast.io.Parser.class,
+ com.rometools.modules.mediarss.io.MediaModuleParser.class,
+ com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
+ com.rometools.modules.cc.io.ModuleParserRSS2.class,
+ com.rometools.modules.base.io.GoogleBaseParser.class,
+ com.rometools.modules.base.io.CustomTagParser.class,
+ com.rometools.modules.slash.io.SlashModuleParser.class,
+ com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
+ com.rometools.modules.georss.SimpleParser.class,
+ com.rometools.modules.georss.W3CGeoParser.class,
+ com.rometools.modules.photocast.io.Parser.class,
+ com.rometools.modules.mediarss.io.MediaModuleParser.class,
+ com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
+ com.rometools.modules.thr.io.ThreadingModuleParser.class,
+ com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
+ com.rometools.modules.cc.io.CCModuleGenerator.class,
+ com.rometools.modules.content.io.ContentModuleGenerator.class,
+ com.rometools.modules.itunes.io.ITunesGenerator.class,
+ com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
+ com.rometools.modules.georss.SimpleGenerator.class,
+ com.rometools.modules.georss.W3CGeoGenerator.class,
+ com.rometools.modules.photocast.io.Generator.class,
+ com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
+ com.rometools.modules.atom.io.AtomModuleGenerator.class,
+ com.rometools.modules.sle.io.ModuleGenerator.class,
+ com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
+ com.rometools.modules.feedpress.io.FeedpressGenerator.class,
+ com.rometools.modules.fyyd.io.FyydGenerator.class,
+ com.rometools.modules.content.io.ContentModuleGenerator.class,
+ com.rometools.modules.cc.io.CCModuleGenerator.class,
+ com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
+ com.rometools.modules.georss.SimpleGenerator.class,
+ com.rometools.modules.georss.W3CGeoGenerator.class,
+ com.rometools.modules.photocast.io.Generator.class,
+ com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
+ com.rometools.modules.cc.io.CCModuleGenerator.class,
+ com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
+ com.rometools.modules.georss.SimpleGenerator.class,
+ com.rometools.modules.georss.W3CGeoGenerator.class,
+ com.rometools.modules.photocast.io.Generator.class,
+ com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
+ com.rometools.modules.feedpress.io.FeedpressGenerator.class,
+ com.rometools.modules.fyyd.io.FyydGenerator.class,
+ com.rometools.modules.cc.io.CCModuleGenerator.class,
+ com.rometools.modules.base.io.GoogleBaseGenerator.class,
+ com.rometools.modules.base.io.CustomTagGenerator.class,
+ com.rometools.modules.content.io.ContentModuleGenerator.class,
+ com.rometools.modules.slash.io.SlashModuleGenerator.class,
+ com.rometools.modules.itunes.io.ITunesGenerator.class,
+ com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
+ com.rometools.modules.georss.SimpleGenerator.class,
+ com.rometools.modules.georss.W3CGeoGenerator.class,
+ com.rometools.modules.photocast.io.Generator.class,
+ com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
+ com.rometools.modules.atom.io.AtomModuleGenerator.class,
+ com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
+ com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
+ com.rometools.modules.base.io.GoogleBaseGenerator.class,
+ com.rometools.modules.content.io.ContentModuleGenerator.class,
+ com.rometools.modules.slash.io.SlashModuleGenerator.class,
+ com.rometools.modules.cc.io.CCModuleGenerator.class,
+ com.rometools.modules.base.io.GoogleBaseGenerator.class,
+ com.rometools.modules.base.io.CustomTagGenerator.class,
+ com.rometools.modules.slash.io.SlashModuleGenerator.class,
+ com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
+ com.rometools.modules.georss.SimpleGenerator.class,
+ com.rometools.modules.georss.W3CGeoGenerator.class,
+ com.rometools.modules.photocast.io.Generator.class,
+ com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
+ com.rometools.modules.cc.io.CCModuleGenerator.class,
+ com.rometools.modules.base.io.CustomTagGenerator.class,
+ com.rometools.modules.slash.io.SlashModuleGenerator.class,
+ com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
+ com.rometools.modules.georss.SimpleGenerator.class,
+ com.rometools.modules.georss.W3CGeoGenerator.class,
+ com.rometools.modules.photocast.io.Generator.class,
+ com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
+ com.rometools.modules.thr.io.ThreadingModuleGenerator.class,
+ com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
+ com.rometools.modules.mediarss.io.MediaModuleParser.class,
+ com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
+ com.rometools.opml.io.impl.OPML10Generator.class,
+ com.rometools.opml.io.impl.OPML20Generator.class,
+ com.rometools.opml.io.impl.OPML10Parser.class,
+ com.rometools.opml.io.impl.OPML20Parser.class,
+ com.rometools.opml.feed.synd.impl.ConverterForOPML10.class,
+ com.rometools.opml.feed.synd.impl.ConverterForOPML20.class,
+ })
+public class NativeImageClasses {}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/Digests.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/Digests.java
new file mode 100644
index 000000000..a8a5a7d51
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/Digests.java
@@ -0,0 +1,29 @@
+package com.commafeed.backend;
+
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+
+import lombok.experimental.UtilityClass;
+
+import java.nio.charset.StandardCharsets;
+
+@UtilityClass
+@SuppressWarnings("deprecation")
+public class Digests {
+
+ public static String sha1Hex(byte[] input) {
+ return hashBytesToHex(Hashing.sha1(), input);
+ }
+
+ public static String sha1Hex(String input) {
+ return hashBytesToHex(Hashing.sha1(), input.getBytes(StandardCharsets.UTF_8));
+ }
+
+ public static String md5Hex(String input) {
+ return hashBytesToHex(Hashing.md5(), input.getBytes(StandardCharsets.UTF_8));
+ }
+
+ private static String hashBytesToHex(HashFunction function, byte[] input) {
+ return function.hashBytes(input).toString();
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/HttpClientFactory.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/HttpClientFactory.java
new file mode 100644
index 000000000..ff6c88b10
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/HttpClientFactory.java
@@ -0,0 +1,227 @@
+package com.commafeed.backend;
+
+import com.commafeed.CommaFeedConfiguration;
+import com.commafeed.CommaFeedVersion;
+import com.google.common.net.HttpHeaders;
+
+import inet.ipaddr.IPAddress;
+import inet.ipaddr.IPAddressNetwork;
+import inet.ipaddr.IPAddressString;
+import inet.ipaddr.ipv4.IPv4Address;
+import inet.ipaddr.ipv6.IPv6Address;
+
+import jakarta.inject.Singleton;
+
+import lombok.RequiredArgsConstructor;
+
+import nl.altindag.ssl.SSLFactory;
+import nl.altindag.ssl.apache5.util.Apache5SslUtils;
+
+import org.apache.hc.client5.http.DnsResolver;
+import org.apache.hc.client5.http.SystemDefaultDnsResolver;
+import org.apache.hc.client5.http.config.ConnectionConfig;
+import org.apache.hc.client5.http.config.TlsConfig;
+import org.apache.hc.client5.http.entity.DeflateInputStream;
+import org.apache.hc.client5.http.entity.InputStreamFactory;
+import org.apache.hc.client5.http.entity.compress.ContentCoding;
+import org.apache.hc.client5.http.impl.DefaultRedirectStrategy;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.io.HttpClientConnectionManager;
+import org.apache.hc.client5.http.protocol.RedirectStrategy;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.HttpException;
+import org.apache.hc.core5.http.HttpRequest;
+import org.apache.hc.core5.http.HttpResponse;
+import org.apache.hc.core5.http.message.BasicHeader;
+import org.apache.hc.core5.http.protocol.HttpContext;
+import org.apache.hc.core5.util.TimeValue;
+import org.apache.hc.core5.util.Timeout;
+import org.brotli.dec.BrotliInputStream;
+
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.SequencedMap;
+import java.util.zip.GZIPInputStream;
+
+@Singleton
+@RequiredArgsConstructor
+public class HttpClientFactory {
+
+ private static final DnsResolver DNS_RESOLVER = SystemDefaultDnsResolver.INSTANCE;
+ private static final IPAddress CGNAT_RANGE = new IPAddressString("100.64.0.0/10").getAddress();
+
+ private final CommaFeedConfiguration config;
+ private final CommaFeedVersion version;
+
+ public CloseableHttpClient newClient(int poolSize) {
+ PoolingHttpClientConnectionManager connectionManager =
+ newConnectionManager(config, poolSize);
+ String userAgent =
+ config.httpClient()
+ .userAgent()
+ .orElseGet(
+ () ->
+ String.format(
+ "CommaFeed/%s (https://github.com/Athou/commafeed)",
+ version.getVersion()));
+ return newClient(config, connectionManager, userAgent);
+ }
+
+ private CloseableHttpClient newClient(
+ CommaFeedConfiguration config,
+ HttpClientConnectionManager connectionManager,
+ String userAgent) {
+ List headers = new ArrayList<>();
+ headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
+ headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
+ headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
+
+ SequencedMap contentDecoderMap = new LinkedHashMap<>();
+ contentDecoderMap.put(ContentCoding.GZIP.token(), GZIPInputStream::new);
+ contentDecoderMap.put(ContentCoding.DEFLATE.token(), DeflateInputStream::new);
+ contentDecoderMap.put(ContentCoding.BROTLI.token(), BrotliInputStream::new);
+
+ RedirectStrategy redirectStrategy =
+ config.httpClient().blockLocalAddresses()
+ ? new BlockLocalAddressesRedirectStrategy(DNS_RESOLVER)
+ : new DefaultRedirectStrategy();
+
+ return HttpClientBuilder.create()
+ .disableConnectionState()
+ .useSystemProperties()
+ .disableAutomaticRetries()
+ .disableCookieManagement()
+ .setUserAgent(userAgent)
+ .setDefaultHeaders(headers)
+ .setConnectionManager(connectionManager)
+ .evictExpiredConnections()
+ .evictIdleConnections(
+ TimeValue.of(config.httpClient().idleConnectionsEvictionInterval()))
+ .setContentDecoderRegistry(new LinkedHashMap<>(contentDecoderMap))
+ .setRedirectStrategy(redirectStrategy)
+ .build();
+ }
+
+ private PoolingHttpClientConnectionManager newConnectionManager(
+ CommaFeedConfiguration config, int poolSize) {
+ SSLFactory sslFactory =
+ SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
+ DnsResolver dnsResolver =
+ config.httpClient().blockLocalAddresses()
+ ? new BlockLocalAddressesDnsResolver(DNS_RESOLVER)
+ : DNS_RESOLVER;
+
+ return PoolingHttpClientConnectionManagerBuilder.create()
+ .setTlsSocketStrategy(Apache5SslUtils.toTlsSocketStrategy(sslFactory))
+ .setDefaultConnectionConfig(
+ ConnectionConfig.custom()
+ .setConnectTimeout(Timeout.of(config.httpClient().connectTimeout()))
+ .setSocketTimeout(Timeout.of(config.httpClient().socketTimeout()))
+ .setTimeToLive(
+ Timeout.of(config.httpClient().connectionTimeToLive()))
+ .build())
+ .setDefaultTlsConfig(
+ TlsConfig.custom()
+ .setHandshakeTimeout(
+ Timeout.of(config.httpClient().sslHandshakeTimeout()))
+ .build())
+ .setMaxConnPerRoute(poolSize)
+ .setMaxConnTotal(poolSize)
+ .setDnsResolver(dnsResolver)
+ .build();
+ }
+
+ private static boolean isLocalAddress(InetAddress address) {
+ return isLocalAddress(new IPAddressNetwork.IPAddressGenerator().from(address));
+ }
+
+ private static boolean isLocalAddress(IPAddress ip) {
+ if (ip.isLocal() || ip.isLoopback() || ip.isMulticast() || CGNAT_RANGE.contains(ip)) {
+ return true;
+ }
+
+ if (!ip.isIPv6()) {
+ return false;
+ }
+
+ // IPv6 transition mechanisms embed an IPv4 address that must be validated too, otherwise
+ // they could be used to smuggle a blocked IPv4 target past the IPv6-only checks above
+ IPv6Address ipv6 = ip.toIPv6();
+ if (ipv6.isIPv4Mapped() || ipv6.isIPv4Compatible() || ipv6.isWellKnownIPv4Translatable()) {
+ // IPv4-mapped (::ffff:x.x.x.x), IPv4-compatible (::x.x.x.x) and NAT64
+ // (64:ff9b::/96, RFC 6052) addresses all embed the IPv4 address in the lowest 32 bits
+ return isLocalAddress(ipv6.getEmbeddedIPv4Address());
+ }
+ if (ipv6.is6To4()) {
+ // 6to4 (2002::/16, RFC 3056) embeds the IPv4 address in bits 16-47
+ return isLocalAddress(ipv6.get6To4IPv4Address());
+ }
+ if (ipv6.isTeredo()) {
+ // Teredo (2001::/32, RFC 4380) embeds the IPv4 address in the lowest 32 bits,
+ // obfuscated with a bitwise complement
+ IPv4Address obfuscated = ipv6.getEmbeddedIPv4Address();
+ return isLocalAddress(new IPv4Address(~obfuscated.intValue()));
+ }
+
+ return false;
+ }
+
+ private record BlockLocalAddressesDnsResolver(DnsResolver delegate) implements DnsResolver {
+ @Override
+ public InetAddress[] resolve(String host) throws UnknownHostException {
+ InetAddress[] addresses = delegate.resolve(host);
+ for (InetAddress addr : addresses) {
+ if (isLocalAddress(addr)) {
+ throw new UnknownHostException(
+ "Access to local address blocked: " + addr.getHostAddress());
+ }
+ }
+ return addresses;
+ }
+
+ @Override
+ public String resolveCanonicalHostname(String host) throws UnknownHostException {
+ return delegate.resolveCanonicalHostname(host);
+ }
+ }
+
+ @RequiredArgsConstructor
+ private static class BlockLocalAddressesRedirectStrategy extends DefaultRedirectStrategy {
+
+ private final DnsResolver delegate;
+
+ @Override
+ public URI getLocationURI(HttpRequest request, HttpResponse response, HttpContext context)
+ throws HttpException {
+ URI redirectUri = super.getLocationURI(request, response, context);
+
+ String host = redirectUri.getHost();
+ if (host == null) {
+ throw new HttpException("Redirect URI does not have a host: " + redirectUri);
+ }
+
+ InetAddress[] addresses;
+ try {
+ addresses = delegate.resolve(host);
+ } catch (UnknownHostException e) {
+ throw new HttpException("Unknown host: " + host);
+ }
+
+ for (InetAddress addr : addresses) {
+ if (isLocalAddress(addr)) {
+ throw new HttpException(
+ "Access to local address blocked: " + addr.getHostAddress());
+ }
+ }
+
+ return redirectUri;
+ }
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java
new file mode 100644
index 000000000..9c44c6264
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java
@@ -0,0 +1,368 @@
+package com.commafeed.backend;
+
+import com.codahale.metrics.MetricRegistry;
+import com.commafeed.CommaFeedConfiguration;
+import com.commafeed.CommaFeedConfiguration.HttpClientCache;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.Iterables;
+import com.google.common.io.ByteStreams;
+import com.google.common.net.HttpHeaders;
+
+import jakarta.inject.Singleton;
+import jakarta.ws.rs.core.CacheControl;
+
+import lombok.Builder;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Lombok;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hc.client5.http.config.RequestConfig;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.client5.http.protocol.RedirectLocations;
+import org.apache.hc.client5.http.utils.DateUtils;
+import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpStatus;
+import org.apache.hc.core5.http.NameValuePair;
+import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
+import org.apache.hc.core5.util.Timeout;
+import org.jboss.resteasy.reactive.common.headers.CacheControlDelegate;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.InstantSource;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+
+/** Smart HTTP getter: handles gzip, ssl, last modified and etag headers */
+@Singleton
+@Slf4j
+public class HttpGetter {
+ private final CommaFeedConfiguration config;
+ private final InstantSource instantSource;
+ private final CloseableHttpClient client;
+ private final Cache cache;
+
+ public HttpGetter(
+ CommaFeedConfiguration config,
+ InstantSource instantSource,
+ HttpClientFactory httpClientFactory,
+ MetricRegistry metrics) {
+ this.config = config;
+ this.instantSource = instantSource;
+ this.client = httpClientFactory.newClient(config.feedRefresh().httpThreads());
+ this.cache = newCache(config);
+
+ metrics.registerGauge(
+ MetricRegistry.name(getClass(), "cache", "size"),
+ () -> cache == null ? 0 : cache.size());
+ metrics.registerGauge(
+ MetricRegistry.name(getClass(), "cache", "memoryUsage"),
+ () ->
+ cache == null
+ ? 0
+ : cache.asMap().values().stream()
+ .mapToInt(e -> ArrayUtils.getLength(e.content))
+ .sum());
+ }
+
+ public HttpResult get(String url)
+ throws IOException,
+ NotModifiedException,
+ TooManyRequestsException,
+ SchemeNotAllowedException {
+ return get(HttpRequest.builder(url).build());
+ }
+
+ public HttpResult get(HttpRequest request)
+ throws IOException,
+ NotModifiedException,
+ TooManyRequestsException,
+ SchemeNotAllowedException {
+ URI uri = URI.create(request.getUrl());
+ ensureHttpScheme(uri.getScheme());
+
+ final HttpResponse response;
+ if (cache == null) {
+ response = invoke(request);
+ } else {
+ try {
+ response = cache.get(request, () -> invoke(request));
+ } catch (ExecutionException e) {
+ if (e.getCause() instanceof IOException ioe) {
+ throw ioe;
+ } else {
+ throw Lombok.sneakyThrow(e);
+ }
+ }
+ }
+
+ int code = response.code();
+ if (code == HttpStatus.SC_TOO_MANY_REQUESTS
+ || code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.retryAfter() != null) {
+ throw new TooManyRequestsException(response.retryAfter());
+ }
+
+ if (code == HttpStatus.SC_NOT_MODIFIED) {
+ throw new NotModifiedException("'304 - not modified' http code received");
+ }
+
+ if (code >= 300) {
+ throw new HttpResponseException(code, "Server returned HTTP error code " + code);
+ }
+
+ String lastModifiedHeader = response.lastModifiedHeader();
+ String eTagHeader = response.eTagHeader();
+
+ Duration validFor =
+ Optional.ofNullable(response.cacheControl())
+ .filter(cc -> cc.getMaxAge() >= 0)
+ .map(cc -> Duration.ofSeconds(cc.getMaxAge()))
+ .orElse(Duration.ZERO);
+
+ return new HttpResult(
+ response.content(),
+ response.contentType(),
+ lastModifiedHeader,
+ eTagHeader,
+ response.urlAfterRedirect(),
+ validFor);
+ }
+
+ private void ensureHttpScheme(String scheme) throws SchemeNotAllowedException {
+ if (!"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme)) {
+ throw new SchemeNotAllowedException(scheme);
+ }
+ }
+
+ private HttpResponse invoke(HttpRequest request) throws IOException {
+ log.debug("fetching {}", request.getUrl());
+
+ HttpClientContext context = HttpClientContext.create();
+ context.setRequestConfig(
+ RequestConfig.custom()
+ .setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
+ // causes issues with some feeds
+ // see https://github.com/Athou/commafeed/issues/1572
+ // and https://issues.apache.org/jira/browse/HTTPCLIENT-2344
+ .setProtocolUpgradeEnabled(false)
+ .build());
+
+ return client.execute(
+ request.toClassicHttpRequest(),
+ context,
+ resp -> {
+ byte[] content =
+ resp.getEntity() == null
+ ? null
+ : toByteArray(
+ resp.getEntity(),
+ config.httpClient().maxResponseSize().asLongValue());
+ int code = resp.getCode();
+ String lastModifiedHeader =
+ Optional.ofNullable(resp.getFirstHeader(HttpHeaders.LAST_MODIFIED))
+ .map(NameValuePair::getValue)
+ .map(StringUtils::trimToNull)
+ .orElse(null);
+ String eTagHeader =
+ Optional.ofNullable(resp.getFirstHeader(HttpHeaders.ETAG))
+ .map(NameValuePair::getValue)
+ .map(StringUtils::trimToNull)
+ .orElse(null);
+
+ CacheControl cacheControl =
+ Optional.ofNullable(resp.getFirstHeader(HttpHeaders.CACHE_CONTROL))
+ .map(NameValuePair::getValue)
+ .map(StringUtils::trimToNull)
+ .map(HttpGetter::toCacheControl)
+ .orElse(null);
+
+ Instant retryAfter =
+ Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER))
+ .map(NameValuePair::getValue)
+ .map(StringUtils::trimToNull)
+ .map(this::toInstant)
+ .orElse(null);
+
+ String contentType =
+ Optional.ofNullable(resp.getEntity())
+ .map(HttpEntity::getContentType)
+ .orElse(null);
+ String urlAfterRedirect =
+ Optional.ofNullable(context.getRedirectLocations())
+ .map(RedirectLocations::getAll)
+ .map(l -> Iterables.getLast(l, null))
+ .map(URI::toString)
+ .orElse(request.getUrl());
+
+ return new HttpResponse(
+ code,
+ lastModifiedHeader,
+ eTagHeader,
+ cacheControl,
+ retryAfter,
+ content,
+ contentType,
+ urlAfterRedirect);
+ });
+ }
+
+ private static CacheControl toCacheControl(String headerValue) {
+ try {
+ return CacheControlDelegate.INSTANCE.fromString(headerValue);
+ } catch (Exception e) {
+ log.debug("Invalid Cache-Control header: {}", headerValue);
+ return null;
+ }
+ }
+
+ private Instant toInstant(String headerValue) {
+ if (headerValue == null) {
+ return null;
+ }
+
+ if (StringUtils.isNumeric(headerValue)) {
+ return instantSource.instant().plusSeconds(Long.parseLong(headerValue));
+ }
+
+ return DateUtils.parseStandardDate(headerValue);
+ }
+
+ private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
+ if (entity.getContentLength() > maxBytes) {
+ throw new IOException(
+ "Response size (%s bytes) exceeds the maximum allowed size (%s bytes)"
+ .formatted(entity.getContentLength(), maxBytes));
+ }
+
+ try (InputStream input = entity.getContent()) {
+ if (input == null) {
+ return null;
+ }
+
+ byte[] bytes = ByteStreams.limit(input, maxBytes + 1).readAllBytes();
+ if (bytes.length > maxBytes) {
+ throw new IOException(
+ "Response size exceeds the maximum allowed size (%s bytes)"
+ .formatted(maxBytes));
+ }
+ return bytes;
+ }
+ }
+
+ private static Cache newCache(CommaFeedConfiguration config) {
+ HttpClientCache cacheConfig = config.httpClient().cache();
+ if (!cacheConfig.enabled()) {
+ return null;
+ }
+
+ return CacheBuilder.newBuilder()
+ .weigher(
+ (HttpRequest key, HttpResponse value) ->
+ value.content() != null ? value.content().length : 0)
+ .maximumWeight(cacheConfig.maximumMemorySize().asLongValue())
+ .expireAfterWrite(cacheConfig.expiration())
+ .build();
+ }
+
+ public static class SchemeNotAllowedException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ public SchemeNotAllowedException(String scheme) {
+ super("Scheme not allowed: " + scheme);
+ }
+ }
+
+ @Getter
+ public static class NotModifiedException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ /** if the value of this header changed, this is its new value */
+ private final String newLastModifiedHeader;
+
+ /** if the value of this header changed, this is its new value */
+ private final String newEtagHeader;
+
+ public NotModifiedException(String message) {
+ this(message, null, null);
+ }
+
+ public NotModifiedException(
+ String message, String newLastModifiedHeader, String newEtagHeader) {
+ super(message);
+ this.newLastModifiedHeader = newLastModifiedHeader;
+ this.newEtagHeader = newEtagHeader;
+ }
+ }
+
+ @RequiredArgsConstructor
+ @Getter
+ public static class TooManyRequestsException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ private final Instant retryAfter;
+ }
+
+ @Getter
+ public static class HttpResponseException extends IOException {
+ private static final long serialVersionUID = 1L;
+
+ private final int code;
+
+ public HttpResponseException(int code, String message) {
+ super(message);
+ this.code = code;
+ }
+ }
+
+ @Builder(builderMethodName = "")
+ @EqualsAndHashCode
+ @Getter
+ public static class HttpRequest {
+ private String url;
+ private String lastModified;
+ private String eTag;
+
+ public static HttpRequestBuilder builder(String url) {
+ return new HttpRequestBuilder().url(url);
+ }
+
+ public ClassicHttpRequest toClassicHttpRequest() {
+ ClassicHttpRequest req = ClassicRequestBuilder.get(url).build();
+ if (lastModified != null) {
+ req.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
+ }
+ if (eTag != null) {
+ req.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
+ }
+ return req;
+ }
+ }
+
+ private record HttpResponse(
+ int code,
+ String lastModifiedHeader,
+ String eTagHeader,
+ CacheControl cacheControl,
+ Instant retryAfter,
+ byte[] content,
+ String contentType,
+ String urlAfterRedirect) {}
+
+ public record HttpResult(
+ byte[] content,
+ String contentType,
+ String lastModifiedSince,
+ String eTag,
+ String urlAfterRedirect,
+ Duration validFor) {}
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/Urls.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/Urls.java
new file mode 100644
index 000000000..cf54db166
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/Urls.java
@@ -0,0 +1,129 @@
+package com.commafeed.backend;
+
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang3.Strings;
+import org.netpreserve.urlcanon.Canonicalizer;
+import org.netpreserve.urlcanon.ParsedUrl;
+
+import java.net.URI;
+import java.util.Locale;
+import java.util.regex.Pattern;
+
+@UtilityClass
+@Slf4j
+public class Urls {
+
+ private static final Pattern QUESTION_MARK = Pattern.compile(Pattern.quote("?"));
+
+ public static boolean isHttp(String url) {
+ if (url == null) {
+ return false;
+ }
+
+ return url.toLowerCase(Locale.ROOT).startsWith("http://");
+ }
+
+ public static boolean isHttps(String url) {
+ if (url == null) {
+ return false;
+ }
+
+ return url.toLowerCase(Locale.ROOT).startsWith("https://");
+ }
+
+ public static boolean isAbsolute(String url) {
+ return isHttp(url) || isHttps(url);
+ }
+
+ /** remove malicious 'javascript: 'URLs * */
+ public static String sanitize(String url) {
+ if (url == null) {
+ return null;
+ }
+
+ if (!isHttp(url) && !isHttps(url)) {
+ return null;
+ }
+
+ return url;
+ }
+
+ /**
+ * @param relativeUrl the url of the entry
+ * @param feedLink the url of the feed as described in the feed
+ * @param feedUrl the url of the feed that we used to fetch the feed
+ * @return an absolute url pointing to the entry
+ */
+ public static String toAbsolute(String relativeUrl, String feedLink, String feedUrl) {
+ String baseUrl = (feedLink != null && isAbsolute(feedLink)) ? feedLink : feedUrl;
+ if (baseUrl == null) {
+ return null;
+ }
+
+ try {
+ return URI.create(baseUrl).resolve(relativeUrl).toString();
+ } catch (IllegalArgumentException e) {
+ log.debug(
+ "Unable to create absolute url from relative url: {} base: {}",
+ relativeUrl,
+ baseUrl,
+ e);
+ return null;
+ }
+ }
+
+ public static String removeTrailingSlash(String url) {
+ if (url == null) {
+ return null;
+ }
+
+ if (url.endsWith("/")) {
+ url = url.substring(0, url.length() - 1);
+ }
+ return url;
+ }
+
+ /**
+ * Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to
+ * identify a feed and avoid duplicates
+ */
+ public static String normalize(String url) {
+ if (url == null) {
+ return null;
+ }
+
+ ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
+ Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
+ String normalized = parsedUrl.toString();
+ if (normalized == null) {
+ normalized = url;
+ }
+
+ // convert to lower case, the url probably won't work in some cases
+ // after that but we don't care we just want to compare urls to avoid
+ // duplicates
+ normalized = normalized.toLowerCase();
+
+ // store all urls as http
+ if (normalized.startsWith("https")) {
+ normalized = "http" + normalized.substring(5);
+ }
+
+ // remove the www. part
+ normalized = normalized.replace("//www.", "//");
+
+ // feedproxy redirects to feedburner
+ normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
+
+ // feedburner feeds have a special treatment
+ if (QUESTION_MARK.split(normalized)[0].contains("feedburner.com")) {
+ normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
+ normalized = QUESTION_MARK.split(normalized)[0];
+ normalized = Strings.CS.removeEnd(normalized, "/");
+ }
+
+ return normalized;
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java
new file mode 100644
index 000000000..58bded578
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java
@@ -0,0 +1,78 @@
+package com.commafeed.backend.dao;
+
+import com.commafeed.backend.model.FeedCategory;
+import com.commafeed.backend.model.QFeedCategory;
+import com.commafeed.backend.model.QUser;
+import com.commafeed.backend.model.User;
+import com.querydsl.core.types.Predicate;
+
+import jakarta.inject.Singleton;
+import jakarta.persistence.EntityManager;
+
+import java.util.List;
+import java.util.Objects;
+
+@Singleton
+public class FeedCategoryDAO extends GenericDAO {
+
+ private static final QFeedCategory CATEGORY = QFeedCategory.feedCategory;
+
+ public FeedCategoryDAO(EntityManager entityManager) {
+ super(entityManager, FeedCategory.class);
+ }
+
+ public List findAll(User user) {
+ return query().selectFrom(CATEGORY)
+ .where(CATEGORY.user.eq(user))
+ .join(CATEGORY.user, QUser.user)
+ .fetchJoin()
+ .fetch();
+ }
+
+ public FeedCategory findById(User user, Long id) {
+ return query().selectFrom(CATEGORY)
+ .where(CATEGORY.user.eq(user), CATEGORY.id.eq(id))
+ .fetchOne();
+ }
+
+ public FeedCategory findByName(User user, String name, FeedCategory parent) {
+ Predicate parentPredicate;
+ if (parent == null) {
+ parentPredicate = CATEGORY.parent.isNull();
+ } else {
+ parentPredicate = CATEGORY.parent.eq(parent);
+ }
+ return query().selectFrom(CATEGORY)
+ .where(CATEGORY.user.eq(user), CATEGORY.name.eq(name), parentPredicate)
+ .fetchOne();
+ }
+
+ public List findByParent(User user, FeedCategory parent) {
+ Predicate parentPredicate;
+ if (parent == null) {
+ parentPredicate = CATEGORY.parent.isNull();
+ } else {
+ parentPredicate = CATEGORY.parent.eq(parent);
+ }
+ return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), parentPredicate).fetch();
+ }
+
+ public List findAllChildrenCategories(User user, FeedCategory parent) {
+ return findAll(user).stream().filter(c -> isChild(c, parent)).toList();
+ }
+
+ private boolean isChild(FeedCategory child, FeedCategory parent) {
+ if (parent == null) {
+ return true;
+ }
+ boolean isChild = false;
+ while (child != null) {
+ if (Objects.equals(child.getId(), parent.getId())) {
+ isChild = true;
+ break;
+ }
+ child = child.getParent();
+ }
+ return isChild;
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedDAO.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedDAO.java
new file mode 100644
index 000000000..2a9ce5c2c
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedDAO.java
@@ -0,0 +1,72 @@
+package com.commafeed.backend.dao;
+
+import com.commafeed.backend.model.Feed;
+import com.commafeed.backend.model.QFeed;
+import com.commafeed.backend.model.QFeedSubscription;
+import com.querydsl.jpa.JPAExpressions;
+import com.querydsl.jpa.impl.JPAQuery;
+
+import jakarta.inject.Singleton;
+import jakarta.persistence.EntityManager;
+
+import org.apache.commons.lang3.Strings;
+
+import java.time.Instant;
+import java.util.List;
+
+@Singleton
+public class FeedDAO extends GenericDAO {
+
+ private static final QFeed FEED = QFeed.feed;
+ private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
+
+ public FeedDAO(EntityManager entityManager) {
+ super(entityManager, Feed.class);
+ }
+
+ public List findByIds(List id) {
+ return query().selectFrom(FEED).where(FEED.id.in(id)).fetch();
+ }
+
+ public List findNextUpdatable(int count, Instant lastLoginThreshold) {
+ JPAQuery query =
+ query().selectFrom(FEED)
+ .distinct()
+ // join on subscriptions to only refresh feeds that have subscribers
+ .join(SUBSCRIPTION)
+ .on(SUBSCRIPTION.feed.eq(FEED))
+ .where(
+ FEED.disabledUntil
+ .isNull()
+ .or(FEED.disabledUntil.lt(Instant.now())));
+
+ if (lastLoginThreshold != null) {
+ query.join(SUBSCRIPTION.user).where(SUBSCRIPTION.user.lastLogin.gt(lastLoginThreshold));
+ }
+
+ return query.orderBy(FEED.disabledUntil.asc()).limit(count).fetch();
+ }
+
+ public void setDisabledUntil(List feedIds, Instant date) {
+ updateQuery(FEED).set(FEED.disabledUntil, date).where(FEED.id.in(feedIds)).execute();
+ }
+
+ public Feed findByUrl(String normalizedUrl, String normalizedUrlHash) {
+ return query()
+ .selectFrom(FEED)
+ .where(FEED.normalizedUrlHash.eq(normalizedUrlHash))
+ .fetch()
+ .stream()
+ .filter(f -> Strings.CS.equals(normalizedUrl, f.getNormalizedUrl()))
+ .findFirst()
+ .orElse(null);
+ }
+
+ public List findWithoutSubscriptions(int max) {
+ QFeedSubscription sub = QFeedSubscription.feedSubscription;
+ return query().selectFrom(FEED)
+ .where(JPAExpressions.selectOne().from(sub).where(sub.feed.eq(FEED)).notExists())
+ .limit(max)
+ .fetch();
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java
new file mode 100644
index 000000000..b1f0a886c
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java
@@ -0,0 +1,40 @@
+package com.commafeed.backend.dao;
+
+import com.commafeed.backend.model.FeedEntryContent;
+import com.commafeed.backend.model.QFeedEntry;
+import com.commafeed.backend.model.QFeedEntryContent;
+
+import jakarta.inject.Singleton;
+import jakarta.persistence.EntityManager;
+
+import java.util.List;
+
+@Singleton
+public class FeedEntryContentDAO extends GenericDAO {
+
+ private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
+ private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
+
+ public FeedEntryContentDAO(EntityManager entityManager) {
+ super(entityManager, FeedEntryContent.class);
+ }
+
+ public List findExisting(String contentHash, String titleHash) {
+ return query().select(CONTENT)
+ .from(CONTENT)
+ .where(CONTENT.contentHash.eq(contentHash), CONTENT.titleHash.eq(titleHash))
+ .fetch();
+ }
+
+ public long deleteWithoutEntries(int max) {
+ List ids =
+ query().select(CONTENT.id)
+ .from(CONTENT)
+ .leftJoin(ENTRY)
+ .on(ENTRY.content.id.eq(CONTENT.id))
+ .where(ENTRY.id.isNull())
+ .limit(max)
+ .fetch();
+ return deleteQuery(CONTENT).where(CONTENT.id.in(ids)).execute();
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
new file mode 100644
index 000000000..7e77ae8cd
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
@@ -0,0 +1,106 @@
+package com.commafeed.backend.dao;
+
+import com.commafeed.backend.model.Feed;
+import com.commafeed.backend.model.FeedEntry;
+import com.commafeed.backend.model.QFeedEntry;
+import com.google.common.collect.Lists;
+import com.querydsl.core.Tuple;
+import com.querydsl.core.types.dsl.NumberExpression;
+import com.querydsl.jpa.impl.JPAQuery;
+
+import jakarta.inject.Singleton;
+import jakarta.persistence.EntityManager;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+@Singleton
+public class FeedEntryDAO extends GenericDAO {
+
+ private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
+ private static final int IN_CLAUSE_BATCH_SIZE = 1000;
+
+ public FeedEntryDAO(EntityManager entityManager) {
+ super(entityManager, FeedEntry.class);
+ }
+
+ public FeedEntry findExisting(String guidHash, Feed feed) {
+ return query().select(ENTRY)
+ .from(ENTRY)
+ .where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed))
+ .limit(1)
+ .fetchOne();
+ }
+
+ public Set findExistingGuidHashes(Set guidHashes, Feed feed) {
+ if (guidHashes.isEmpty()) {
+ return Set.of();
+ }
+
+ Set result = new HashSet<>();
+ for (List batch :
+ Lists.partition(new ArrayList<>(guidHashes), IN_CLAUSE_BATCH_SIZE)) {
+ result.addAll(
+ query().select(ENTRY.guidHash)
+ .from(ENTRY)
+ .where(ENTRY.feed.eq(feed), ENTRY.guidHash.in(batch))
+ .fetch());
+ }
+ return result;
+ }
+
+ public List findFeedsExceedingCapacity(
+ long maxCapacity, long max, boolean keepStarredEntries) {
+ NumberExpression count = ENTRY.id.count();
+ JPAQuery query = query().select(ENTRY.feed.id, count).from(ENTRY);
+
+ if (keepStarredEntries) {
+ query.where(Predicates.isNotStarred(ENTRY));
+ }
+
+ return query.groupBy(ENTRY.feed).having(count.gt(maxCapacity)).limit(max).fetch().stream()
+ .map(t -> new FeedCapacity(t.get(ENTRY.feed.id), t.get(count)))
+ .toList();
+ }
+
+ public int delete(Long feedId, long max) {
+ List list =
+ query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).limit(max).fetch();
+ return delete(list);
+ }
+
+ /** Delete entries older than a certain date */
+ public int deleteEntriesOlderThan(Instant olderThan, long max, boolean keepStarredEntries) {
+ JPAQuery query =
+ query().selectFrom(ENTRY)
+ .where(ENTRY.published.lt(olderThan))
+ .orderBy(ENTRY.published.asc())
+ .limit(max);
+
+ if (keepStarredEntries) {
+ query.where(Predicates.isNotStarred(ENTRY));
+ }
+
+ return delete(query.fetch());
+ }
+
+ /** Delete the oldest entries of a feed */
+ public int deleteOldEntries(Long feedId, long max, boolean keepStarredEntries) {
+ JPAQuery query =
+ query().selectFrom(ENTRY)
+ .where(ENTRY.feed.id.eq(feedId))
+ .orderBy(ENTRY.published.asc())
+ .limit(max);
+
+ if (keepStarredEntries) {
+ query.where(Predicates.isNotStarred(ENTRY));
+ }
+
+ return delete(query.fetch());
+ }
+
+ public record FeedCapacity(Long id, Long capacity) {}
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java
new file mode 100644
index 000000000..8e953a24c
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java
@@ -0,0 +1,339 @@
+package com.commafeed.backend.dao;
+
+import com.commafeed.CommaFeedConfiguration;
+import com.commafeed.backend.feed.FeedEntryKeyword;
+import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
+import com.commafeed.backend.model.FeedEntry;
+import com.commafeed.backend.model.FeedEntryStatus;
+import com.commafeed.backend.model.FeedEntryTag;
+import com.commafeed.backend.model.FeedSubscription;
+import com.commafeed.backend.model.QFeed;
+import com.commafeed.backend.model.QFeedEntry;
+import com.commafeed.backend.model.QFeedEntryContent;
+import com.commafeed.backend.model.QFeedEntryStatus;
+import com.commafeed.backend.model.QFeedEntryTag;
+import com.commafeed.backend.model.QFeedSubscription;
+import com.commafeed.backend.model.User;
+import com.commafeed.backend.model.UserSettings.ReadingOrder;
+import com.commafeed.frontend.model.UnreadCount;
+import com.querydsl.core.BooleanBuilder;
+import com.querydsl.core.Tuple;
+import com.querydsl.core.types.dsl.Expressions;
+import com.querydsl.core.types.dsl.NumberExpression;
+import com.querydsl.jpa.impl.JPAQuery;
+
+import jakarta.inject.Singleton;
+import jakarta.persistence.EntityManager;
+
+import org.apache.commons.collections4.CollectionUtils;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Singleton
+public class FeedEntryStatusDAO extends GenericDAO {
+
+ private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
+ private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
+ private static final QFeed FEED = QFeed.feed;
+ private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
+ private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
+ private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
+
+ private final FeedEntryTagDAO feedEntryTagDAO;
+ private final CommaFeedConfiguration config;
+
+ public FeedEntryStatusDAO(
+ EntityManager entityManager,
+ FeedEntryTagDAO feedEntryTagDAO,
+ CommaFeedConfiguration config) {
+ super(entityManager, FeedEntryStatus.class);
+ this.feedEntryTagDAO = feedEntryTagDAO;
+ this.config = config;
+ }
+
+ public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) {
+ List statuses =
+ query().selectFrom(STATUS)
+ .where(STATUS.entry.eq(entry), STATUS.subscription.eq(sub))
+ .fetch();
+ FeedEntryStatus status = statuses.stream().findFirst().orElse(null);
+ return handleStatus(user, status, sub, entry);
+ }
+
+ /** creates an artificial "unread" status if status is null */
+ private FeedEntryStatus handleStatus(
+ User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
+ if (status == null) {
+ Instant statusesInstantThreshold =
+ config.database().cleanup().statusesInstantThreshold();
+ boolean read =
+ statusesInstantThreshold != null
+ && entry.getPublished().isBefore(statusesInstantThreshold);
+ status = new FeedEntryStatus(user, sub, entry);
+ status.setRead(read);
+ status.setMarkable(!read);
+ } else {
+ status.setMarkable(true);
+ }
+ return status;
+ }
+
+ private void fetchTags(User user, List statuses) {
+ Map> tagsByEntryIds =
+ feedEntryTagDAO.findByEntries(
+ user, statuses.stream().map(FeedEntryStatus::getEntry).toList());
+ for (FeedEntryStatus status : statuses) {
+ List tags = tagsByEntryIds.get(status.getEntry().getId());
+ status.setTags(tags == null ? List.of() : tags);
+ }
+ }
+
+ public List findStarred(
+ User user,
+ List keywords,
+ Instant newerThan,
+ int offset,
+ int limit,
+ ReadingOrder order,
+ boolean includeContent) {
+ JPAQuery query =
+ query().selectFrom(STATUS).where(STATUS.user.eq(user), STATUS.starred.isTrue());
+ if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
+ query.join(STATUS.entry).fetchJoin();
+ query.join(STATUS.entry.content, CONTENT).fetchJoin();
+ }
+
+ if (CollectionUtils.isNotEmpty(keywords)) {
+ applyKeywordsFilter(query, keywords);
+ }
+
+ if (newerThan != null) {
+ query.where(STATUS.entryInserted.gt(newerThan));
+ }
+
+ if (order == ReadingOrder.ASC) {
+ query.orderBy(STATUS.entryPublished.asc(), STATUS.id.asc());
+ } else {
+ query.orderBy(STATUS.entryPublished.desc(), STATUS.id.desc());
+ }
+
+ if (offset > -1) {
+ query.offset(offset);
+ }
+
+ if (limit > -1) {
+ query.limit(limit);
+ }
+
+ setTimeout(query, config.database().queryTimeout());
+
+ List statuses = query.fetch();
+ statuses.forEach(s -> s.setMarkable(true));
+ if (includeContent) {
+ fetchTags(user, statuses);
+ }
+
+ return statuses;
+ }
+
+ public List findBySubscriptions(
+ User user,
+ List subs,
+ boolean unreadOnly,
+ List keywords,
+ Instant newerThan,
+ int offset,
+ int limit,
+ ReadingOrder order,
+ boolean includeContent,
+ String tag,
+ Long minEntryId,
+ Long maxEntryId) {
+ Map> subsByFeedId =
+ subs.stream().collect(Collectors.groupingBy(s -> s.getFeed().getId()));
+
+ JPAQuery query = query().select(ENTRY, STATUS).from(ENTRY);
+ query.leftJoin(ENTRY.statuses, STATUS).on(STATUS.subscription.in(subs));
+ query.where(ENTRY.feed.id.in(subsByFeedId.keySet()));
+
+ if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
+ query.join(ENTRY.content, CONTENT).fetchJoin();
+ }
+
+ if (CollectionUtils.isNotEmpty(keywords)) {
+ applyKeywordsFilter(query, keywords);
+ }
+
+ if (unreadOnly && tag == null) {
+ query.where(buildUnreadPredicate());
+ }
+
+ if (tag != null) {
+ BooleanBuilder and = new BooleanBuilder();
+ and.and(TAG.user.id.eq(user.getId()));
+ and.and(TAG.name.eq(tag));
+ query.join(ENTRY.tags, TAG).on(and);
+ }
+
+ if (newerThan != null) {
+ query.where(ENTRY.inserted.goe(newerThan));
+ }
+
+ if (minEntryId != null) {
+ query.where(ENTRY.id.gt(minEntryId));
+ }
+
+ if (maxEntryId != null) {
+ query.where(ENTRY.id.lt(maxEntryId));
+ }
+
+ if (order != null) {
+ if (order == ReadingOrder.ASC) {
+ query.orderBy(ENTRY.published.asc(), ENTRY.id.asc());
+ } else {
+ query.orderBy(ENTRY.published.desc(), ENTRY.id.desc());
+ }
+ }
+
+ if (offset > -1) {
+ query.offset(offset);
+ }
+
+ if (limit > -1) {
+ query.limit(limit);
+ }
+
+ setTimeout(query, config.database().queryTimeout());
+
+ List statuses = new ArrayList<>();
+ List tuples = query.fetch();
+ for (Tuple tuple : tuples) {
+ FeedEntry e = tuple.get(ENTRY);
+ FeedEntryStatus s = tuple.get(STATUS);
+ for (FeedSubscription sub : subsByFeedId.get(e.getFeed().getId())) {
+ statuses.add(handleStatus(user, s, sub, e));
+ }
+ }
+
+ if (includeContent) {
+ fetchTags(user, statuses);
+ }
+
+ return statuses;
+ }
+
+ private void applyKeywordsFilter(JPAQuery> query, List keywords) {
+ for (FeedEntryKeyword keyword : keywords) {
+ BooleanBuilder or = new BooleanBuilder();
+ or.or(CONTENT.content.containsIgnoreCase(keyword.keyword()));
+ or.or(CONTENT.title.containsIgnoreCase(keyword.keyword()));
+ if (keyword.mode() == Mode.EXCLUDE) {
+ or.not();
+ }
+ query.where(or);
+ }
+ }
+
+ public UnreadCount getUnreadCount(FeedSubscription sub) {
+ JPAQuery query =
+ query().select(ENTRY.count(), ENTRY.published.max())
+ .from(ENTRY)
+ .leftJoin(ENTRY.statuses, STATUS)
+ .on(STATUS.subscription.eq(sub))
+ .where(ENTRY.feed.eq(sub.getFeed()))
+ .where(buildUnreadPredicate());
+
+ Tuple tuple = query.fetchOne();
+ Long count = tuple.get(ENTRY.count());
+ Instant published = tuple.get(ENTRY.published.max());
+ return new UnreadCount(sub.getId(), count == null ? 0 : count, published);
+ }
+
+ private BooleanBuilder buildUnreadPredicate() {
+ BooleanBuilder or = new BooleanBuilder();
+ or.or(STATUS.read.isNull());
+ or.or(STATUS.read.isFalse());
+
+ Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
+ if (statusesInstantThreshold != null) {
+ return or.and(ENTRY.published.goe(statusesInstantThreshold));
+ } else {
+ return or;
+ }
+ }
+
+ public long deleteOldStatuses(Instant olderThan, int limit) {
+ List ids =
+ query().select(STATUS.id)
+ .from(STATUS)
+ .where(STATUS.entryInserted.lt(olderThan), STATUS.starred.isFalse())
+ .limit(limit)
+ .fetch();
+ return deleteQuery(STATUS).where(STATUS.id.in(ids)).execute();
+ }
+
+ public long autoMarkAsRead(int limit) {
+ Instant now = Instant.now();
+
+ BooleanBuilder where = new BooleanBuilder();
+ where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.isNotNull());
+ where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.gt(0));
+
+ NumberExpression daysDiff =
+ Expressions.numberTemplate(
+ Integer.class, "TIMESTAMPDIFF(DAY, {0}, {1})", ENTRY.published, now);
+ where.and(daysDiff.goe(SUBSCRIPTION.autoMarkAsReadAfterDays));
+
+ where.and(buildUnreadPredicate());
+
+ List tuples =
+ query().select(ENTRY, STATUS, SUBSCRIPTION)
+ .from(ENTRY)
+ .join(ENTRY.feed, FEED)
+ .join(SUBSCRIPTION)
+ .on(SUBSCRIPTION.feed.eq(FEED))
+ .leftJoin(ENTRY.statuses, STATUS)
+ .on(STATUS.subscription.eq(SUBSCRIPTION))
+ .where(where)
+ .limit(limit)
+ .fetch();
+
+ long updated = 0;
+
+ // Update existing statuses
+ List statusIdsToUpdate =
+ tuples.stream()
+ .map(t -> t.get(STATUS))
+ .filter(s -> s != null && s.getId() != null)
+ .map(FeedEntryStatus::getId)
+ .distinct()
+ .toList();
+
+ if (!statusIdsToUpdate.isEmpty()) {
+ updated +=
+ updateQuery(STATUS)
+ .where(STATUS.id.in(statusIdsToUpdate))
+ .set(STATUS.read, true)
+ .execute();
+ }
+
+ // Insert new statuses for entries without existing status
+ for (Tuple tuple : tuples) {
+ FeedEntryStatus status = tuple.get(STATUS);
+ if (status == null || status.getId() == null) {
+ FeedEntry entry = tuple.get(ENTRY);
+ FeedSubscription sub = tuple.get(SUBSCRIPTION);
+ FeedEntryStatus newStatus = new FeedEntryStatus(sub.getUser(), sub, entry);
+ newStatus.setRead(true);
+ persist(newStatus);
+ updated++;
+ }
+ }
+
+ return updated;
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java
new file mode 100644
index 000000000..807ba986d
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java
@@ -0,0 +1,40 @@
+package com.commafeed.backend.dao;
+
+import com.commafeed.backend.model.FeedEntry;
+import com.commafeed.backend.model.FeedEntryTag;
+import com.commafeed.backend.model.QFeedEntryTag;
+import com.commafeed.backend.model.User;
+
+import jakarta.inject.Singleton;
+import jakarta.persistence.EntityManager;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Singleton
+public class FeedEntryTagDAO extends GenericDAO {
+
+ private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
+
+ public FeedEntryTagDAO(EntityManager entityManager) {
+ super(entityManager, FeedEntryTag.class);
+ }
+
+ public List findByUser(User user) {
+ return query().selectDistinct(TAG.name).from(TAG).where(TAG.user.eq(user)).fetch();
+ }
+
+ public List findByEntry(User user, FeedEntry entry) {
+ return query().selectFrom(TAG).where(TAG.user.eq(user), TAG.entry.eq(entry)).fetch();
+ }
+
+ public Map> findByEntries(User user, List entries) {
+ return query()
+ .selectFrom(TAG)
+ .where(TAG.user.eq(user), TAG.entry.in(entries))
+ .fetch()
+ .stream()
+ .collect(Collectors.groupingBy(t -> t.getEntry().getId()));
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java
new file mode 100644
index 000000000..d7b329515
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java
@@ -0,0 +1,146 @@
+package com.commafeed.backend.dao;
+
+import com.commafeed.backend.model.AbstractModel;
+import com.commafeed.backend.model.Feed;
+import com.commafeed.backend.model.FeedCategory;
+import com.commafeed.backend.model.FeedSubscription;
+import com.commafeed.backend.model.Models;
+import com.commafeed.backend.model.QFeedSubscription;
+import com.commafeed.backend.model.User;
+import com.querydsl.jpa.JPQLQuery;
+
+import jakarta.inject.Singleton;
+import jakarta.persistence.EntityManager;
+
+import org.hibernate.engine.spi.SharedSessionContractImplementor;
+import org.hibernate.event.service.spi.EventListenerRegistry;
+import org.hibernate.event.spi.EventType;
+import org.hibernate.event.spi.PostCommitInsertEventListener;
+import org.hibernate.event.spi.PostInsertEvent;
+import org.hibernate.persister.entity.EntityPersister;
+
+import java.util.List;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+@Singleton
+public class FeedSubscriptionDAO extends GenericDAO {
+
+ private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
+
+ private final EntityManager entityManager;
+
+ public FeedSubscriptionDAO(EntityManager entityManager) {
+ super(entityManager, FeedSubscription.class);
+ this.entityManager = entityManager;
+ }
+
+ public void onPostCommitInsert(Consumer consumer) {
+ entityManager
+ .unwrap(SharedSessionContractImplementor.class)
+ .getFactory()
+ .getServiceRegistry()
+ .getService(EventListenerRegistry.class)
+ .getEventListenerGroup(EventType.POST_COMMIT_INSERT)
+ .appendListener(
+ new PostCommitInsertEventListener() {
+ @Override
+ public void onPostInsert(PostInsertEvent event) {
+ if (event.getEntity() instanceof FeedSubscription s) {
+ consumer.accept(s);
+ }
+ }
+
+ @Override
+ public boolean requiresPostCommitHandling(EntityPersister persister) {
+ return true;
+ }
+
+ @Override
+ public void onPostInsertCommitFailed(PostInsertEvent event) {
+ // do nothing
+ }
+ });
+ }
+
+ public FeedSubscription findById(User user, Long id) {
+ List subs =
+ query().selectFrom(SUBSCRIPTION)
+ .where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.id.eq(id))
+ .leftJoin(SUBSCRIPTION.feed)
+ .fetchJoin()
+ .leftJoin(SUBSCRIPTION.category)
+ .fetchJoin()
+ .fetch();
+ FeedSubscription sub = subs.stream().findFirst().orElse(null);
+ return initRelations(sub);
+ }
+
+ public List findByFeed(Feed feed) {
+ return query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.feed.eq(feed)).fetch();
+ }
+
+ public FeedSubscription findByFeed(User user, Feed feed) {
+ List subs =
+ query().selectFrom(SUBSCRIPTION)
+ .where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.feed.eq(feed))
+ .fetch();
+ FeedSubscription sub = subs.stream().findFirst().orElse(null);
+ return initRelations(sub);
+ }
+
+ public List findAll(User user) {
+ List subs =
+ query().selectFrom(SUBSCRIPTION)
+ .where(SUBSCRIPTION.user.eq(user))
+ .leftJoin(SUBSCRIPTION.feed)
+ .fetchJoin()
+ .leftJoin(SUBSCRIPTION.category)
+ .fetchJoin()
+ .fetch();
+ return initRelations(subs);
+ }
+
+ public Long count(User user) {
+ return query().select(SUBSCRIPTION.count())
+ .from(SUBSCRIPTION)
+ .where(SUBSCRIPTION.user.eq(user))
+ .fetchOne();
+ }
+
+ public List findByCategory(User user, FeedCategory category) {
+ JPQLQuery query =
+ query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user));
+ if (category == null) {
+ query.where(SUBSCRIPTION.category.isNull());
+ } else {
+ query.where(SUBSCRIPTION.category.eq(category));
+ }
+ return initRelations(query.fetch());
+ }
+
+ public List findByCategories(User user, List categories) {
+ Set categoryIds =
+ categories.stream().map(AbstractModel::getId).collect(Collectors.toSet());
+ return findAll(user).stream()
+ .filter(
+ s ->
+ s.getCategory() != null
+ && categoryIds.contains(s.getCategory().getId()))
+ .toList();
+ }
+
+ private List initRelations(List list) {
+ list.forEach(this::initRelations);
+ return list;
+ }
+
+ private FeedSubscription initRelations(FeedSubscription sub) {
+ if (sub != null) {
+ Models.initialize(sub.getFeed());
+ Models.initialize(sub.getCategory());
+ }
+ return sub;
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/GenericDAO.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/GenericDAO.java
new file mode 100644
index 000000000..62774147d
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/GenericDAO.java
@@ -0,0 +1,65 @@
+package com.commafeed.backend.dao;
+
+import com.commafeed.backend.model.AbstractModel;
+import com.querydsl.core.types.EntityPath;
+import com.querydsl.jpa.impl.JPADeleteClause;
+import com.querydsl.jpa.impl.JPAQuery;
+import com.querydsl.jpa.impl.JPAQueryFactory;
+import com.querydsl.jpa.impl.JPAUpdateClause;
+
+import jakarta.persistence.EntityManager;
+
+import lombok.RequiredArgsConstructor;
+
+import org.hibernate.jpa.SpecHints;
+
+import java.time.Duration;
+import java.util.Collection;
+
+@RequiredArgsConstructor
+public abstract class GenericDAO {
+
+ private final EntityManager entityManager;
+ private final Class entityClass;
+
+ protected JPAQueryFactory query() {
+ return new JPAQueryFactory(entityManager);
+ }
+
+ protected JPAUpdateClause updateQuery(EntityPath entityPath) {
+ return new JPAUpdateClause(entityManager, entityPath);
+ }
+
+ protected JPADeleteClause deleteQuery(EntityPath entityPath) {
+ return new JPADeleteClause(entityManager, entityPath);
+ }
+
+ public void persist(T model) {
+ entityManager.persist(model);
+ }
+
+ public T merge(T model) {
+ return entityManager.merge(model);
+ }
+
+ public T findById(Long id) {
+ return entityManager.find(entityClass, id);
+ }
+
+ public void delete(T object) {
+ if (object != null) {
+ entityManager.remove(object);
+ }
+ }
+
+ public int delete(Collection objects) {
+ objects.forEach(this::delete);
+ return objects.size();
+ }
+
+ protected void setTimeout(JPAQuery> query, Duration timeout) {
+ if (!timeout.isZero()) {
+ query.setHint(SpecHints.HINT_SPEC_QUERY_TIMEOUT, Math.toIntExact(timeout.toMillis()));
+ }
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/Predicates.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/Predicates.java
new file mode 100644
index 000000000..c606d10ea
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/Predicates.java
@@ -0,0 +1,21 @@
+package com.commafeed.backend.dao;
+
+import com.commafeed.backend.model.QFeedEntry;
+import com.commafeed.backend.model.QFeedEntryStatus;
+import com.querydsl.core.types.dsl.BooleanExpression;
+import com.querydsl.jpa.JPAExpressions;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class Predicates {
+
+ private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
+
+ public static BooleanExpression isNotStarred(QFeedEntry entry) {
+ return JPAExpressions.selectOne()
+ .from(STATUS)
+ .where(STATUS.entry.eq(entry).and(STATUS.starred.isTrue()))
+ .notExists();
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/UnitOfWork.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/UnitOfWork.java
new file mode 100644
index 000000000..e13ac3e08
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/UnitOfWork.java
@@ -0,0 +1,19 @@
+package com.commafeed.backend.dao;
+
+import io.quarkus.narayana.jta.QuarkusTransaction;
+
+import jakarta.inject.Singleton;
+
+import java.util.concurrent.Callable;
+
+@Singleton
+public class UnitOfWork {
+
+ public void run(Runnable runnable) {
+ QuarkusTransaction.joiningExisting().run(runnable);
+ }
+
+ public T call(Callable callable) {
+ return QuarkusTransaction.joiningExisting().call(callable);
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/UserDAO.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/UserDAO.java
new file mode 100644
index 000000000..c1994c448
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/UserDAO.java
@@ -0,0 +1,33 @@
+package com.commafeed.backend.dao;
+
+import com.commafeed.backend.model.QUser;
+import com.commafeed.backend.model.User;
+
+import jakarta.inject.Singleton;
+import jakarta.persistence.EntityManager;
+
+@Singleton
+public class UserDAO extends GenericDAO {
+
+ private static final QUser USER = QUser.user;
+
+ public UserDAO(EntityManager entityManager) {
+ super(entityManager, User.class);
+ }
+
+ public User findByName(String name) {
+ return query().selectFrom(USER).where(USER.name.equalsIgnoreCase(name)).fetchOne();
+ }
+
+ public User findByApiKey(String key) {
+ return query().selectFrom(USER).where(USER.apiKey.equalsIgnoreCase(key)).fetchOne();
+ }
+
+ public User findByEmail(String email) {
+ return query().selectFrom(USER).where(USER.email.equalsIgnoreCase(email)).fetchOne();
+ }
+
+ public long count() {
+ return query().select(USER.count()).from(USER).fetchOne();
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/UserRoleDAO.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/UserRoleDAO.java
new file mode 100644
index 000000000..419ba9e93
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/UserRoleDAO.java
@@ -0,0 +1,39 @@
+package com.commafeed.backend.dao;
+
+import com.commafeed.backend.model.QUserRole;
+import com.commafeed.backend.model.User;
+import com.commafeed.backend.model.UserRole;
+import com.commafeed.backend.model.UserRole.Role;
+
+import jakarta.inject.Singleton;
+import jakarta.persistence.EntityManager;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+@Singleton
+public class UserRoleDAO extends GenericDAO {
+
+ private static final QUserRole ROLE = QUserRole.userRole;
+
+ public UserRoleDAO(EntityManager entityManager) {
+ super(entityManager, UserRole.class);
+ }
+
+ public List findAll() {
+ return query().selectFrom(ROLE).leftJoin(ROLE.user).fetchJoin().distinct().fetch();
+ }
+
+ public List findAll(User user) {
+ return query().selectFrom(ROLE).where(ROLE.user.eq(user)).distinct().fetch();
+ }
+
+ public Set findRoles(User user) {
+ return findAll(user).stream().map(UserRole::getRole).collect(Collectors.toSet());
+ }
+
+ public long countAdmins() {
+ return query().select(ROLE.count()).from(ROLE).where(ROLE.role.eq(Role.ADMIN)).fetchOne();
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/UserSettingsDAO.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/UserSettingsDAO.java
new file mode 100644
index 000000000..86d693fd9
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/dao/UserSettingsDAO.java
@@ -0,0 +1,22 @@
+package com.commafeed.backend.dao;
+
+import com.commafeed.backend.model.QUserSettings;
+import com.commafeed.backend.model.User;
+import com.commafeed.backend.model.UserSettings;
+
+import jakarta.inject.Singleton;
+import jakarta.persistence.EntityManager;
+
+@Singleton
+public class UserSettingsDAO extends GenericDAO {
+
+ private static final QUserSettings SETTINGS = QUserSettings.userSettings;
+
+ public UserSettingsDAO(EntityManager entityManager) {
+ super(entityManager, UserSettings.class);
+ }
+
+ public UserSettings findByUser(User user) {
+ return query().selectFrom(SETTINGS).where(SETTINGS.user.eq(user)).fetchFirst();
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
new file mode 100644
index 000000000..7e4b9478c
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
@@ -0,0 +1,71 @@
+package com.commafeed.backend.favicon;
+
+import com.commafeed.backend.HttpGetter;
+import com.commafeed.backend.HttpGetter.HttpResult;
+import com.commafeed.backend.model.Feed;
+
+import jakarta.annotation.Priority;
+import jakarta.inject.Singleton;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.hc.core5.http.NameValuePair;
+import org.apache.hc.core5.net.URIBuilder;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+
+@Slf4j
+@RequiredArgsConstructor
+@Singleton
+@Priority(3)
+public class FacebookFaviconFetcher implements FaviconFetcher {
+
+ private final HttpGetter getter;
+
+ @Override
+ public Favicon fetch(Feed feed) {
+ String url = feed.getUrl();
+ if (!url.toLowerCase().contains("www.facebook.com")) {
+ return null;
+ }
+
+ String userName = extractUserName(url);
+ if (userName == null) {
+ return null;
+ }
+
+ String iconUrl =
+ String.format(
+ "https://graph.facebook.com/%s/picture?type=square&height=16", userName);
+
+ try {
+ log.debug("Getting Facebook user's icon, {}", url);
+
+ HttpResult iconResult = getter.get(iconUrl);
+ return new Favicon(iconResult.content(), iconResult.contentType());
+ } catch (Exception e) {
+ log.debug("Failed to retrieve Facebook icon", e);
+ return null;
+ }
+ }
+
+ private String extractUserName(String url) {
+ URI uri;
+ try {
+ uri = new URI(url);
+ } catch (URISyntaxException e) {
+ log.debug("could not parse url", e);
+ return null;
+ }
+
+ List params = new URIBuilder(uri).getQueryParams();
+ return params.stream()
+ .filter(p -> "id".equals(p.getName()))
+ .map(NameValuePair::getValue)
+ .findFirst()
+ .orElse(null);
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/Favicon.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/Favicon.java
new file mode 100644
index 000000000..a3cea0b82
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/Favicon.java
@@ -0,0 +1,25 @@
+package com.commafeed.backend.favicon;
+
+import jakarta.ws.rs.core.MediaType;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public record Favicon(byte[] icon, MediaType mediaType) {
+
+ private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.valueOf("image/x-icon");
+
+ public Favicon(byte[] icon, String contentType) {
+ this(icon, parseMediaType(contentType));
+ }
+
+ private static MediaType parseMediaType(String contentType) {
+ try {
+ return MediaType.valueOf(contentType);
+ } catch (Exception e) {
+ log.debug(
+ "invalid content type '{}' received, returning default value", contentType, e);
+ return DEFAULT_MEDIA_TYPE;
+ }
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/FaviconFetcher.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/FaviconFetcher.java
new file mode 100644
index 000000000..b2bad744a
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/FaviconFetcher.java
@@ -0,0 +1,8 @@
+package com.commafeed.backend.favicon;
+
+import com.commafeed.backend.model.Feed;
+
+public interface FaviconFetcher {
+
+ Favicon fetch(Feed feed);
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/FeedFaviconFetcher.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/FeedFaviconFetcher.java
new file mode 100644
index 000000000..f05cb39de
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/FeedFaviconFetcher.java
@@ -0,0 +1,37 @@
+package com.commafeed.backend.favicon;
+
+import com.commafeed.backend.HttpGetter;
+import com.commafeed.backend.HttpGetter.HttpResult;
+import com.commafeed.backend.model.Feed;
+
+import jakarta.annotation.Priority;
+import jakarta.inject.Singleton;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+/** Fetch favicon from the url declared in the feed. */
+@Slf4j
+@RequiredArgsConstructor
+@Singleton
+@Priority(2)
+public class FeedFaviconFetcher implements FaviconFetcher {
+
+ private final HttpGetter getter;
+
+ @Override
+ public Favicon fetch(Feed feed) {
+ String url = feed.getIconUrl();
+ if (url == null) {
+ return null;
+ }
+
+ try {
+ HttpResult result = getter.get(url);
+ return new Favicon(result.content(), result.contentType());
+ } catch (Exception e) {
+ log.debug("Failed to retrieve icon declared in the feed {}", url, e);
+ return null;
+ }
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/HtmlFaviconFetcher.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/HtmlFaviconFetcher.java
new file mode 100644
index 000000000..0646a1ee1
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/HtmlFaviconFetcher.java
@@ -0,0 +1,63 @@
+package com.commafeed.backend.favicon;
+
+import com.commafeed.backend.HttpGetter;
+import com.commafeed.backend.HttpGetter.HttpResult;
+import com.commafeed.backend.model.Feed;
+
+import jakarta.annotation.Priority;
+import jakarta.inject.Singleton;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang3.StringUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.select.Elements;
+
+/** Extracts favicon url from html page. */
+@Slf4j
+@RequiredArgsConstructor
+@Singleton
+@Priority(1)
+public class HtmlFaviconFetcher implements FaviconFetcher {
+
+ private final HttpGetter getter;
+
+ @Override
+ public Favicon fetch(Feed feed) {
+ String url = feed.getLink();
+ if (url == null) {
+ return null;
+ }
+
+ Document doc;
+ try {
+ HttpResult result = getter.get(url);
+ doc = Jsoup.parse(new String(result.content()), url);
+ } catch (Exception e) {
+ log.debug("Failed to retrieve page to find icon", e);
+ return null;
+ }
+
+ Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
+ if (icons.isEmpty()) {
+ log.debug("No icon found in page {}", url);
+ return null;
+ }
+
+ String href = icons.getFirst().attr("abs:href");
+ if (StringUtils.isBlank(href)) {
+ log.debug("No icon found in page");
+ return null;
+ }
+
+ try {
+ HttpResult result = getter.get(href);
+ return new Favicon(result.content(), result.contentType());
+ } catch (Exception e) {
+ log.debug("Failed to retrieve icon found in page {}", href, e);
+ return null;
+ }
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/RootFaviconFetcher.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/RootFaviconFetcher.java
new file mode 100644
index 000000000..20190a400
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/RootFaviconFetcher.java
@@ -0,0 +1,48 @@
+package com.commafeed.backend.favicon;
+
+import com.commafeed.backend.HttpGetter;
+import com.commafeed.backend.HttpGetter.HttpResult;
+import com.commafeed.backend.model.Feed;
+
+import jakarta.annotation.Priority;
+import jakarta.inject.Singleton;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.net.URI;
+
+/** Fetches favicon from root of the domain (e.g. https://example.com/favicon.ico) */
+@Slf4j
+@RequiredArgsConstructor
+@Singleton
+@Priority(0)
+public class RootFaviconFetcher implements FaviconFetcher {
+
+ private final HttpGetter getter;
+
+ @Override
+ public Favicon fetch(Feed feed) {
+ String url = feed.getLink();
+ if (url == null) {
+ url = feed.getUrl();
+ }
+
+ try {
+ URI uri = URI.create(url.trim());
+ String faviconUrl =
+ "%s://%s%s/favicon.ico"
+ .formatted(
+ uri.getScheme(),
+ uri.getHost(),
+ uri.getPort() > 0 ? ":" + uri.getPort() : "");
+
+ log.debug("getting root icon at {}", faviconUrl);
+ HttpResult result = getter.get(faviconUrl);
+ return new Favicon(result.content(), result.contentType());
+ } catch (Exception e) {
+ log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
+ return null;
+ }
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
new file mode 100644
index 000000000..5ca051d53
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
@@ -0,0 +1,153 @@
+package com.commafeed.backend.favicon;
+
+import com.commafeed.CommaFeedConfiguration;
+import com.commafeed.backend.HttpGetter;
+import com.commafeed.backend.HttpGetter.HttpResult;
+import com.commafeed.backend.HttpGetter.NotModifiedException;
+import com.commafeed.backend.HttpGetter.SchemeNotAllowedException;
+import com.commafeed.backend.HttpGetter.TooManyRequestsException;
+import com.commafeed.backend.model.Feed;
+import com.fasterxml.jackson.core.JsonPointer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import jakarta.annotation.Priority;
+import jakarta.inject.Singleton;
+import jakarta.ws.rs.core.UriBuilder;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.hc.core5.http.NameValuePair;
+import org.apache.hc.core5.net.URIBuilder;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Optional;
+
+@Slf4j
+@RequiredArgsConstructor
+@Singleton
+@Priority(3)
+public class YoutubeFaviconFetcher implements FaviconFetcher {
+
+ private static final String PART_SNIPPET = "snippet";
+
+ private static final JsonPointer CHANNEL_THUMBNAIL_URL =
+ JsonPointer.compile("/items/0/snippet/thumbnails/default/url");
+ private static final JsonPointer PLAYLIST_CHANNEL_ID =
+ JsonPointer.compile("/items/0/snippet/channelId");
+
+ private final HttpGetter getter;
+ private final CommaFeedConfiguration config;
+ private final ObjectMapper objectMapper;
+
+ @Override
+ public Favicon fetch(Feed feed) {
+ String url = feed.getUrl();
+ if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
+ return null;
+ }
+
+ Optional googleAuthKey = config.googleAuthKey();
+ if (googleAuthKey.isEmpty()) {
+ log.debug("no google auth key configured");
+ return null;
+ }
+
+ try {
+ List params = new URIBuilder(url).getQueryParams();
+ Optional userId =
+ params.stream()
+ .filter(nvp -> nvp.getName().equalsIgnoreCase("user"))
+ .findFirst();
+ Optional channelId =
+ params.stream()
+ .filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id"))
+ .findFirst();
+ Optional playlistId =
+ params.stream()
+ .filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id"))
+ .findFirst();
+
+ byte[] response = null;
+ if (userId.isPresent()) {
+ log.debug("contacting youtube api for user {}", userId.get().getValue());
+ response = fetchForUser(googleAuthKey.get(), userId.get().getValue());
+ } else if (channelId.isPresent()) {
+ log.debug("contacting youtube api for channel {}", channelId.get().getValue());
+ response = fetchForChannel(googleAuthKey.get(), channelId.get().getValue());
+ } else if (playlistId.isPresent()) {
+ log.debug("contacting youtube api for playlist {}", playlistId.get().getValue());
+ response = fetchForPlaylist(googleAuthKey.get(), playlistId.get().getValue());
+ }
+ if (ArrayUtils.isEmpty(response)) {
+ log.debug("youtube api returned empty response");
+ return null;
+ }
+
+ JsonNode thumbnailUrl = objectMapper.readTree(response).at(CHANNEL_THUMBNAIL_URL);
+ if (thumbnailUrl.isMissingNode()) {
+ log.debug("youtube api returned invalid response");
+ return null;
+ }
+
+ HttpResult iconResult = getter.get(thumbnailUrl.asText());
+ return new Favicon(iconResult.content(), iconResult.contentType());
+ } catch (Exception e) {
+ log.debug("Failed to retrieve YouTube icon", e);
+ return null;
+ }
+ }
+
+ private byte[] fetchForUser(String googleAuthKey, String userId)
+ throws IOException,
+ NotModifiedException,
+ TooManyRequestsException,
+ SchemeNotAllowedException {
+ URI uri =
+ UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
+ .queryParam("part", PART_SNIPPET)
+ .queryParam("key", googleAuthKey)
+ .queryParam("forUsername", userId)
+ .build();
+ return getter.get(uri.toString()).content();
+ }
+
+ private byte[] fetchForChannel(String googleAuthKey, String channelId)
+ throws IOException,
+ NotModifiedException,
+ TooManyRequestsException,
+ SchemeNotAllowedException {
+ URI uri =
+ UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
+ .queryParam("part", PART_SNIPPET)
+ .queryParam("key", googleAuthKey)
+ .queryParam("id", channelId)
+ .build();
+ return getter.get(uri.toString()).content();
+ }
+
+ private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
+ throws IOException,
+ NotModifiedException,
+ TooManyRequestsException,
+ SchemeNotAllowedException {
+ URI uri =
+ UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
+ .queryParam("part", PART_SNIPPET)
+ .queryParam("key", googleAuthKey)
+ .queryParam("id", playlistId)
+ .build();
+ byte[] playlistBytes = getter.get(uri.toString()).content();
+
+ JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID);
+ if (channelId.isMissingNode()) {
+ return new byte[0];
+ }
+
+ return fetchForChannel(googleAuthKey, channelId.asText());
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedEntryKeyword.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedEntryKeyword.java
new file mode 100644
index 000000000..46c59f158
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedEntryKeyword.java
@@ -0,0 +1,30 @@
+package com.commafeed.backend.feed;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** A keyword used in a search query */
+public record FeedEntryKeyword(String keyword, Mode mode) {
+
+ public enum Mode {
+ INCLUDE,
+ EXCLUDE
+ }
+
+ public static List fromQueryString(String keywords) {
+ List list = new ArrayList<>();
+ if (keywords != null) {
+ for (String keyword : StringUtils.split(keywords)) {
+ boolean not = false;
+ if (keyword.startsWith("-") || keyword.startsWith("!")) {
+ not = true;
+ keyword = keyword.substring(1);
+ }
+ list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
+ }
+ }
+ return list;
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
new file mode 100644
index 000000000..1b15597fb
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
@@ -0,0 +1,154 @@
+package com.commafeed.backend.feed;
+
+import com.commafeed.backend.Digests;
+import com.commafeed.backend.HttpGetter;
+import com.commafeed.backend.HttpGetter.HttpRequest;
+import com.commafeed.backend.HttpGetter.HttpResult;
+import com.commafeed.backend.HttpGetter.NotModifiedException;
+import com.commafeed.backend.HttpGetter.SchemeNotAllowedException;
+import com.commafeed.backend.HttpGetter.TooManyRequestsException;
+import com.commafeed.backend.feed.parser.FeedParser;
+import com.commafeed.backend.feed.parser.FeedParser.FeedParsingException;
+import com.commafeed.backend.feed.parser.FeedParserResult;
+import com.commafeed.backend.urlprovider.FeedURLProvider;
+
+import io.quarkus.arc.All;
+
+import jakarta.inject.Singleton;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.Strings;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+
+/** Fetches a feed then parses it */
+@Slf4j
+@Singleton
+public class FeedFetcher {
+
+ private final FeedParser parser;
+ private final HttpGetter getter;
+ private final List urlProviders;
+
+ public FeedFetcher(
+ FeedParser parser, HttpGetter getter, @All List urlProviders) {
+ this.parser = parser;
+ this.getter = getter;
+ this.urlProviders = urlProviders;
+ }
+
+ public FeedFetcherResult fetch(
+ String feedUrl,
+ boolean extractFeedUrlFromHtml,
+ String lastModified,
+ String eTag,
+ Instant lastPublishedDate,
+ String lastContentHash)
+ throws FeedParsingException,
+ IOException,
+ NotModifiedException,
+ TooManyRequestsException,
+ SchemeNotAllowedException,
+ NoFeedFoundException {
+ log.debug("Fetching feed {}", feedUrl);
+
+ HttpResult result =
+ getter.get(
+ HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
+ byte[] content = result.content();
+
+ FeedParserResult parserResult;
+ try {
+ parserResult = parser.parse(result.urlAfterRedirect(), content);
+ } catch (FeedParsingException e) {
+ if (extractFeedUrlFromHtml) {
+ String extractedUrl =
+ extractFeedUrl(
+ urlProviders,
+ feedUrl,
+ new String(result.content(), StandardCharsets.UTF_8));
+ if (StringUtils.isNotBlank(extractedUrl)) {
+ feedUrl = extractedUrl;
+
+ result =
+ getter.get(
+ HttpRequest.builder(extractedUrl)
+ .lastModified(lastModified)
+ .eTag(eTag)
+ .build());
+ content = result.content();
+ parserResult = parser.parse(result.urlAfterRedirect(), content);
+ } else {
+ throw new NoFeedFoundException(e);
+ }
+ } else {
+ throw e;
+ }
+ }
+
+ if (content == null) {
+ throw new IOException("Feed content is empty.");
+ }
+
+ boolean lastModifiedHeaderValueChanged =
+ !Strings.CS.equals(lastModified, result.lastModifiedSince());
+ boolean etagHeaderValueChanged = !Strings.CS.equals(eTag, result.eTag());
+
+ String hash = Digests.sha1Hex(content);
+ if (lastContentHash != null && lastContentHash.equals(hash)) {
+ log.debug("content hash not modified: {}", feedUrl);
+ throw new NotModifiedException(
+ "content hash not modified",
+ lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
+ etagHeaderValueChanged ? result.eTag() : null);
+ }
+
+ if (lastPublishedDate != null
+ && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
+ log.debug("publishedDate not modified: {}", feedUrl);
+ throw new NotModifiedException(
+ "publishedDate not modified",
+ lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
+ etagHeaderValueChanged ? result.eTag() : null);
+ }
+
+ return new FeedFetcherResult(
+ parserResult,
+ result.urlAfterRedirect(),
+ result.lastModifiedSince(),
+ result.eTag(),
+ hash,
+ result.validFor());
+ }
+
+ private static String extractFeedUrl(
+ List urlProviders, String url, String urlContent) {
+ return urlProviders.stream()
+ .flatMap(provider -> provider.get(url, urlContent).stream())
+ .filter(StringUtils::isNotBlank)
+ .findFirst()
+ .orElse(null);
+ }
+
+ public record FeedFetcherResult(
+ FeedParserResult feed,
+ String urlAfterRedirect,
+ String lastModifiedHeader,
+ String lastETagHeader,
+ String contentHash,
+ Duration validFor) {}
+
+ public static class NoFeedFoundException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ public NoFeedFoundException(Throwable cause) {
+ super("This URL does not point to an RSS feed or a website with an RSS feed.", cause);
+ }
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshEngine.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshEngine.java
new file mode 100644
index 000000000..4f85b3cbe
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshEngine.java
@@ -0,0 +1,310 @@
+package com.commafeed.backend.feed;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.commafeed.CommaFeedConfiguration;
+import com.commafeed.backend.dao.FeedDAO;
+import com.commafeed.backend.dao.UnitOfWork;
+import com.commafeed.backend.model.AbstractModel;
+import com.commafeed.backend.model.Feed;
+import com.commafeed.backend.model.FeedEntry;
+import com.commafeed.backend.model.FeedSubscription;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import jakarta.inject.Singleton;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+@Slf4j
+@Singleton
+public class FeedRefreshEngine {
+
+ private final UnitOfWork unitOfWork;
+ private final FeedDAO feedDAO;
+ private final FeedRefreshWorker worker;
+ private final FeedRefreshUpdater updater;
+ private final FeedUpdateNotifier notifier;
+ private final CommaFeedConfiguration config;
+ private final Meter refill;
+
+ private final BlockingDeque queue;
+
+ private ExecutorService feedProcessingLoopExecutor;
+ private ExecutorService refillLoopExecutor;
+ private ThreadPoolExecutor refillExecutor;
+ private ThreadPoolExecutor workerExecutor;
+ private ThreadPoolExecutor databaseUpdaterExecutor;
+ private ThreadPoolExecutor notifierExecutor;
+
+ public FeedRefreshEngine(
+ UnitOfWork unitOfWork,
+ FeedDAO feedDAO,
+ FeedRefreshWorker worker,
+ FeedRefreshUpdater updater,
+ FeedUpdateNotifier notifier,
+ CommaFeedConfiguration config,
+ MetricRegistry metrics) {
+ this.unitOfWork = unitOfWork;
+ this.feedDAO = feedDAO;
+ this.worker = worker;
+ this.updater = updater;
+ this.notifier = notifier;
+ this.config = config;
+ this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
+
+ this.queue = new LinkedBlockingDeque<>();
+
+ metrics.register(
+ MetricRegistry.name(getClass(), "queue", "size"), (Gauge) queue::size);
+ metrics.register(
+ MetricRegistry.name(getClass(), "worker", "active"),
+ (Gauge) () -> workerExecutor.getActiveCount());
+ metrics.register(
+ MetricRegistry.name(getClass(), "updater", "active"),
+ (Gauge) () -> databaseUpdaterExecutor.getActiveCount());
+ metrics.register(
+ MetricRegistry.name(getClass(), "notifier", "active"),
+ (Gauge) () -> notifierExecutor.getActiveCount());
+ metrics.register(
+ MetricRegistry.name(getClass(), "notifier", "queue"),
+ (Gauge) () -> notifierExecutor.getQueue().size());
+ }
+
+ private void createExecutors() {
+ this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor();
+ this.refillLoopExecutor = Executors.newSingleThreadExecutor();
+ this.refillExecutor = newDiscardingSingleThreadExecutorService();
+ this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
+ this.databaseUpdaterExecutor =
+ newBlockingExecutorService(config.feedRefresh().databaseThreads());
+ this.notifierExecutor =
+ newDiscardingExecutorService(
+ config.pushNotifications().threads(),
+ config.pushNotifications().queueCapacity());
+ }
+
+ public void start() {
+ createExecutors();
+ startFeedProcessingLoop();
+ startRefillLoop();
+ }
+
+ private void startFeedProcessingLoop() {
+ // take a feed from the queue, process it, rince, repeat
+ feedProcessingLoopExecutor.submit(
+ () -> {
+ while (!feedProcessingLoopExecutor.isShutdown()) {
+ try {
+ // take() is blocking until a feed is available from the queue
+ Feed feed = queue.take();
+
+ // send the feed to be processed
+ log.debug(
+ "got feed {} from the queue, send it for processing",
+ feed.getId());
+ processFeedAsync(feed);
+
+ // we removed a feed from the queue, try to refill it as it may now be
+ // empty
+ if (queue.isEmpty()) {
+ log.debug("took the last feed from the queue, try to refill");
+ refillQueueAsync();
+ }
+ } catch (InterruptedException e) {
+ log.debug("interrupted while waiting for a feed in the queue");
+ Thread.currentThread().interrupt();
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ }
+ }
+ });
+ }
+
+ private void startRefillLoop() {
+ // refill the queue at regular intervals if it's empty
+ refillLoopExecutor.submit(
+ () -> {
+ while (!refillLoopExecutor.isShutdown()) {
+ try {
+ if (queue.isEmpty()) {
+ log.debug("refilling queue");
+ refillQueueAsync();
+ }
+
+ log.debug("sleeping for 15s");
+ TimeUnit.SECONDS.sleep(15);
+ } catch (InterruptedException e) {
+ log.debug("interrupted while sleeping");
+ Thread.currentThread().interrupt();
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ }
+ }
+ });
+ }
+
+ public void refreshImmediately(Feed feed) {
+ log.debug("add feed {} at the start of the queue", feed.getId());
+ // remove the feed from the queue if it was already queued to avoid refreshing it twice
+ queue.removeIf(f -> f.getId().equals(feed.getId()));
+ queue.addFirst(feed);
+ }
+
+ private void refillQueueAsync() {
+ CompletableFuture.runAsync(
+ () -> {
+ if (!queue.isEmpty()) {
+ return;
+ }
+
+ refill.mark();
+
+ List nextUpdatableFeeds = getNextUpdatableFeeds(getBatchSize());
+ log.debug(
+ "found {} feeds that are up for refresh",
+ nextUpdatableFeeds.size());
+ for (Feed feed : nextUpdatableFeeds) {
+ // add the feed only if it was not already queued
+ if (queue.stream().noneMatch(f -> f.getId().equals(feed.getId()))) {
+ queue.addLast(feed);
+ }
+ }
+ },
+ refillExecutor)
+ .whenComplete(
+ (data, ex) -> {
+ if (ex != null) {
+ log.error("error while refilling the queue", ex);
+ }
+ });
+ }
+
+ private void processFeedAsync(Feed feed) {
+ CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
+ .thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
+ .thenCompose(
+ r -> {
+ List> futures =
+ r.insertedUnreadEntriesBySubscription().entrySet().stream()
+ .map(
+ e -> {
+ FeedSubscription sub = e.getKey();
+ List entries = e.getValue();
+
+ notifier.notifyOverWebsocket(sub, entries);
+ return CompletableFuture.runAsync(
+ () ->
+ notifier
+ .sendPushNotifications(
+ sub,
+ entries),
+ notifierExecutor);
+ })
+ .toList();
+ return CompletableFuture.allOf(
+ futures.toArray(CompletableFuture[]::new));
+ })
+ .exceptionally(
+ ex -> {
+ log.error("error while processing feed {}", feed.getUrl(), ex);
+ return null;
+ });
+ }
+
+ private List getNextUpdatableFeeds(int max) {
+ return unitOfWork.call(
+ () -> {
+ Instant lastLoginThreshold =
+ config.feedRefresh().userInactivityPeriod().isZero()
+ ? null
+ : Instant.now()
+ .minus(config.feedRefresh().userInactivityPeriod());
+ List feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
+ if (!feeds.isEmpty()) {
+ // update disabledUntil to prevent feeds from being returned again by
+ // feedDAO.findNextUpdatable()
+ Instant nextUpdateDate =
+ Instant.now().plus(config.feedRefresh().interval());
+ feedDAO.setDisabledUntil(
+ feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
+ }
+ return feeds;
+ });
+ }
+
+ private int getBatchSize() {
+ return Math.min(100, 3 * config.feedRefresh().httpThreads());
+ }
+
+ public void stop() {
+ MoreExecutors.shutdownAndAwaitTermination(
+ this.feedProcessingLoopExecutor, config.shutdownTimeout());
+ MoreExecutors.shutdownAndAwaitTermination(
+ this.refillLoopExecutor, config.shutdownTimeout());
+ MoreExecutors.shutdownAndAwaitTermination(this.refillExecutor, config.shutdownTimeout());
+ MoreExecutors.shutdownAndAwaitTermination(this.workerExecutor, config.shutdownTimeout());
+ MoreExecutors.shutdownAndAwaitTermination(
+ this.databaseUpdaterExecutor, config.shutdownTimeout());
+ MoreExecutors.shutdownAndAwaitTermination(this.notifierExecutor, config.shutdownTimeout());
+
+ queue.clear();
+ }
+
+ /**
+ * returns an ExecutorService with a single thread that discards tasks if a task is already
+ * running
+ */
+ private ThreadPoolExecutor newDiscardingSingleThreadExecutorService() {
+ ThreadPoolExecutor pool =
+ new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
+ pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
+ return pool;
+ }
+
+ /** returns an ExecutorService that discards tasks if the queue is full */
+ private ThreadPoolExecutor newDiscardingExecutorService(int threads, int queueCapacity) {
+ ThreadPoolExecutor pool =
+ new ThreadPoolExecutor(
+ threads,
+ threads,
+ 0L,
+ TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<>(queueCapacity));
+ pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
+ return pool;
+ }
+
+ /** returns an ExecutorService that blocks submissions until a thread is available */
+ private ThreadPoolExecutor newBlockingExecutorService(int threads) {
+ ThreadPoolExecutor pool =
+ new ThreadPoolExecutor(
+ threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
+ pool.setRejectedExecutionHandler(
+ (r, e) -> {
+ if (e.isShutdown()) {
+ return;
+ }
+
+ try {
+ e.getQueue().put(r);
+ } catch (InterruptedException ex) {
+ log.debug("interrupted while waiting for a slot in the queue.", ex);
+ Thread.currentThread().interrupt();
+ }
+ });
+ return pool;
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshIntervalCalculator.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshIntervalCalculator.java
new file mode 100644
index 000000000..6be068676
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshIntervalCalculator.java
@@ -0,0 +1,98 @@
+package com.commafeed.backend.feed;
+
+import com.commafeed.CommaFeedConfiguration;
+import com.commafeed.CommaFeedConfiguration.FeedRefreshErrorHandling;
+import com.google.common.primitives.Longs;
+
+import jakarta.inject.Singleton;
+
+import org.apache.commons.lang3.ObjectUtils;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.time.InstantSource;
+import java.time.temporal.ChronoUnit;
+
+@Singleton
+public class FeedRefreshIntervalCalculator {
+
+ private final Duration interval;
+ private final Duration maxInterval;
+ private final boolean empirical;
+ private final FeedRefreshErrorHandling errorHandling;
+ private final InstantSource instantSource;
+
+ public FeedRefreshIntervalCalculator(
+ CommaFeedConfiguration config, InstantSource instantSource) {
+ this.interval = config.feedRefresh().interval();
+ this.maxInterval = config.feedRefresh().maxInterval();
+ this.empirical = config.feedRefresh().intervalEmpirical();
+ this.errorHandling = config.feedRefresh().errors();
+ this.instantSource = instantSource;
+ }
+
+ public Instant onFetchSuccess(
+ Instant publishedDate, Long averageEntryInterval, Duration validFor) {
+ Instant instant =
+ empirical
+ ? computeEmpiricalRefreshInterval(publishedDate, averageEntryInterval)
+ : instantSource.instant().plus(interval);
+ return constrainToBounds(ObjectUtils.max(instant, instantSource.instant().plus(validFor)));
+ }
+
+ public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) {
+ return onFetchSuccess(publishedDate, averageEntryInterval, Duration.ZERO);
+ }
+
+ public Instant onTooManyRequests(Instant retryAfter, int errorCount) {
+ return constrainToBounds(ObjectUtils.max(retryAfter, onFetchError(errorCount)));
+ }
+
+ public Instant onFetchError(int errorCount) {
+ if (errorCount < errorHandling.retriesBeforeBackoff()) {
+ return constrainToBounds(instantSource.instant().plus(interval));
+ }
+
+ Duration retryInterval =
+ errorHandling
+ .backoffInterval()
+ .multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L);
+ return constrainToBounds(instantSource.instant().plus(retryInterval));
+ }
+
+ private Instant computeEmpiricalRefreshInterval(
+ Instant publishedDate, Long averageEntryInterval) {
+ Instant now = instantSource.instant();
+
+ if (publishedDate == null) {
+ return now.plus(maxInterval);
+ }
+
+ long daysSinceLastPublication = ChronoUnit.DAYS.between(publishedDate, now);
+ if (daysSinceLastPublication >= 30) {
+ return now.plus(maxInterval);
+ } else if (daysSinceLastPublication >= 14) {
+ return now.plus(maxInterval.dividedBy(2));
+ } else if (daysSinceLastPublication >= 7) {
+ return now.plus(maxInterval.dividedBy(4));
+ } else if (averageEntryInterval != null) {
+ // use average time between entries to decide when to refresh next, divided by factor
+ int factor = 2;
+ long millis =
+ Longs.constrainToRange(
+ averageEntryInterval / factor,
+ interval.toMillis(),
+ maxInterval.dividedBy(4).toMillis());
+ return now.plusMillis(millis);
+ } else {
+ // unknown case
+ return now.plus(maxInterval);
+ }
+ }
+
+ private Instant constrainToBounds(Instant instant) {
+ return ObjectUtils.max(
+ ObjectUtils.min(instant, instantSource.instant().plus(maxInterval)),
+ instantSource.instant().plus(interval));
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
new file mode 100644
index 000000000..4dde767f2
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
@@ -0,0 +1,188 @@
+package com.commafeed.backend.feed;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.commafeed.backend.Digests;
+import com.commafeed.backend.dao.FeedSubscriptionDAO;
+import com.commafeed.backend.dao.UnitOfWork;
+import com.commafeed.backend.feed.parser.FeedParserResult.Content;
+import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
+import com.commafeed.backend.model.Feed;
+import com.commafeed.backend.model.FeedEntry;
+import com.commafeed.backend.model.FeedSubscription;
+import com.commafeed.backend.model.Models;
+import com.commafeed.backend.service.FeedEntryService;
+import com.commafeed.backend.service.FeedService;
+import com.google.common.util.concurrent.Striped;
+
+import jakarta.inject.Singleton;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+
+/** Updates the feed in the database and inserts new entries */
+@Slf4j
+@Singleton
+public class FeedRefreshUpdater {
+
+ private final UnitOfWork unitOfWork;
+ private final FeedService feedService;
+ private final FeedEntryService feedEntryService;
+ private final FeedSubscriptionDAO feedSubscriptionDAO;
+
+ private final Striped locks;
+
+ private final Meter feedUpdated;
+ private final Meter entryInserted;
+
+ public FeedRefreshUpdater(
+ UnitOfWork unitOfWork,
+ FeedService feedService,
+ FeedEntryService feedEntryService,
+ MetricRegistry metrics,
+ FeedSubscriptionDAO feedSubscriptionDAO) {
+ this.unitOfWork = unitOfWork;
+ this.feedService = feedService;
+ this.feedEntryService = feedEntryService;
+ this.feedSubscriptionDAO = feedSubscriptionDAO;
+
+ locks = Striped.lazyWeakLock(100000);
+
+ feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
+ entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
+ }
+
+ private AddEntryResult addEntry(
+ final Feed feed, final Entry entry, final List subscriptions) {
+ boolean processed = false;
+ FeedEntry insertedEntry = null;
+ Set subscriptionsForWhichEntryIsUnread = new HashSet<>();
+
+ // lock on feed, make sure we are not updating the same feed twice at
+ // the same time
+ String key1 = StringUtils.trimToEmpty(String.valueOf(feed.getId()));
+
+ // lock on content, make sure we are not updating the same entry
+ // twice at the same time
+ Content content = entry.content();
+ String key2 = Digests.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title()));
+
+ Iterator iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
+ Lock lock1 = iterator.next();
+ Lock lock2 = iterator.next();
+ boolean locked1 = false;
+ boolean locked2 = false;
+ try {
+ // try to lock, give up after 1 minute
+ locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
+ locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
+ if (locked1 && locked2) {
+ processed = true;
+ insertedEntry =
+ unitOfWork.call(
+ () -> {
+ if (feedEntryService.find(feed, entry) != null) {
+ // entry already exists, nothing to do
+ return null;
+ }
+
+ FeedEntry feedEntry = feedEntryService.create(feed, entry);
+ entryInserted.mark();
+ for (FeedSubscription sub : subscriptions) {
+ boolean unread =
+ feedEntryService.applyFilter(sub, feedEntry);
+ if (unread) {
+ subscriptionsForWhichEntryIsUnread.add(sub);
+ }
+ }
+ return feedEntry;
+ });
+ } else {
+ log.error("lock timeout for {} - {}", feed.getUrl(), key1);
+ }
+ } catch (InterruptedException e) {
+ log.error(
+ "interrupted while waiting for lock for {} : {}",
+ feed.getUrl(),
+ e.getMessage(),
+ e);
+ Thread.currentThread().interrupt();
+ } finally {
+ if (locked1) {
+ lock1.unlock();
+ }
+ if (locked2) {
+ lock2.unlock();
+ }
+ }
+ return new AddEntryResult(processed, insertedEntry, subscriptionsForWhichEntryIsUnread);
+ }
+
+ public FeedRefreshUpdaterResult update(Feed feed, List entries) {
+ boolean processed = true;
+ long inserted = 0;
+ Map> insertedUnreadEntriesBySubscription =
+ new HashMap<>();
+
+ if (!entries.isEmpty()) {
+ List subscriptions = null;
+ List newEntries =
+ unitOfWork.call(() -> feedEntryService.removeExistingEntries(feed, entries));
+ for (Entry entry : newEntries) {
+ if (subscriptions == null) {
+ subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
+ }
+ AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
+ processed &= addEntryResult.processed;
+ inserted += addEntryResult.insertedEntry != null ? 1 : 0;
+ addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(
+ sub -> {
+ if (addEntryResult.insertedEntry != null) {
+ insertedUnreadEntriesBySubscription
+ .computeIfAbsent(sub, k -> new ArrayList<>())
+ .add(addEntryResult.insertedEntry);
+ }
+ });
+ }
+
+ if (inserted == 0) {
+ feed.setMessage("No new entries found");
+ } else if (inserted > 0) {
+ feed.setMessage("Found %s new entries".formatted(inserted));
+ }
+ }
+
+ if (!processed) {
+ // requeue asap
+ feed.setDisabledUntil(Models.MINIMUM_INSTANT);
+ }
+
+ if (inserted > 0) {
+ feedUpdated.mark();
+ }
+
+ unitOfWork.run(() -> feedService.update(feed));
+
+ return new FeedRefreshUpdaterResult(insertedUnreadEntriesBySubscription);
+ }
+
+ private record AddEntryResult(
+ boolean processed,
+ FeedEntry insertedEntry,
+ Set subscriptionsForWhichEntryIsUnread) {}
+
+ public record FeedRefreshUpdaterResult(
+ Map> insertedUnreadEntriesBySubscription) {}
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
new file mode 100644
index 000000000..c67b77b59
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
@@ -0,0 +1,143 @@
+package com.commafeed.backend.feed;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.commafeed.CommaFeedConfiguration;
+import com.commafeed.backend.HttpGetter.NotModifiedException;
+import com.commafeed.backend.HttpGetter.TooManyRequestsException;
+import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
+import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
+import com.commafeed.backend.model.Feed;
+
+import jakarta.inject.Singleton;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang3.Strings;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link
+ * FeedRefreshUpdater} does that)
+ */
+@Slf4j
+@Singleton
+public class FeedRefreshWorker {
+
+ private final FeedRefreshIntervalCalculator refreshIntervalCalculator;
+ private final FeedFetcher fetcher;
+ private final CommaFeedConfiguration config;
+ private final Meter feedFetched;
+
+ public FeedRefreshWorker(
+ FeedRefreshIntervalCalculator refreshIntervalCalculator,
+ FeedFetcher fetcher,
+ CommaFeedConfiguration config,
+ MetricRegistry metrics) {
+ this.refreshIntervalCalculator = refreshIntervalCalculator;
+ this.fetcher = fetcher;
+ this.config = config;
+ this.feedFetched = metrics.meter(MetricRegistry.name(getClass(), "feedFetched"));
+ }
+
+ public FeedRefreshWorkerResult update(Feed feed) {
+ try {
+ String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
+ FeedFetcherResult result =
+ fetcher.fetch(
+ url,
+ false,
+ feed.getLastModifiedHeader(),
+ feed.getEtagHeader(),
+ feed.getLastPublishedDate(),
+ feed.getLastContentHash());
+ // stops here if NotModifiedException or any other exception is thrown
+
+ List entries = result.feed().entries();
+
+ int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
+ if (maxFeedCapacity > 0) {
+ entries = entries.stream().limit(maxFeedCapacity).toList();
+ }
+
+ Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
+ if (!entriesMaxAge.isZero()) {
+ Instant threshold = Instant.now().minus(entriesMaxAge);
+ entries =
+ entries.stream()
+ .filter(entry -> entry.published().isAfter(threshold))
+ .toList();
+ }
+
+ String urlAfterRedirect = result.urlAfterRedirect();
+ if (Strings.CS.equals(url, urlAfterRedirect)) {
+ urlAfterRedirect = null;
+ }
+
+ feed.setUrlAfterRedirect(urlAfterRedirect);
+ feed.setLink(result.feed().link());
+ feed.setIconUrl(result.feed().iconUrl());
+ feed.setLastModifiedHeader(result.lastModifiedHeader());
+ feed.setEtagHeader(result.lastETagHeader());
+ feed.setLastContentHash(result.contentHash());
+ feed.setLastPublishedDate(result.feed().lastPublishedDate());
+ feed.setAverageEntryInterval(result.feed().averageEntryInterval());
+ feed.setLastEntryDate(result.feed().lastEntryDate());
+
+ feed.setErrorCount(0);
+ feed.setMessage(null);
+ feed.setDisabledUntil(
+ refreshIntervalCalculator.onFetchSuccess(
+ result.feed().lastPublishedDate(),
+ result.feed().averageEntryInterval(),
+ result.validFor()));
+
+ return new FeedRefreshWorkerResult(feed, entries);
+ } catch (NotModifiedException e) {
+ log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
+
+ feed.setErrorCount(0);
+ feed.setMessage(e.getMessage());
+ feed.setDisabledUntil(
+ refreshIntervalCalculator.onFeedNotModified(
+ feed.getLastPublishedDate(), feed.getAverageEntryInterval()));
+
+ if (e.getNewLastModifiedHeader() != null) {
+ feed.setLastModifiedHeader(e.getNewLastModifiedHeader());
+ }
+
+ if (e.getNewEtagHeader() != null) {
+ feed.setEtagHeader(e.getNewEtagHeader());
+ }
+
+ return new FeedRefreshWorkerResult(feed, Collections.emptyList());
+ } catch (TooManyRequestsException e) {
+ log.debug("Too many requests : {}", feed.getUrl());
+
+ feed.setErrorCount(feed.getErrorCount() + 1);
+ feed.setMessage("Server indicated that we are sending too many requests");
+ feed.setDisabledUntil(
+ refreshIntervalCalculator.onTooManyRequests(
+ e.getRetryAfter(), feed.getErrorCount()));
+
+ return new FeedRefreshWorkerResult(feed, Collections.emptyList());
+ } catch (Exception e) {
+ log.debug("unable to refresh feed {}", feed.getUrl(), e);
+
+ feed.setErrorCount(feed.getErrorCount() + 1);
+ feed.setMessage("Unable to refresh feed : " + e.getMessage());
+ feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount()));
+
+ return new FeedRefreshWorkerResult(feed, Collections.emptyList());
+ } finally {
+ feedFetched.mark();
+ }
+ }
+
+ public record FeedRefreshWorkerResult(Feed feed, List entries) {}
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUpdateNotifier.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUpdateNotifier.java
new file mode 100644
index 000000000..1e71b2950
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUpdateNotifier.java
@@ -0,0 +1,54 @@
+package com.commafeed.backend.feed;
+
+import com.commafeed.CommaFeedConfiguration;
+import com.commafeed.backend.dao.UnitOfWork;
+import com.commafeed.backend.dao.UserSettingsDAO;
+import com.commafeed.backend.model.FeedEntry;
+import com.commafeed.backend.model.FeedSubscription;
+import com.commafeed.backend.model.UserSettings;
+import com.commafeed.backend.service.PushNotificationService;
+import com.commafeed.frontend.ws.WebSocketMessageBuilder;
+import com.commafeed.frontend.ws.WebSocketSessions;
+
+import jakarta.inject.Singleton;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.List;
+
+@Slf4j
+@Singleton
+@RequiredArgsConstructor
+public class FeedUpdateNotifier {
+
+ private final CommaFeedConfiguration config;
+ private final UnitOfWork unitOfWork;
+ private final UserSettingsDAO userSettingsDAO;
+ private final WebSocketSessions webSocketSessions;
+ private final PushNotificationService pushNotificationService;
+
+ public void notifyOverWebsocket(FeedSubscription sub, List entries) {
+ if (!entries.isEmpty()) {
+ webSocketSessions.sendMessage(
+ sub.getUser(), WebSocketMessageBuilder.newFeedEntries(sub, entries.size()));
+ }
+ }
+
+ public void sendPushNotifications(FeedSubscription sub, List entries) {
+ if (!config.pushNotifications().enabled()
+ || !sub.isPushNotificationsEnabled()
+ || entries.isEmpty()) {
+ return;
+ }
+
+ UserSettings settings = unitOfWork.call(() -> userSettingsDAO.findByUser(sub.getUser()));
+ if (settings != null
+ && settings.getPushNotifications() != null
+ && settings.getPushNotifications().getType() != null) {
+ for (FeedEntry entry : entries) {
+ pushNotificationService.notify(settings.getPushNotifications(), sub, entry);
+ }
+ }
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java
new file mode 100644
index 000000000..691c6bced
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java
@@ -0,0 +1,99 @@
+package com.commafeed.backend.feed;
+
+import com.commafeed.backend.feed.parser.TextDirectionDetector;
+import com.commafeed.backend.model.FeedSubscription;
+import com.commafeed.frontend.model.Entry;
+import com.rometools.rome.feed.synd.SyndContentImpl;
+import com.rometools.rome.feed.synd.SyndEnclosureImpl;
+import com.rometools.rome.feed.synd.SyndEntry;
+import com.rometools.rome.feed.synd.SyndEntryImpl;
+
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang3.StringUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import java.util.Collections;
+import java.util.Date;
+
+/** Utility methods related to feed handling */
+@UtilityClass
+@Slf4j
+public class FeedUtils {
+
+ public static String truncate(String string, int length) {
+ return StringUtils.truncate(string, length);
+ }
+
+ public static boolean isRTL(String title, String content) {
+ String text = StringUtils.isNotBlank(content) ? content : title;
+ if (StringUtils.isBlank(text)) {
+ return false;
+ }
+
+ String stripped = Jsoup.parse(text).text();
+ if (StringUtils.isBlank(stripped)) {
+ return false;
+ }
+
+ return TextDirectionDetector.detect(stripped)
+ == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
+ }
+
+ public static String getFaviconUrl(FeedSubscription subscription) {
+ return "rest/feed/favicon/" + subscription.getId();
+ }
+
+ public static String proxyImages(String content) {
+ if (StringUtils.isBlank(content)) {
+ return content;
+ }
+
+ Document doc = Jsoup.parse(content);
+ Elements elements = doc.select("img");
+ for (Element element : elements) {
+ String href = element.attr("src");
+ if (StringUtils.isNotBlank(href)) {
+ String proxy = proxyImage(href);
+ element.attr("src", proxy);
+ }
+ }
+
+ return doc.body().html();
+ }
+
+ public static String proxyImage(String url) {
+ if (StringUtils.isBlank(url)) {
+ return url;
+ }
+
+ return "rest/server/proxy?u=" + ImageProxyUrl.encode(url);
+ }
+
+ public static SyndEntry asRss(Entry entry) {
+ SyndEntry e = new SyndEntryImpl();
+
+ e.setUri(entry.getGuid());
+ e.setTitle(entry.getTitle());
+ e.setAuthor(entry.getAuthor());
+
+ SyndContentImpl c = new SyndContentImpl();
+ c.setValue(entry.getContent());
+ e.setContents(Collections.singletonList(c));
+
+ if (entry.getEnclosureUrl() != null) {
+ SyndEnclosureImpl enclosure = new SyndEnclosureImpl();
+ enclosure.setType(entry.getEnclosureType());
+ enclosure.setUrl(entry.getEnclosureUrl());
+ e.setEnclosures(Collections.singletonList(enclosure));
+ }
+
+ e.setLink(entry.getUrl());
+ e.setPublishedDate(entry.getDate() == null ? null : Date.from(entry.getDate()));
+ return e;
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/ImageProxyUrl.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/ImageProxyUrl.java
new file mode 100644
index 000000000..253c4ba7e
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/ImageProxyUrl.java
@@ -0,0 +1,69 @@
+package com.commafeed.backend.feed;
+
+import com.google.common.primitives.Bytes;
+
+import lombok.experimental.UtilityClass;
+
+import org.apache.commons.lang3.RandomUtils;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Base64;
+
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.GCMParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+
+@UtilityClass
+public class ImageProxyUrl {
+
+ private static final int GCM_IV_LENGTH = 12;
+ private static final int GCM_TAG_LENGTH = 128;
+
+ private static SecretKey key;
+
+ public static void generateKey() {
+ key = new SecretKeySpec(RandomUtils.secure().randomBytes(32), "AES");
+ }
+
+ public static String encode(String url) {
+ if (key == null) {
+ throw new IllegalStateException("Key not initialized");
+ }
+
+ try {
+ byte[] iv = RandomUtils.secure().randomBytes(GCM_IV_LENGTH);
+
+ Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
+ cipher.init(Cipher.ENCRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_LENGTH, iv));
+ byte[] encrypted = cipher.doFinal(url.getBytes(StandardCharsets.UTF_8));
+
+ byte[] combined = Bytes.concat(iv, encrypted);
+ return Base64.getUrlEncoder().withoutPadding().encodeToString(combined);
+ } catch (Exception e) {
+ throw new IllegalStateException("Failed to encode URL", e);
+ }
+ }
+
+ public static String decode(String code) {
+ if (key == null) {
+ throw new IllegalStateException("Key not initialized");
+ }
+
+ try {
+ byte[] combined = Base64.getUrlDecoder().decode(code);
+
+ byte[] iv = Arrays.copyOfRange(combined, 0, GCM_IV_LENGTH);
+ byte[] encrypted = Arrays.copyOfRange(combined, GCM_IV_LENGTH, combined.length);
+
+ Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
+ cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_LENGTH, iv));
+ byte[] decrypted = cipher.doFinal(encrypted);
+
+ return new String(decrypted, StandardCharsets.UTF_8);
+ } catch (Exception e) {
+ throw new IllegalStateException("Failed to decode URL", e);
+ }
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/EncodingDetector.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/EncodingDetector.java
new file mode 100644
index 000000000..eb9043c7b
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/EncodingDetector.java
@@ -0,0 +1,64 @@
+package com.commafeed.backend.feed.parser;
+
+import com.ibm.icu.text.CharsetDetector;
+import com.ibm.icu.text.CharsetMatch;
+
+import jakarta.inject.Singleton;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.Strings;
+
+import java.nio.charset.Charset;
+
+@Singleton
+public class EncodingDetector {
+
+ /**
+ * Detect feed encoding by using the declared encoding in the xml processing instruction and by
+ * detecting the characters used in the feed
+ */
+ public Charset getEncoding(byte[] bytes) {
+ String extracted = extractDeclaredEncoding(bytes);
+ if (Strings.CI.startsWith(extracted, "iso-8859-")) {
+ if (!Strings.CS.endsWith(extracted, "1")) {
+ return Charset.forName(extracted);
+ }
+ } else if (Strings.CI.startsWith(extracted, "windows-")) {
+ return Charset.forName(extracted);
+ }
+ return detectEncoding(bytes);
+ }
+
+ /** Extract the declared encoding from the xml */
+ public String extractDeclaredEncoding(byte[] bytes) {
+ int index = ArrayUtils.indexOf(bytes, (byte) '>');
+ if (index == -1) {
+ return null;
+ }
+
+ String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
+ index = Strings.CS.indexOf(pi, "encoding=\"");
+ if (index == -1) {
+ return null;
+ }
+ String encoding = pi.substring(index + 10);
+ encoding = encoding.substring(0, encoding.indexOf('"'));
+ return encoding;
+ }
+
+ /** Detect encoding by analyzing characters in the array */
+ private Charset detectEncoding(byte[] bytes) {
+ String encoding = "UTF-8";
+
+ CharsetDetector detector = new CharsetDetector();
+ detector.setText(bytes);
+ CharsetMatch match = detector.detect();
+ if (match != null) {
+ encoding = match.getName();
+ }
+ if (encoding.equalsIgnoreCase("ISO-8859-1")) {
+ encoding = "windows-1252";
+ }
+ return Charset.forName(encoding);
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParser.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParser.java
new file mode 100644
index 000000000..ad22118cc
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParser.java
@@ -0,0 +1,303 @@
+package com.commafeed.backend.feed.parser;
+
+import com.commafeed.backend.Urls;
+import com.commafeed.backend.feed.parser.FeedParserResult.Content;
+import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
+import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
+import com.commafeed.backend.feed.parser.FeedParserResult.Media;
+import com.rometools.modules.mediarss.MediaEntryModule;
+import com.rometools.modules.mediarss.MediaModule;
+import com.rometools.modules.mediarss.types.MediaGroup;
+import com.rometools.modules.mediarss.types.Metadata;
+import com.rometools.modules.mediarss.types.Thumbnail;
+import com.rometools.rome.feed.synd.SyndCategory;
+import com.rometools.rome.feed.synd.SyndContent;
+import com.rometools.rome.feed.synd.SyndEnclosure;
+import com.rometools.rome.feed.synd.SyndEntry;
+import com.rometools.rome.feed.synd.SyndFeed;
+import com.rometools.rome.feed.synd.SyndLink;
+import com.rometools.rome.feed.synd.SyndLinkImpl;
+import com.rometools.rome.io.SyndFeedInput;
+
+import jakarta.inject.Singleton;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.jdom2.Element;
+import org.jdom2.Namespace;
+import org.xml.sax.InputSource;
+
+import java.io.StringReader;
+import java.nio.charset.Charset;
+import java.text.DateFormat;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/** Parses raw xml into a FeedParserResult object */
+@Singleton
+public class FeedParser {
+
+ private static final Namespace ATOM_10_NS =
+ Namespace.getNamespace("http://www.w3.org/2005/Atom");
+
+ private static final Instant START = Instant.ofEpochMilli(86400000);
+ private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
+
+ private static final Comparator ENTRY_COMPARATOR =
+ Comparator.comparing(Entry::published).reversed();
+
+ private final EncodingDetector encodingDetector;
+ private final XMLCleaner xmlCleaner;
+
+ public FeedParser(EncodingDetector encodingDetector, XMLCleaner xmlCleaner) {
+ this.encodingDetector = encodingDetector;
+ this.xmlCleaner = xmlCleaner;
+ }
+
+ public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedParsingException {
+ try {
+ Charset encoding = encodingDetector.getEncoding(xml);
+
+ String xmlString = xmlCleaner.clean(new String(xml, encoding));
+ if (xmlString == null) {
+ throw new FeedParsingException("Input string is empty for url " + feedUrl);
+ }
+
+ InputSource source = new InputSource(new StringReader(xmlString));
+ SyndFeed feed = new SyndFeedInput().build(source);
+ handleForeignMarkup(feed);
+
+ String title = feed.getTitle();
+ String link = Urls.sanitize(feed.getLink());
+ String iconUrl = Urls.sanitize(feed.getIcon() != null ? feed.getIcon().getUrl() : null);
+ List entries = buildEntries(feed, feedUrl);
+ Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
+ Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
+ if (lastPublishedDate == null
+ || lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
+ lastPublishedDate = lastEntryDate;
+ }
+ Long averageEntryInterval = averageTimeBetweenEntries(entries);
+
+ return new FeedParserResult(
+ title,
+ link,
+ iconUrl,
+ lastPublishedDate,
+ averageEntryInterval,
+ lastEntryDate,
+ entries);
+ } catch (FeedParsingException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new FeedParsingException(
+ String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
+ }
+ }
+
+ /** Adds atom links for rss feeds */
+ private void handleForeignMarkup(SyndFeed feed) {
+ List foreignMarkup = feed.getForeignMarkup();
+ if (foreignMarkup == null) {
+ return;
+ }
+ for (Element element : foreignMarkup) {
+ if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
+ SyndLink link = new SyndLinkImpl();
+ link.setRel(element.getAttributeValue("rel"));
+ link.setHref(element.getAttributeValue("href"));
+ feed.getLinks().add(link);
+ }
+ }
+ }
+
+ private List buildEntries(SyndFeed feed, String feedUrl) {
+ List entries = new ArrayList<>();
+
+ for (SyndEntry item : feed.getEntries()) {
+ String guid = item.getUri();
+ if (StringUtils.isBlank(guid)) {
+ guid = item.getLink();
+ }
+ if (StringUtils.isBlank(guid)) {
+ // no guid and no link, skip entry
+ continue;
+ }
+
+ String url = buildEntryUrl(feed, feedUrl, item);
+ if (StringUtils.isBlank(url) && Urls.isAbsolute(guid)) {
+ // if link is empty but guid is used as url, use guid
+ url = guid;
+ }
+
+ Instant publishedDate = buildEntryPublishedDate(item);
+ Content content = buildContent(item);
+
+ entries.add(new Entry(guid, Urls.sanitize(url), publishedDate, content));
+ }
+
+ entries.sort(ENTRY_COMPARATOR);
+ return entries;
+ }
+
+ private Content buildContent(SyndEntry item) {
+ String title = getTitle(item);
+ String content = getContent(item);
+ String author = StringUtils.trimToNull(item.getAuthor());
+ String categories =
+ StringUtils.trimToNull(
+ item.getCategories().stream()
+ .map(SyndCategory::getName)
+ .collect(Collectors.joining(", ")));
+
+ Enclosure enclosure = buildEnclosure(item);
+ Media media = buildMedia(item);
+ return new Content(title, content, author, categories, enclosure, media);
+ }
+
+ private Enclosure buildEnclosure(SyndEntry item) {
+ SyndEnclosure enclosure = item.getEnclosures().stream().findFirst().orElse(null);
+ if (enclosure == null) {
+ return null;
+ }
+
+ return new Enclosure(Urls.sanitize(enclosure.getUrl()), enclosure.getType());
+ }
+
+ private Instant buildEntryPublishedDate(SyndEntry item) {
+ Date date = item.getPublishedDate();
+ if (date == null) {
+ date = item.getUpdatedDate();
+ }
+ return toValidInstant(date, true);
+ }
+
+ private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
+ String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
+ if (url == null || Urls.isAbsolute(url)) {
+ // url is absolute, nothing to do
+ return url;
+ }
+
+ // url is relative, trying to resolve it
+ String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
+ return Urls.toAbsolute(url, feedLink, feedUrl);
+ }
+
+ private Instant toValidInstant(Date date, boolean nullToNow) {
+ Instant now = Instant.now();
+ if (date == null) {
+ return nullToNow ? now : null;
+ }
+
+ Instant instant = date.toInstant();
+ if (instant.isBefore(START) || instant.isAfter(END)) {
+ return now;
+ }
+
+ if (instant.isAfter(now)) {
+ return now;
+ }
+ return instant;
+ }
+
+ private String getContent(SyndEntry item) {
+ String content;
+ if (item.getContents().isEmpty()) {
+ content = item.getDescription() == null ? null : item.getDescription().getValue();
+ } else {
+ content =
+ item.getContents().stream()
+ .map(SyndContent::getValue)
+ .collect(Collectors.joining(System.lineSeparator()));
+ }
+ return StringUtils.trimToNull(content);
+ }
+
+ private String getTitle(SyndEntry item) {
+ String title = item.getTitle();
+ if (StringUtils.isBlank(title)) {
+ Date date = item.getPublishedDate();
+ if (date != null) {
+ title = DateFormat.getInstance().format(date);
+ } else {
+ title = "(no title)";
+ }
+ }
+ return StringUtils.trimToNull(title);
+ }
+
+ private Media buildMedia(SyndEntry item) {
+ MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
+ if (module == null) {
+ return null;
+ }
+
+ Media media = buildMedia(module.getMetadata());
+ if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
+ MediaGroup group = module.getMediaGroups()[0];
+ media = buildMedia(group.getMetadata());
+ }
+
+ return media;
+ }
+
+ private Media buildMedia(Metadata metadata) {
+ if (metadata == null) {
+ return null;
+ }
+
+ String description = metadata.getDescription();
+
+ String thumbnailUrl = null;
+ Integer thumbnailWidth = null;
+ Integer thumbnailHeight = null;
+ if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
+ Thumbnail thumbnail = metadata.getThumbnail()[0];
+ thumbnailWidth = thumbnail.getWidth();
+ thumbnailHeight = thumbnail.getHeight();
+ if (thumbnail.getUrl() != null) {
+ thumbnailUrl = thumbnail.getUrl().toString();
+ }
+ }
+
+ if (description == null && thumbnailUrl == null) {
+ return null;
+ }
+
+ return new Media(description, Urls.sanitize(thumbnailUrl), thumbnailWidth, thumbnailHeight);
+ }
+
+ private Long averageTimeBetweenEntries(List entries) {
+ if (entries.isEmpty() || entries.size() == 1) {
+ return null;
+ }
+
+ SummaryStatistics stats = new SummaryStatistics();
+ for (int i = 0; i < entries.size() - 1; i++) {
+ long diff =
+ Math.abs(
+ entries.get(i).published().toEpochMilli()
+ - entries.get(i + 1).published().toEpochMilli());
+ stats.addValue(diff);
+ }
+ return (long) stats.getMean();
+ }
+
+ public static class FeedParsingException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ public FeedParsingException(String message) {
+ super(message);
+ }
+
+ public FeedParsingException(String message, Throwable cause) {
+ super(message, cause);
+ }
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParserResult.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParserResult.java
new file mode 100644
index 000000000..cb343da13
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParserResult.java
@@ -0,0 +1,31 @@
+package com.commafeed.backend.feed.parser;
+
+import java.time.Instant;
+import java.util.List;
+
+public record FeedParserResult(
+ String title,
+ String link,
+ String iconUrl,
+ Instant lastPublishedDate,
+ Long averageEntryInterval,
+ Instant lastEntryDate,
+ List entries) {
+ public record Entry(String guid, String url, Instant published, Content content) {}
+
+ public record Content(
+ String title,
+ String content,
+ String author,
+ String categories,
+ Enclosure enclosure,
+ Media media) {}
+
+ public record Enclosure(String url, String type) {}
+
+ public record Media(
+ String description,
+ String thumbnailUrl,
+ Integer thumbnailWidth,
+ Integer thumbnailHeight) {}
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/HtmlEntities.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/HtmlEntities.java
new file mode 100644
index 000000000..c8e230977
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/HtmlEntities.java
@@ -0,0 +1,271 @@
+package com.commafeed.backend.feed.parser;
+
+import lombok.experimental.UtilityClass;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+@UtilityClass
+class HtmlEntities {
+ public static final Map HTML_TO_NUMERIC_MAP;
+ public static final List HTML_ENTITIES;
+
+ static {
+ Map map = new LinkedHashMap<>();
+ map.put("Á", "Á");
+ map.put("á", "á");
+ map.put("Â", "Â");
+ map.put("â", "â");
+ map.put("´", "´");
+ map.put("Æ", "Æ");
+ map.put("æ", "æ");
+ map.put("À", "À");
+ map.put("à", "à");
+ map.put("ℵ", "ℵ");
+ map.put("Α", "Α");
+ map.put("α", "α");
+ map.put("&", "&");
+ map.put("∧", "∧");
+ map.put("∠", "∠");
+ map.put("Å", "Å");
+ map.put("å", "å");
+ map.put("≈", "≈");
+ map.put("Ã", "Ã");
+ map.put("ã", "ã");
+ map.put("Ä", "Ä");
+ map.put("ä", "ä");
+ map.put("„", "„");
+ map.put("Β", "Β");
+ map.put("β", "β");
+ map.put("¦", "¦");
+ map.put("•", "•");
+ map.put("∩", "∩");
+ map.put("Ç", "Ç");
+ map.put("ç", "ç");
+ map.put("¸", "¸");
+ map.put("¢", "¢");
+ map.put("Χ", "Χ");
+ map.put("χ", "χ");
+ map.put("ˆ", "ˆ");
+ map.put("♣", "♣");
+ map.put("≅", "≅");
+ map.put("©", "©");
+ map.put("↵", "↵");
+ map.put("∪", "∪");
+ map.put("¤", "¤");
+ map.put("†", "†");
+ map.put("‡", "‡");
+ map.put("↓", "↓");
+ map.put("⇓", "⇓");
+ map.put("°", "°");
+ map.put("Δ", "Δ");
+ map.put("δ", "δ");
+ map.put("♦", "♦");
+ map.put("÷", "÷");
+ map.put("É", "É");
+ map.put("é", "é");
+ map.put("Ê", "Ê");
+ map.put("ê", "ê");
+ map.put("È", "È");
+ map.put("è", "è");
+ map.put("∅", "∅");
+ map.put(" ", " ");
+ map.put(" ", " ");
+ map.put("Ε", "Ε");
+ map.put("ε", "ε");
+ map.put("≡", "≡");
+ map.put("Η", "Η");
+ map.put("η", "η");
+ map.put("Ð", "Ð");
+ map.put("ð", "ð");
+ map.put("Ë", "Ë");
+ map.put("ë", "ë");
+ map.put("€", "€");
+ map.put("∃", "∃");
+ map.put("ƒ", "ƒ");
+ map.put("∀", "∀");
+ map.put("½", "½");
+ map.put("¼", "¼");
+ map.put("¾", "¾");
+ map.put("⁄", "⁄");
+ map.put("Γ", "Γ");
+ map.put("γ", "γ");
+ map.put("≥", "≥");
+ map.put("↔", "↔");
+ map.put("⇔", "⇔");
+ map.put("♥", "♥");
+ map.put("…", "…");
+ map.put("Í", "Í");
+ map.put("í", "í");
+ map.put("Î", "Î");
+ map.put("î", "î");
+ map.put("¡", "¡");
+ map.put("Ì", "Ì");
+ map.put("ì", "ì");
+ map.put("ℑ", "ℑ");
+ map.put("∞", "∞");
+ map.put("∫", "∫");
+ map.put("Ι", "Ι");
+ map.put("ι", "ι");
+ map.put("¿", "¿");
+ map.put("∈", "∈");
+ map.put("Ï", "Ï");
+ map.put("ï", "ï");
+ map.put("Κ", "Κ");
+ map.put("κ", "κ");
+ map.put("Λ", "Λ");
+ map.put("λ", "λ");
+ map.put("〈", "〈");
+ map.put("«", "«");
+ map.put("←", "←");
+ map.put("⇐", "⇐");
+ map.put("⌈", "⌈");
+ map.put("“", "“");
+ map.put("≤", "≤");
+ map.put("⌊", "⌊");
+ map.put("∗", "∗");
+ map.put("◊", "◊");
+ map.put("", "");
+ map.put("‹", "‹");
+ map.put("‘", "‘");
+ map.put("¯", "¯");
+ map.put("—", "—");
+ map.put("µ", "µ");
+ map.put("·", "·");
+ map.put("−", "−");
+ map.put("Μ", "Μ");
+ map.put("μ", "μ");
+ map.put("∇", "∇");
+ map.put(" ", " ");
+ map.put("–", "–");
+ map.put("≠", "≠");
+ map.put("∋", "∋");
+ map.put("¬", "¬");
+ map.put("∉", "∉");
+ map.put("⊄", "⊄");
+ map.put("Ñ", "Ñ");
+ map.put("ñ", "ñ");
+ map.put("Ν", "Ν");
+ map.put("ν", "ν");
+ map.put("Ó", "Ó");
+ map.put("ó", "ó");
+ map.put("Ô", "Ô");
+ map.put("ô", "ô");
+ map.put("Œ", "Œ");
+ map.put("œ", "œ");
+ map.put("Ò", "Ò");
+ map.put("ò", "ò");
+ map.put("‾", "‾");
+ map.put("Ω", "Ω");
+ map.put("ω", "ω");
+ map.put("Ο", "Ο");
+ map.put("ο", "ο");
+ map.put("⊕", "⊕");
+ map.put("∨", "∨");
+ map.put("ª", "ª");
+ map.put("º", "º");
+ map.put("Ø", "Ø");
+ map.put("ø", "ø");
+ map.put("Õ", "Õ");
+ map.put("õ", "õ");
+ map.put("⊗", "⊗");
+ map.put("Ö", "Ö");
+ map.put("ö", "ö");
+ map.put("¶", "¶");
+ map.put("∂", "∂");
+ map.put("‰", "‰");
+ map.put("⊥", "⊥");
+ map.put("Φ", "Φ");
+ map.put("φ", "φ");
+ map.put("Π", "Π");
+ map.put("π", "π");
+ map.put("ϖ", "ϖ");
+ map.put("±", "±");
+ map.put("£", "£");
+ map.put("′", "′");
+ map.put("″", "″");
+ map.put("∏", "∏");
+ map.put("∝", "∝");
+ map.put("Ψ", "Ψ");
+ map.put("ψ", "ψ");
+ map.put(""", """);
+ map.put("√", "√");
+ map.put("〉", "〉");
+ map.put("»", "»");
+ map.put("→", "→");
+ map.put("⇒", "⇒");
+ map.put("⌉", "⌉");
+ map.put("”", "”");
+ map.put("ℜ", "ℜ");
+ map.put("®", "®");
+ map.put("⌋", "⌋");
+ map.put("Ρ", "Ρ");
+ map.put("ρ", "ρ");
+ map.put("", "");
+ map.put("›", "›");
+ map.put("’", "’");
+ map.put("‚", "‚");
+ map.put("Š", "Š");
+ map.put("š", "š");
+ map.put("⋅", "⋅");
+ map.put("§", "§");
+ map.put("", "");
+ map.put("Σ", "Σ");
+ map.put("σ", "σ");
+ map.put("ς", "ς");
+ map.put("∼", "∼");
+ map.put("♠", "♠");
+ map.put("⊂", "⊂");
+ map.put("⊆", "⊆");
+ map.put("∑", "∑");
+ map.put("¹", "¹");
+ map.put("²", "²");
+ map.put("³", "³");
+ map.put("⊃", "⊃");
+ map.put("⊇", "⊇");
+ map.put("ß", "ß");
+ map.put("Τ", "Τ");
+ map.put("τ", "τ");
+ map.put("∴", "∴");
+ map.put("Θ", "Θ");
+ map.put("θ", "θ");
+ map.put("ϑ", "ϑ");
+ map.put(" ", " ");
+ map.put("Þ", "Þ");
+ map.put("þ", "þ");
+ map.put("˜", "˜");
+ map.put("×", "×");
+ map.put("™", "™");
+ map.put("Ú", "Ú");
+ map.put("ú", "ú");
+ map.put("↑", "↑");
+ map.put("⇑", "⇑");
+ map.put("Û", "Û");
+ map.put("û", "û");
+ map.put("Ù", "Ù");
+ map.put("ù", "ù");
+ map.put("¨", "¨");
+ map.put("ϒ", "ϒ");
+ map.put("Υ", "Υ");
+ map.put("υ", "υ");
+ map.put("Ü", "Ü");
+ map.put("ü", "ü");
+ map.put("℘", "℘");
+ map.put("Ξ", "Ξ");
+ map.put("ξ", "ξ");
+ map.put("Ý", "Ý");
+ map.put("ý", "ý");
+ map.put("¥", "¥");
+ map.put("ÿ", "ÿ");
+ map.put("Ÿ", "Ÿ");
+ map.put("Ζ", "Ζ");
+ map.put("ζ", "ζ");
+ map.put("", "");
+ map.put("", "");
+
+ HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map);
+ HTML_ENTITIES = List.copyOf(map.keySet());
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/TextDirectionDetector.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/TextDirectionDetector.java
new file mode 100644
index 000000000..cba4e2eb2
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/TextDirectionDetector.java
@@ -0,0 +1,56 @@
+package com.commafeed.backend.feed.parser;
+
+import org.apache.commons.lang3.math.NumberUtils;
+
+import java.text.Bidi;
+import java.util.regex.Pattern;
+
+public class TextDirectionDetector {
+
+ private static final Pattern WORDS_PATTERN = Pattern.compile("\\s+");
+ private static final Pattern URL_PATTERN = Pattern.compile("^https?://.*");
+
+ private static final double RTL_THRESHOLD = 0.4D;
+
+ public enum Direction {
+ LEFT_TO_RIGHT,
+ RIGHT_TO_LEFT
+ }
+
+ public static Direction detect(String input) {
+ if (input == null || input.isBlank()) {
+ return Direction.LEFT_TO_RIGHT;
+ }
+
+ long rtl = 0;
+ long total = 0;
+ for (String token : WORDS_PATTERN.split(input)) {
+ // skip urls
+ if (URL_PATTERN.matcher(token).matches()) {
+ continue;
+ }
+
+ // skip numbers
+ if (NumberUtils.isCreatable(token)) {
+ continue;
+ }
+
+ boolean requiresBidi = Bidi.requiresBidi(token.toCharArray(), 0, token.length());
+ if (requiresBidi) {
+ Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT);
+ if (bidi.getBaseLevel() == 1) {
+ rtl++;
+ }
+ }
+
+ total++;
+ }
+
+ if (total == 0) {
+ return Direction.LEFT_TO_RIGHT;
+ }
+
+ double ratio = (double) rtl / total;
+ return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/XMLCleaner.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/XMLCleaner.java
new file mode 100644
index 000000000..99b9897ec
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/feed/parser/XMLCleaner.java
@@ -0,0 +1,83 @@
+package com.commafeed.backend.feed.parser;
+
+import jakarta.inject.Singleton;
+
+import org.ahocorasick.trie.Emit;
+import org.ahocorasick.trie.Trie;
+import org.apache.commons.lang3.StringUtils;
+import org.jdom2.Verifier;
+
+import java.util.Collection;
+import java.util.regex.Pattern;
+
+@Singleton
+public class XMLCleaner {
+
+ private static final Pattern DOCTYPE_PATTERN =
+ Pattern.compile("]*>", Pattern.CASE_INSENSITIVE);
+
+ private final Trie trie =
+ Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build();
+
+ public String clean(String xml) {
+ xml = removeCharactersBeforeFirstXmlTag(xml);
+ xml = removeInvalidXmlCharacters(xml);
+ xml = replaceHtmlEntitiesWithNumericEntities(xml);
+ xml = removeDoctypeDeclarations(xml);
+ return xml;
+ }
+
+ String removeCharactersBeforeFirstXmlTag(String xml) {
+ if (StringUtils.isBlank(xml)) {
+ return null;
+ }
+
+ int pos = xml.indexOf('<');
+ return pos < 0 ? null : xml.substring(pos);
+ }
+
+ String removeInvalidXmlCharacters(String xml) {
+ if (StringUtils.isBlank(xml)) {
+ return null;
+ }
+
+ return xml.codePoints()
+ .filter(Verifier::isXMLCharacter)
+ .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append)
+ .toString();
+ }
+
+ // https://stackoverflow.com/a/40836618
+ String replaceHtmlEntitiesWithNumericEntities(String source) {
+ if (StringUtils.isBlank(source)) {
+ return null;
+ }
+
+ // Create a buffer sufficiently large that re-allocations are minimized.
+ StringBuilder sb = new StringBuilder(source.length() << 1);
+
+ Collection emits = trie.parseText(source);
+
+ int prevIndex = 0;
+ for (Emit emit : emits) {
+ int matchIndex = emit.getStart();
+
+ sb.append(source, prevIndex, matchIndex);
+ sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
+ prevIndex = emit.getEnd() + 1;
+ }
+
+ // Add the remainder of the string (contains no more matches).
+ sb.append(source.substring(prevIndex));
+
+ return sb.toString();
+ }
+
+ String removeDoctypeDeclarations(String xml) {
+ if (StringUtils.isBlank(xml)) {
+ return null;
+ }
+
+ return DOCTYPE_PATTERN.matcher(xml).replaceAll("");
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/AbstractModel.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/AbstractModel.java
new file mode 100644
index 000000000..9f187a2fc
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/AbstractModel.java
@@ -0,0 +1,30 @@
+package com.commafeed.backend.model;
+
+import jakarta.persistence.GeneratedValue;
+import jakarta.persistence.GenerationType;
+import jakarta.persistence.Id;
+import jakarta.persistence.MappedSuperclass;
+import jakarta.persistence.TableGenerator;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.io.Serializable;
+
+/** Abstract model for all entities, defining id and table generator */
+@SuppressWarnings("serial")
+@MappedSuperclass
+@Getter
+@Setter
+public abstract class AbstractModel implements Serializable {
+
+ @Id
+ @GeneratedValue(strategy = GenerationType.TABLE, generator = "gen")
+ @TableGenerator(
+ name = "gen",
+ table = "hibernate_sequences",
+ pkColumnName = "sequence_name",
+ valueColumnName = "sequence_next_hi_value",
+ allocationSize = 1000)
+ private Long id;
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/Feed.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/Feed.java
new file mode 100644
index 000000000..3784b80f3
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/Feed.java
@@ -0,0 +1,85 @@
+package com.commafeed.backend.model;
+
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.Lob;
+import jakarta.persistence.Table;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import org.hibernate.annotations.JdbcTypeCode;
+
+import java.sql.Types;
+import java.time.Instant;
+
+@Entity
+@Table(name = "FEEDS")
+@SuppressWarnings("serial")
+@Getter
+@Setter
+public class Feed extends AbstractModel {
+
+ /** The url of the feed */
+ @Lob
+ @Column(length = Integer.MAX_VALUE, nullable = false)
+ @JdbcTypeCode(Types.LONGVARCHAR)
+ private String url;
+
+ /** cache the url after potential http 30x redirects */
+ @Column(name = "url_after_redirect", length = 2048, nullable = false)
+ private String urlAfterRedirect;
+
+ @Column(length = 2048, nullable = false)
+ private String normalizedUrl;
+
+ @Column(length = 40, nullable = false)
+ private String normalizedUrlHash;
+
+ /** The url of the website, extracted from the feed */
+ @Lob
+ @Column(length = Integer.MAX_VALUE)
+ @JdbcTypeCode(Types.LONGVARCHAR)
+ private String link;
+
+ @Lob
+ @Column(name = "icon_url", length = Integer.MAX_VALUE)
+ @JdbcTypeCode(Types.LONGVARCHAR)
+ private String iconUrl;
+
+ /** Last time we tried to fetch the feed */
+ @Column private Instant lastUpdated;
+
+ /** Last publishedDate value in the feed */
+ @Column private Instant lastPublishedDate;
+
+ /** date of the last entry of the feed */
+ @Column private Instant lastEntryDate;
+
+ /** error message while retrieving the feed */
+ @Lob
+ @Column(length = Integer.MAX_VALUE)
+ @JdbcTypeCode(Types.LONGVARCHAR)
+ private String message;
+
+ /** times we failed to retrieve the feed */
+ private int errorCount;
+
+ /** feed refresh is disabled until this date */
+ @Column private Instant disabledUntil;
+
+ /** http header returned by the feed */
+ @Column(length = 64)
+ private String lastModifiedHeader;
+
+ /** http header returned by the feed */
+ @Column(length = 255)
+ private String etagHeader;
+
+ /** average time between entries in the feed */
+ private Long averageEntryInterval;
+
+ /** last hash of the content of the feed xml */
+ @Column(length = 40)
+ private String lastContentHash;
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/FeedCategory.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/FeedCategory.java
new file mode 100644
index 000000000..041f8ad7b
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/FeedCategory.java
@@ -0,0 +1,33 @@
+package com.commafeed.backend.model;
+
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.FetchType;
+import jakarta.persistence.JoinColumn;
+import jakarta.persistence.ManyToOne;
+import jakarta.persistence.Table;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Entity
+@Table(name = "FEEDCATEGORIES")
+@SuppressWarnings("serial")
+@Getter
+@Setter
+public class FeedCategory extends AbstractModel {
+
+ @Column(length = 128, nullable = false)
+ private String name;
+
+ @ManyToOne(fetch = FetchType.LAZY)
+ @JoinColumn(nullable = false)
+ private User user;
+
+ @ManyToOne(fetch = FetchType.LAZY)
+ private FeedCategory parent;
+
+ private boolean collapsed;
+
+ private int position;
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntry.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntry.java
new file mode 100644
index 000000000..8ac4fb170
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntry.java
@@ -0,0 +1,53 @@
+package com.commafeed.backend.model;
+
+import jakarta.persistence.CascadeType;
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.FetchType;
+import jakarta.persistence.JoinColumn;
+import jakarta.persistence.ManyToOne;
+import jakarta.persistence.OneToMany;
+import jakarta.persistence.Table;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.time.Instant;
+import java.util.Set;
+
+@Entity
+@Table(name = "FEEDENTRIES")
+@SuppressWarnings("serial")
+@Getter
+@Setter
+public class FeedEntry extends AbstractModel {
+
+ @Column(length = 2048, nullable = false)
+ private String guid;
+
+ @Column(length = 40, nullable = false)
+ private String guidHash;
+
+ @ManyToOne(fetch = FetchType.LAZY)
+ private Feed feed;
+
+ @ManyToOne(fetch = FetchType.LAZY, optional = false)
+ @JoinColumn(nullable = false, updatable = false)
+ private FeedEntryContent content;
+
+ @Column(length = 2048)
+ private String url;
+
+ /** the moment the entry was inserted in the database */
+ @Column private Instant inserted;
+
+ /** the moment the entry was published in the feed */
+ @Column(name = "updated")
+ private Instant published;
+
+ @OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
+ private Set statuses;
+
+ @OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
+ private Set tags;
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryContent.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryContent.java
new file mode 100644
index 000000000..8b5d54c59
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryContent.java
@@ -0,0 +1,97 @@
+package com.commafeed.backend.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.EnumType;
+import jakarta.persistence.Enumerated;
+import jakarta.persistence.Lob;
+import jakarta.persistence.Table;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.hibernate.annotations.JdbcTypeCode;
+
+import java.sql.Types;
+
+@Entity
+@Table(name = "FEEDENTRYCONTENTS")
+@SuppressWarnings("serial")
+@Getter
+@Setter
+public class FeedEntryContent extends AbstractModel {
+
+ public enum Direction {
+ @JsonProperty("ltr")
+ LTR,
+
+ @JsonProperty("rtl")
+ RTL,
+
+ @JsonProperty("unknown")
+ UNKNOWN
+ }
+
+ @Column(length = 2048)
+ private String title;
+
+ @Column(length = 40)
+ private String titleHash;
+
+ @Lob
+ @Column(length = Integer.MAX_VALUE)
+ @JdbcTypeCode(Types.LONGVARCHAR)
+ private String content;
+
+ @Column(length = 40)
+ private String contentHash;
+
+ @Column(name = "author", length = 128)
+ private String author;
+
+ @Column(length = 2048)
+ private String enclosureUrl;
+
+ @Column(length = 255)
+ private String enclosureType;
+
+ @Lob
+ @Column(length = Integer.MAX_VALUE)
+ @JdbcTypeCode(Types.LONGVARCHAR)
+ private String mediaDescription;
+
+ @Column(length = 2048)
+ private String mediaThumbnailUrl;
+
+ private Integer mediaThumbnailWidth;
+ private Integer mediaThumbnailHeight;
+
+ @Column(length = 4096)
+ private String categories;
+
+ @Column
+ @Enumerated(EnumType.STRING)
+ private Direction direction = Direction.UNKNOWN;
+
+ public boolean equivalentTo(FeedEntryContent c) {
+ if (c == null) {
+ return false;
+ }
+
+ return new EqualsBuilder()
+ .append(title, c.title)
+ .append(content, c.content)
+ .append(author, c.author)
+ .append(categories, c.categories)
+ .append(enclosureUrl, c.enclosureUrl)
+ .append(enclosureType, c.enclosureType)
+ .append(mediaDescription, c.mediaDescription)
+ .append(mediaThumbnailUrl, c.mediaThumbnailUrl)
+ .append(mediaThumbnailWidth, c.mediaThumbnailWidth)
+ .append(mediaThumbnailHeight, c.mediaThumbnailHeight)
+ .build();
+ }
+}
diff --git a/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryStatus.java b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryStatus.java
new file mode 100644
index 000000000..10749b801
--- /dev/null
+++ b/jdk_25_maven/cs/rest/commafeed/commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryStatus.java
@@ -0,0 +1,61 @@
+package com.commafeed.backend.model;
+
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.FetchType;
+import jakarta.persistence.JoinColumn;
+import jakarta.persistence.ManyToOne;
+import jakarta.persistence.Table;
+import jakarta.persistence.Transient;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+
+@Entity
+@Table(name = "FEEDENTRYSTATUSES")
+@SuppressWarnings("serial")
+@Getter
+@Setter
+public class FeedEntryStatus extends AbstractModel {
+
+ @ManyToOne(fetch = FetchType.LAZY)
+ @JoinColumn(nullable = false)
+ private FeedSubscription subscription;
+
+ @ManyToOne(fetch = FetchType.LAZY)
+ @JoinColumn(nullable = false)
+ private FeedEntry entry;
+
+ @Column(name = "read_status")
+ private boolean read;
+
+ private boolean starred;
+
+ @Transient private boolean markable;
+
+ @Transient private List