Java RSS Feed (#1037)

nateweisz · christolis · bowbahdoe · web-flow · commit 835a951b812b · 2024-04-13T23:33:06.000+03:00
* wip: initial commit * style(spotlessApply): run uncalled task * fix: register routine in features list * feat: update config.json.template * wip(rss-config): change config design * feat: make targetChannelPattern into a Map * refactor: don't hardcode number * feat: improve constructEmbedMessage * feat: improve the description constructing * feat: increase MAX_CONTENTS to 300 * wip: changes * feat(rss-feed): convert polling interval to be configurable * feat: improve date parsing Co-authored-by: Ethan McCue <5004262+bowbahdoe@users.noreply.github.com> * feat: update config.json.template * feat: improve date handling and add embed timestamp * fix: malformed `config.json.template` * fix: now correctly finds the latest date * feat: improve date handling and add embed timestamp * fix: now correctly finds the latest date * feat: you can now optionally declare a specific channel for a feed to go to * finished javadoc todos * improved embed * feat(rss-routine): working version - Added JavaDocs on most parts - Simplified the code and cleaned up unused stuff * feat(rss-routine): increase MAX_CONTENTS to 1000 * refactor(rss-routine): rename to RSSHandlerRoutine While the original idea was to add a Java news and changes RSS feed, this was expanded to all types of RSS feeds, so a more appropriate name makes things more clear. * docs(rss-routine): add JavaDocs for constructor and class * feat: add @NotNull annotation * Update RSSHandlerRoutine.java * fix: reverse feed so it posts in correct order * refactor: use variable types instead of var * feat: use fetchAny() instead of fetch() * fix(rss-handler): remove redundant empty check Co-authored-by: Ethan McCue <5004262+bowbahdoe@users.noreply.github.com> * refactor(rss-handler): remove AtomicReference usage Co-authored-by: Ethan McCue <5004262+bowbahdoe@users.noreply.github.com> * resolve a couple issues * refactor(rss): switch to using a record for the config * feat: rename to fallbackChannelPattern * refactor: remove unnecessary throws exception in method signature * docs: add a few missing parameters and fix typos * refactor: reduce try-catch scope and added clarifying comment * perf: use Stream instead of StringBuilder * refactor: put fallback case into an else statement * refactor: switch to Map#containsKey() for targetChannelPatterns * feat: modularize sendRss() method and improve variable names This also changes the functionality of how new RSS feeds get dealt with for the first time. Before this commit, all items would get posted as embeds on Discord, and as a result, that would bombard the target channel with RSS posts once the routine executes. This commit changes this behavior by assuming that all RSS posts have been posted and it should only consider posting posts newer than the registered date. * refactor: remove star import * fix: add Objects#requireNonNull() on rssConfig * changes * feat: improvements from code reviews Co-authored-by: Nathan Weisz <151496498+nateweisz@users.noreply.github.com> * feat: add DateTimeParseException in signature * refactor: use .orElseThrow() instead of .get() * fixed missing coma after rss configuration Co-authored-by: Tanish <tanishazad03@gmail.com> * fix(rss): feeds support multiple channels fix(rss): polling interval is now clear on unit of time fix(rss): posting messages now uses forEachOrdered * Optional<List<TextChannel>> -> List<TextChannel> * extract item post predicate to seperate function * various changes from null to optional --------- Co-authored-by: christolis <work@chris-sdogkos.com> Co-authored-by: Ethan McCue <5004262+bowbahdoe@users.noreply.github.com> Co-authored-by: Tanish <tanishazad03@gmail.com>
diff --git a/application/build.gradle b/application/build.gradle
@@ -77,6 +77,9 @@ dependencies {
     
     implementation 'org.kohsuke:github-api:1.321'
 
+    implementation 'org.apache.commons:commons-text:1.11.0'
+    implementation 'com.apptasticsoftware:rssreader:3.6.0'
+
     testImplementation 'org.mockito:mockito-core:5.11.0'
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.0'
     testImplementation 'org.junit.jupiter:junit-jupiter-params:5.10.0'
diff --git a/application/config.json.template b/application/config.json.template
@@ -103,6 +103,17 @@
         "special": [
         ]
     },
-    "memberCountCategoryPattern": "Info",
-    "selectRolesChannelPattern": "select-your-roles"
+    "selectRolesChannelPattern": "select-your-roles",
+    "rssConfig": {
+        "feeds": [
+            {
+                "url": "<rss_feed_url>",
+                "targetChannelPattern": "<target_channel_pattern>",
+                "dateFormatterPattern": "EEE, dd MMM yyyy HH:mm:ss zzz"
+            }
+        ],
+        "fallbackChannelPattern": "<fallback_channel_pattern>",
+        "pollIntervalInMinutes": 10
+    },
+    "memberCountCategoryPattern": "Info"
 }
diff --git a/application/src/main/java/org/togetherjava/tjbot/config/Config.java b/application/src/main/java/org/togetherjava/tjbot/config/Config.java
@@ -11,6 +11,7 @@
 import java.util.List;
 import java.util.Objects;
 
+
 /**
  * Configuration of the application. Create instances using {@link #load(Path)}.
  */
@@ -42,6 +43,7 @@ public final class Config {
     private final String sourceCodeBaseUrl;
     private final JShellConfig jshell;
     private final FeatureBlacklistConfig featureBlacklistConfig;
+    private final RSSFeedsConfig rssFeedsConfig;
     private final String selectRolesChannelPattern;
     private final String memberCountCategoryPattern;
 
@@ -90,6 +92,7 @@ private Config(@JsonProperty(value = "token", required = true) String token,
                     required = true) String memberCountCategoryPattern,
             @JsonProperty(value = "featureBlacklist",
                     required = true) FeatureBlacklistConfig featureBlacklistConfig,
+            @JsonProperty(value = "rssConfig", required = true) RSSFeedsConfig rssFeedsConfig,
             @JsonProperty(value = "selectRolesChannelPattern",
                     required = true) String selectRolesChannelPattern) {
         this.token = Objects.requireNonNull(token);
@@ -122,6 +125,7 @@ private Config(@JsonProperty(value = "token", required = true) String token,
         this.sourceCodeBaseUrl = Objects.requireNonNull(sourceCodeBaseUrl);
         this.jshell = Objects.requireNonNull(jshell);
         this.featureBlacklistConfig = Objects.requireNonNull(featureBlacklistConfig);
+        this.rssFeedsConfig = Objects.requireNonNull(rssFeedsConfig);
         this.selectRolesChannelPattern = Objects.requireNonNull(selectRolesChannelPattern);
     }
 
@@ -405,4 +409,13 @@ public String getSelectRolesChannelPattern() {
     public String getMemberCountCategoryPattern() {
         return memberCountCategoryPattern;
     }
+
+    /**
+     * Gets the RSS feeds configuration.
+     *
+     * @return the RSS feeds configuration
+     */
+    public RSSFeedsConfig getRSSFeedsConfig() {
+        return rssFeedsConfig;
+    }
 }
diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java
@@ -0,0 +1,30 @@
+package org.togetherjava.tjbot.config;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import javax.annotation.Nullable;
+
+import java.util.Objects;
+
+/**
+ * Represents an RSS feed configuration.
+ */
+public record RSSFeed(@JsonProperty(value = "url", required = true) String url,
+        @JsonProperty(value = "targetChannelPattern") @Nullable String targetChannelPattern,
+        @JsonProperty(value = "dateFormatterPattern",
+                required = true) String dateFormatterPattern) {
+
+    /**
+     * Constructs an RSSFeed object.
+     *
+     * @param url the URL of the RSS feed
+     * @param targetChannelPattern the target channel pattern
+     * @param dateFormatterPattern the date formatter pattern
+     * @throws NullPointerException if any of the parameters are null
+     */
+    public RSSFeed {
+        Objects.requireNonNull(url);
+        Objects.requireNonNull(targetChannelPattern);
+        Objects.requireNonNull(dateFormatterPattern);
+    }
+}
diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java
@@ -0,0 +1,34 @@
+package org.togetherjava.tjbot.config;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Represents the configuration for an RSS feed, which includes the list of feeds to subscribe to, a
+ * pattern for identifying Java news channels, and the interval (in minutes) for polling the feeds.
+ *
+ * @param feeds The list of RSS feeds to subscribe to.
+ * @param fallbackChannelPattern The pattern used to identify the fallback text channel to use.
+ * @param pollIntervalInMinutes The interval (in minutes) for polling the RSS feeds for updates.
+ */
+public record RSSFeedsConfig(@JsonProperty(value = "feeds", required = true) List<RSSFeed> feeds,
+        @JsonProperty(value = "fallbackChannelPattern",
+                required = true) String fallbackChannelPattern,
+        @JsonProperty(value = "pollIntervalInMinutes", required = true) int pollIntervalInMinutes) {
+
+    /**
+     * Constructs a new {@link RSSFeedsConfig}.
+     *
+     * @param feeds The list of RSS feeds to subscribe to.
+     * @param fallbackChannelPattern The pattern used to identify the fallback text channel to use.
+     * @param pollIntervalInMinutes The interval (in minutes) for polling the RSS feeds for updates.
+     * @throws NullPointerException if any of the parameters (feeds or fallbackChannelPattern) are
+     *         null
+     */
+    public RSSFeedsConfig {
+        Objects.requireNonNull(feeds);
+        Objects.requireNonNull(fallbackChannelPattern);
+    }
+}
diff --git a/application/src/main/java/org/togetherjava/tjbot/features/Features.java b/application/src/main/java/org/togetherjava/tjbot/features/Features.java
@@ -24,6 +24,7 @@
 import org.togetherjava.tjbot.features.github.GitHubCommand;
 import org.togetherjava.tjbot.features.github.GitHubReference;
 import org.togetherjava.tjbot.features.help.*;
+import org.togetherjava.tjbot.features.javamail.RSSHandlerRoutine;
 import org.togetherjava.tjbot.features.jshell.JShellCommand;
 import org.togetherjava.tjbot.features.jshell.JShellEval;
 import org.togetherjava.tjbot.features.mathcommands.TeXCommand;
@@ -109,6 +110,7 @@ public static Collection<Feature> createFeatures(JDA jda, Database database, Con
         features.add(new HelpThreadAutoArchiver(helpSystemHelper));
         features.add(new LeftoverBookmarksCleanupRoutine(bookmarksSystem));
         features.add(new MemberCountDisplayRoutine(config));
+        features.add(new RSSHandlerRoutine(config, database));
 
         // Message receivers
         features.add(new TopHelpersMessageListener(database, config));
diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java
diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/package-info.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/package-info.java
diff --git a/application/src/main/resources/db/V14__Add_Rss_Feed_Cache.sql b/application/src/main/resources/db/V14__Add_Rss_Feed_Cache.sql
