Refactor merge/route.ts; fix category issue (#18)

* WIP

* WIP

* WIP

Author: jmduke

.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: Test
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Install dependencies
+        run: bun install
+
+      - name: Run tests
+        run: bun test

src/app/api/merge/route.test.ts

@@ -93,6 +93,10 @@ describe("GET /api/merge - Multiple feeds with one returning 403", () => {
     });
 
     try {
+      // Suppress console.error output for expected 403 error
+      const originalError = console.error;
+      console.error = () => {};
+
       const baseUrl = new URL("http://localhost:3000/api/merge");
       baseUrl.searchParams.append("url", "http://localhost:9999/feed1.xml");
       baseUrl.searchParams.append("url", "http://localhost:9999/feed2.xml");
@@ -102,6 +106,9 @@ describe("GET /api/merge - Multiple feeds with one returning 403", () => {
       const response = await GET(request);
       const text = await response.text();
 
+      // Restore console.error
+      console.error = originalError;
+
       // Verify response is valid RSS
       expect(response.status).toBe(200);
       expect(response.headers.get("content-type")).toContain(

src/app/api/merge/route.ts

@@ -1,109 +1,13 @@
 import LZString from "lz-string";
-import { encodeContent } from "@/lib/encoding";
 import { type NextRequest, NextResponse } from "next/server";
-import Parser from "rss-parser";
-import { CustomFeed, CustomItem, JSONFeed } from "@/lib/types";
-
-const parser = new Parser({
-  customFields: {
-    item: [
-      ["content:encoded", "content"],
-      ["dc:creator", "creator"],
-    ],
-  },
-});
+import {
+  parseFeedFromUrl,
+  mergeFeeds,
+  generateRSS,
+  generateJSONFeed,
+} from "@/lib/rss";
 
 const GENERATOR = "rssrssrssrss";
-const FEED_TITLE = "Merged Feed";
-
-// Helper functions for JSON Feed detection and parsing
-async function isJSONFeed(url: string): Promise<boolean> {
-  try {
-    const response = await fetch(url, {
-      headers: { Accept: "application/json, application/feed+json, */*" },
-    });
-    const contentType = response.headers.get("content-type") || "";
-
-    if (
-      contentType.includes("application/feed+json") ||
-      contentType.includes("application/json")
-    ) {
-      const text = await response.text();
-      const data = JSON.parse(text);
-      return data.version && data.version.includes("jsonfeed.org");
-    }
-
-    return false;
-  } catch {
-    return false;
-  }
-}
-
-async function parseJSONFeed(url: string): Promise<CustomFeed> {
-  const response = await fetch(url, {
-    headers: { Accept: "application/json, application/feed+json, */*" },
-  });
-  const jsonFeed: JSONFeed = await response.json();
-
-  // Convert JSON Feed items to CustomItem format
-  const items: CustomItem[] = jsonFeed.items.map((item) => ({
-    title: item.title,
-    link: item.url || item.external_url,
-    pubDate: item.date_published,
-    content: item.content_html,
-    contentSnippet: item.content_text || item.summary,
-    creator: item.author?.name,
-    isoDate: item.date_published,
-    guid: item.id,
-    categories: item.tags,
-    sourceFeedTitle: jsonFeed.title,
-    sourceFeedUrl: url,
-  }));
-
-  return {
-    title: jsonFeed.title,
-    description: jsonFeed.description,
-    link: jsonFeed.home_page_url,
-    items,
-  };
-}
-
-// Helper functions for XML generation
-function escapeXml(unsafe: string): string {
-  return unsafe
-    .replace(/&/g, "&amp;")
-    .replace(/</g, "&lt;")
-    .replace(/>/g, "&gt;")
-    .replace(/"/g, "&quot;")
-    .replace(/'/g, "&apos;");
-}
-
-function wrapCDATA(content: string): string {
-  return `<![CDATA[${content}]]>`;
-}
-
-// Helper function to generate JSON Feed output
-function generateJSONFeed(mergedFeed: CustomFeed, requestUrl: string): string {
-  const jsonFeed: JSONFeed = {
-    version: "https://jsonfeed.org/version/1.1",
-    title: mergedFeed.title || FEED_TITLE,
-    description: mergedFeed.description,
-    home_page_url: mergedFeed.link,
-    feed_url: requestUrl,
-    items: mergedFeed.items.map((item) => ({
-      id: item.guid || item.link || crypto.randomUUID(),
-      url: item.link,
-      title: item.title,
-      content_html: item.content,
-      content_text: item.contentSnippet,
-      date_published: item.isoDate || item.pubDate,
-      author: item.creator ? { name: item.creator } : undefined,
-      tags: item.categories,
-    })),
-  };
-
-  return JSON.stringify(jsonFeed, null, 2);
-}
 
 const HEADERS = {
   "Content-Type": "application/rss+xml; charset=utf-8",
@@ -141,8 +45,7 @@ export async function GET(request: NextRequest) {
       }
       return NextResponse.json(
         {
-          error:
-            `${GENERATOR} cannot parse that payload. Are you sure you copied/pasted it correctly?`,
+          error: `${GENERATOR} cannot parse that payload. Are you sure you copied/pasted it correctly?`,
           payload: compressedFeeds,
         },
         { status: 400 }
@@ -163,86 +66,14 @@ export async function GET(request: NextRequest) {
 
   // Fetch and parse all feeds (RSS and JSON) in parallel
   const feedPromises = urls.map(async (url) => {
-    try {
-      // Check if it's a JSON Feed first
-      if (await isJSONFeed(url)) {
-        return { feed: await parseJSONFeed(url), error: null, url };
-      } else {
-        // Fall back to RSS parsing
-        const feed = await parser.parseURL(url);
-        return {
-          feed: {
-            ...feed,
-            items: feed.items.map((item: CustomItem) => ({
-              ...item,
-              sourceFeedTitle: feed.title,
-              sourceFeedUrl: url,
-            })),
-          },
-          error: null,
-          url,
-        };
-      }
-    } catch (error) {
-      console.error(`Error fetching feed from ${url}:`, error);
-      return {
-        feed: null,
-        error: error instanceof Error ? error.message : String(error),
-        url,
-      };
-    }
+    const result = await parseFeedFromUrl(url);
+    return { ...result, url };
   });
 
   const results = await Promise.all(feedPromises);
 
-  // Combine all items into a single array, and collect failed feeds
-  const allItems: CustomItem[] = [];
-  const failedFeeds: Array<{ url: string; error: string }> = [];
-
-  results.forEach(({ feed, error, url }) => {
-    if (error) {
-      failedFeeds.push({ url, error });
-    } else if (feed && feed.items && feed.items.length > 0) {
-      allItems.push(...feed.items);
-    }
-  });
-
-  // Create error items for failed feeds and add them to the beginning
-  const errorItems: CustomItem[] = failedFeeds.map((failed) => ({
-    title: `⚠️ Failed to load feed: ${failed.url}`,
-    link: failed.url,
-    pubDate: new Date().toUTCString(),
-    isoDate: new Date().toISOString(),
-    contentSnippet: `Error: ${failed.error}`,
-    content: `<p>Failed to load this feed:</p><p><code>${escapeXml(failed.url)}</code></p><p>Error: ${escapeXml(failed.error)}</p>`,
-    guid: `error-${failed.url}-${Date.now()}`,
-  }));
-
-  // Sort regular items by date (newest first), keep error items at top
-  allItems.sort((a, b) => {
-    const dateA = a.isoDate ? new Date(a.isoDate) : new Date(a.pubDate || 0);
-    const dateB = b.isoDate ? new Date(b.isoDate) : new Date(b.pubDate || 0);
-    return dateB.getTime() - dateA.getTime();
-  });
-
-  // Combine error items (at the top) with sorted regular items
-  const allItemsWithErrors: CustomItem[] = [...errorItems, ...allItems];
-
-  // Get feed titles from successful feeds for the description
-  const successfulFeedTitles = results
-    .filter(({ feed }) => feed && feed.title)
-    .map(({ feed }) => feed?.title)
-    .filter(Boolean) as string[];
-
-  // Create a merged feed
-  const mergedFeed: CustomFeed = {
-    title: FEED_TITLE,
-    description: `Combined feed from ${successfulFeedTitles.join(", ")}${
-      failedFeeds.length > 0 ? ` (${failedFeeds.length} feed(s) failed to load)` : ""
-    }`,
-    link: request.nextUrl.toString(),
-    items: allItemsWithErrors.slice(0, 100),
-  };
+  // Merge all feeds into a single feed
+  const mergedFeed = mergeFeeds(results, request.nextUrl.toString());
 
   // Check if JSON format is requested
   if (format === "json" || format === "jsonfeed") {
@@ -257,85 +88,7 @@ export async function GET(request: NextRequest) {
   }
 
   // Generate XML using string concatenation (default RSS output)
-  const items = mergedFeed.items
-    .map((item) => {
-      let itemXml = "    <item>\n";
-
-      // Title
-      if (item.title) {
-        itemXml += `      <title>${escapeXml(item.title)}</title>\n`;
-      } else {
-        itemXml += `      <title />\n`;
-      }
-
-      // Link
-      if (item.link) {
-        itemXml += `      <link>${escapeXml(item.link)}</link>\n`;
-      }
-
-      // GUID
-      itemXml += `      <guid>${escapeXml(
-        item.guid || item.link || ""
-      )}</guid>\n`;
-
-      // Publication date
-      if (item.pubDate) {
-        itemXml += `      <pubDate>${escapeXml(item.pubDate)}</pubDate>\n`;
-      } else if (item.isoDate) {
-        itemXml += `      <pubDate>${escapeXml(item.isoDate)}</pubDate>\n`;
-      }
-
-      // Creator (DC namespace)
-      if (item.creator) {
-        itemXml += `      <dc:creator>${wrapCDATA(
-          item.creator
-        )}</dc:creator>\n`;
-      }
-
-      // Content or description
-      if (item.content) {
-        // Note that we don't need to encode this because we're wrapping it in CData.
-        // Per #11, encoding it just removes smart quotes and things of that nature unnecessarily.
-        itemXml += `      <content:encoded>${wrapCDATA(
-          item.content
-        )}</content:encoded>\n`;
-      } else if (item.contentSnippet) {
-        itemXml += `      <description>${escapeXml(
-          encodeContent(item.contentSnippet)
-        )}</description>\n`;
-      }
-
-      // Categories
-      if (item.categories && item.categories.length > 0) {
-        item.categories.forEach((category) => {
-          itemXml += `      <category>${escapeXml(category)}</category>\n`;
-        });
-      }
-
-      // Source information
-      if (item.sourceFeedTitle && item.sourceFeedUrl) {
-        itemXml += `      <source url="${escapeXml(
-          item.sourceFeedUrl
-        )}">${escapeXml(item.sourceFeedTitle)}</source>\n`;
-      }
-
-      itemXml += "    </item>\n";
-      return itemXml;
-    })
-    .join("");
-
-  const xml = `<?xml version="1.0" encoding="UTF-8"?>
-<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:dc="http://purl.org/dc/elements/1.1/">
-  <channel>
-    <title>${escapeXml(mergedFeed.title || FEED_TITLE)}</title>
-    <description>${escapeXml(
-      mergedFeed.description || "Combined feed from multiple sources"
-    )}</description>
-    <link>${escapeXml(mergedFeed.link || request.nextUrl.toString())}</link>
-    <lastBuildDate>${new Date().toUTCString()}</lastBuildDate>
-    <generator>${GENERATOR}</generator>
-${items}  </channel>
-</rss>`;
+  const xml = generateRSS(mergedFeed, request.nextUrl.toString());
 
   // Return the XML response
   return new NextResponse(xml, {