Merge pull request #161 from codelitdev/rajat1saxena/issue159

Large file uploads via TUS

Author: rajat1saxena

.changeset/good-books-double.md

@@ -0,0 +1,5 @@
+---
+"medialit": minor
+---
+
+API key and signature are passed via header instead of request body

.github/workflows/codeql.yml

@@ -0,0 +1,41 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: "51 18 * * 2"
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ javascript ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          queries: +security-and-quality
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:${{ matrix.language }}"

apps/api/__tests__/media/handlers.test.ts

@@ -80,6 +80,7 @@ describe("Media handlers", () => {
             },
             body: {},
             query: {},
+            headers: {},
         };
 
         const res = {
@@ -97,6 +98,7 @@ describe("Media handlers", () => {
         );
 
         const response = await uploadMedia(req, res, () => {});
+        console.log("Response", response);
         assert.strictEqual(response.code, 200);
     });
 });

apps/api/__tests__/media/storage-middleware.test.ts

@@ -7,6 +7,7 @@ import {
     maxStorageAllowedNotSubscribed,
     maxStorageAllowedSubscribed,
 } from "../../src/config/constants";
+import { NOT_ENOUGH_STORAGE } from "../../src/config/strings";
 
 describe("storageValidation middleware", () => {
     afterEach(() => {
@@ -108,11 +109,8 @@ describe("storageValidation middleware", () => {
         };
 
         const response = await storageValidation(req, res, next);
-        assert.strictEqual(response.code, 400);
-        assert.strictEqual(
-            response.data.error,
-            "You do not have enough storage space in your account to upload this file",
-        );
+        assert.strictEqual(response.code, 403);
+        assert.strictEqual(response.data.error, NOT_ENOUGH_STORAGE);
         assert.strictEqual(nextCalled, false);
     });
 
@@ -145,11 +143,8 @@ describe("storageValidation middleware", () => {
         };
 
         const response = await storageValidation(req, res, next);
-        assert.strictEqual(response.code, 400);
-        assert.strictEqual(
-            response.data.error,
-            "You do not have enough storage space in your account to upload this file",
-        );
+        assert.strictEqual(response.code, 403);
+        assert.strictEqual(response.data.error, NOT_ENOUGH_STORAGE);
         assert.strictEqual(nextCalled, false);
     });
 

apps/api/package.json

@@ -38,10 +38,12 @@
     "@medialit/models": "workspace:*",
     "@medialit/thumbnail": "workspace:*",
     "@medialit/utils": "workspace:^0.1.0",
+    "@tus/file-store": "^2.0.0",
+    "@tus/server": "^2.3.0",
     "aws-sdk": "^2.1692.0",
     "cors": "^2.8.5",
     "dotenv": "^16.4.7",
-    "express": "^4.18.2",
+    "express": "^4.2.0",
     "express-fileupload": "^1.3.1",
     "joi": "^17.6.0",
     "mongoose": "^8.0.1",

apps/api/src/apikey/middleware.ts

@@ -1,20 +1,18 @@
-import {
-    BAD_REQUEST,
-    SUBSCRIPTION_NOT_VALID,
-    UNAUTHORISED,
-} from "../config/strings";
+import { BAD_REQUEST, UNAUTHORISED } from "../config/strings";
 import { getApiKeyUsingKeyId } from "./queries";
 import { getUser } from "../user/queries";
 import { Apikey } from "@medialit/models";
+import logger from "../services/log";
 
 export default async function apikey(
     req: any,
     res: any,
     next: (...args: any[]) => void,
 ) {
-    const reqKey = req.body.apikey;
+    const reqKey = req.body?.apikey || req.headers["x-medialit-apikey"];
 
     if (!reqKey) {
+        logger.error({}, "API key is missing");
         return res.status(400).json({ error: BAD_REQUEST });
     }
 
@@ -23,13 +21,6 @@ export default async function apikey(
         return res.status(401).json({ error: UNAUTHORISED });
     }
 
-    // const isSubscriptionValid = await validateSubscription(
-    //     apiKey!.userId.toString(),
-    // );
-    // if (!isSubscriptionValid) {
-    //     return res.status(403).json({ error: SUBSCRIPTION_NOT_VALID });
-    // }
-
     req.user = await getUser(apiKey!.userId.toString());
     req.apikey = apiKey.key;
 

apps/api/src/config/constants.ts

@@ -6,19 +6,19 @@ export const tempFileDirForUploads = process.env.TEMP_FILE_DIR_FOR_UPLOADS;
 export const maxFileUploadSizeSubscribed = process.env
     .MAX_UPLOAD_SIZE_SUBSCRIBED
     ? +process.env.MAX_UPLOAD_SIZE_SUBSCRIBED
-    : 2147483648;
+    : 2147483648; // 2GB
 export const maxFileUploadSizeNotSubscribed = process.env
     .MAX_UPLOAD_SIZE_NOT_SUBSCRIBED
     ? +process.env.MAX_UPLOAD_SIZE_NOT_SUBSCRIBED
-    : 52428800;
+    : 52428800; // 50MB
 export const maxStorageAllowedSubscribed = process.env
     .MAX_STORAGE_ALLOWED_SUBSCRIBED
     ? +process.env.MAX_STORAGE_ALLOWED_SUBSCRIBED
-    : 107374182400;
+    : 107374182400; // 100GB
 export const maxStorageAllowedNotSubscribed = process.env
     .MAX_STORAGE_ALLOWED_NOT_SUBSCRIBED
     ? +process.env.MAX_STORAGE_ALLOWED_NOT_SUBSCRIBED
-    : 1073741824;
+    : 1073741824; // 1GB
 export const PRESIGNED_URL_VALIDITY_MINUTES = 5;
 export const PRESIGNED_URL_LENGTH = 100;
 export const MEDIA_ID_LENGTH = 40;
@@ -62,3 +62,8 @@ export const CDN_MAX_AGE = process.env.CDN_MAX_AGE
 
 export const ENDPOINT = USE_CLOUDFRONT ? CLOUDFRONT_ENDPOINT : S3_ENDPOINT;
 export const HOSTNAME_OVERRIDE = process.env.HOSTNAME_OVERRIDE || ""; // Useful for hosting via Docker
+
+// Tus upload config
+export const TUS_UPLOAD_EXPIRATION_HOURS = parseInt(
+    process.env.TUS_UPLOAD_EXPIRATION_HOURS || "48",
+);

apps/api/src/config/strings.ts

@@ -6,3 +6,5 @@ export const FILE_IS_REQUIRED = "File is required";
 export const FILE_SIZE_EXCEEDED = "File size exceeded";
 export const NOT_FOUND = "Not found";
 export const PRESIGNED_URL_INVALID = "The link is invalid";
+export const NOT_ENOUGH_STORAGE =
+    "Not enough storage space in your account to upload this file";

apps/api/src/index.ts

@@ -5,13 +5,15 @@ import express from "express";
 import connectToDatabase from "./config/db";
 import passport from "passport";
 import mediaRoutes from "./media/routes";
-import presignedUrlRoutes from "./presigning/routes";
+import signatureRoutes from "./signature/routes";
 import mediaSettingsRoutes from "./media-settings/routes";
+import tusRoutes from "./tus/routes";
 import logger from "./services/log";
 import { createUser, findByEmail } from "./user/queries";
-import { Apikey, Constants, User } from "@medialit/models";
+import { Apikey, User } from "@medialit/models";
 import { createApiKey } from "./apikey/queries";
 import { spawn } from "child_process";
+import { Cleanup } from "./tus/cleanup";
 
 connectToDatabase();
 const app = express();
@@ -28,7 +30,8 @@ app.get("/health", (req, res) => {
 });
 
 app.use("/settings/media", mediaSettingsRoutes(passport));
-app.use("/media/presigned", presignedUrlRoutes);
+app.use("/media/signature", signatureRoutes);
+app.use("/media", tusRoutes);
 app.use("/media", mediaRoutes);
 
 const port = process.env.PORT || 80;
@@ -41,6 +44,14 @@ checkDependencies().then(() => {
     app.listen(port, () => {
         logger.info(`Medialit server running at ${port}`);
     });
+
+    // Setup background cleanup job for expired tus uploads
+    setInterval(
+        async () => {
+            await Cleanup();
+        },
+        1000 * 60 * 60, // 1 hours
+    );
 });
 
 async function checkDependencies() {

apps/api/src/media/handlers.ts

@@ -15,7 +15,8 @@ import logger from "../services/log";
 import { Request } from "express";
 import mediaService from "./service";
 import { getMediaCount as getCount, getTotalSpace } from "./queries";
-import { Constants, getSubscriptionStatus } from "@medialit/models";
+import { getSubscriptionStatus } from "@medialit/models";
+import { getSignatureFromReq } from "../signature/utils";
 
 function validateUploadOptions(req: Request): Joi.ValidationResult {
     const uploadSchema = Joi.object({
@@ -69,7 +70,7 @@ export async function uploadMedia(
             access,
             caption,
             group,
-            signature: req.query.signature,
+            signature: getSignatureFromReq(req),
         });
 
         const media = await mediaService.getMediaDetails({