implemented audio/video in note content renderer + streaming API #886

package.json

@@ -39,6 +39,7 @@
     "electron-find": "1.0.6",
     "electron-window-state": "5.0.3",
     "express": "4.17.1",
+    "express-partial-content": "^1.0.2",
     "express-session": "1.17.1",
     "fs-extra": "9.1.0",
     "helmet": "4.4.1",

src/public/app/services/note_content_renderer.js

@@ -39,7 +39,7 @@ async function getRenderedContent(note, options = {}) {
                 .css("max-width", "100%")
         );
     }
-    else if (!options.tooltip && (type === 'file' || type === 'pdf')) {
+    else if (!options.tooltip && ['file', 'pdf', 'audio', 'video']) {
         const $downloadButton = $('<button class="file-download btn btn-primary" type="button">Download</button>');
         const $openButton = $('<button class="file-open btn btn-primary" type="button">Open</button>');
 
@@ -57,6 +57,22 @@ async function getRenderedContent(note, options = {}) {
 
             $content.append($pdfPreview);
         }
+        else if (type === 'audio') {
+            const $audioPreview = $('<audio controls></audio>')
+                .attr("src", openService.getUrlForDownload("api/notes/" + note.noteId + "/open"))
+                .attr("type", note.mime)
+                .css("width", "100%");
+
+            $content.append($audioPreview);
+        }
+        else if (type === 'video') {
+            const $videoPreview = $('<video controls></video>')
+                .attr("src", openService.getUrlForDownload("api/notes/" + note.noteId + "/open"))
+                .attr("type", note.mime)
+                .css("width", "100%");
+
+            $content.append($videoPreview);
+        }
 
         $content.append(
             $('<div style="display: flex; justify-content: space-evenly; margin-top: 5px;">')
@@ -110,6 +126,10 @@ function getRenderingType(note) {
 
     if (type === 'file' && note.mime === 'application/pdf') {
         type = 'pdf';
+    } else if (type === 'file' && note.mime.startsWith('audio/')) {
+        type = 'audio';
+    } else if (type === 'file' && note.mime.startsWith('video/')) {
+        type = 'video';
     }
 
     if (note.isProtected) {

src/public/stylesheets/style.css

@@ -757,6 +757,10 @@ a.external:not(.no-arrow):after, a[href^="http://"]:not(.no-arrow):after, a[href
     height: 50em; /* PDF is rendered in iframe and it's not possible to put full height so at least a large height */
 }
 
+.include-note-wrapper {
+    width: 100%;
+}
+
 .alert-warning, .alert-info {
     color: var(--main-text-color) !important;
     background-color: var(--accented-background-color) !important;

src/routes/api/files.js

@@ -6,6 +6,7 @@ const utils = require('../../services/utils');
 const noteRevisionService = require('../../services/note_revisions');
 const tmp = require('tmp');
 const fs = require('fs');
+const { Readable } = require('stream');
 
 function updateFile(req) {
     const {noteId} = req.params;
@@ -73,6 +74,38 @@ function openFile(req, res) {
     return downloadNoteFile(noteId, res, false);
 }
 
+function fileContentProvider(req) {
+    // Read file name from route params.
+    const note = repository.getNote(req.params.noteId);
+    const fileName = getFilename(note);
+    let content = note.getContent();
+
+    if (typeof content === "string") {
+       content = Buffer.from(content, 'utf8');
+    }
+
+    const totalSize = content.byteLength;
+    const mimeType = note.mime;
+
+    const getStream = range => {
+        if (!range) {
+            // Request if for complete content.
+            return Readable.from(content);
+        }
+        // Partial content request.
+        const { start, end } = range;
+
+        return Readable.from(content.slice(start, end + 1));
+    }
+
+    return {
+        fileName,
+        totalSize,
+        mimeType,
+        getStream
+    };
+}
+
 function saveToTmpDir(req) {
     const noteId = req.params.noteId;
 
@@ -95,6 +128,7 @@ function saveToTmpDir(req) {
 module.exports = {
     updateFile,
     openFile,
+    fileContentProvider,
     downloadFile,
     downloadNoteFile,
     saveToTmpDir

src/routes/routes.js

@@ -48,6 +48,8 @@ const sql = require('../services/sql');
 const protectedSessionService = require('../services/protected_session');
 const entityChangesService = require('../services/entity_changes.js');
 const csurf = require('csurf');
+const {createPartialContentHandler} = require("express-partial-content");
+
 
 const csrfMiddleware = csurf({
     cookie: true,
@@ -175,7 +177,10 @@ function register(app) {
     route(PUT, '/api/notes/:noteId/file', [auth.checkApiAuthOrElectron, uploadMiddleware, csrfMiddleware],
         filesRoute.updateFile, apiResultHandler);
 
-    route(GET, '/api/notes/:noteId/open', [auth.checkApiAuthOrElectron], filesRoute.openFile);
+    route(GET, '/api/notes/:noteId/open', [auth.checkApiAuthOrElectron],
+        createPartialContentHandler(filesRoute.fileContentProvider, {
+            debug: (string, extra) => { console.log(string, extra); }
+        }));
     route(GET, '/api/notes/:noteId/download', [auth.checkApiAuthOrElectron], filesRoute.downloadFile);
     // this "hacky" path is used for easier referencing of CSS resources
     route(GET, '/api/notes/download/:noteId', [auth.checkApiAuthOrElectron], filesRoute.downloadFile);