Release v2.4.2

src/web/routes/data-sources/api-sources/add.js

@@ -1,41 +1,8 @@
-// API source add page renderer and form defaults.
+// API source add page renderer.
 
 const { renderView } = require('../../../view');
+const { buildApiSourceFormViewModel } = require('./form-view-model');
 
-function buildDefaultApiSource() {
-  return {
-    id: null,
-    name: '',
-    apiUrl: '',
-    authMethod: 'none',
-    authUsername: '',
-    authPassword: '',
-    authBearerToken: '',
-    authHeaderName: 'X-API-Key',
-    authHeaderValue: '',
-    itemsPath: '',
-    updateIntervalValue: 60,
-    updateIntervalUnit: 'minutes',
-    lastPulledAt: null,
-    lastPullError: '',
-    lastResponseStatus: null,
-    lastResponseContentType: '',
-    lastResponseJson: ''
-  };
-}
-
-module.exports = function renderApiSourceFormPage(apiSource, mode, message, currentUser) {
-  const isEdit = mode === 'edit';
-  const viewApiSource = Object.assign(buildDefaultApiSource(), apiSource || {});
-
-  return renderView(isEdit ? 'data-sources/api-sources/edit' : 'data-sources/api-sources/add', {
-    title: isEdit ? 'Edit API source' : 'Add API source',
-    active: 'api-sources',
-    message: message,
-    currentUser: currentUser || null,
-    apiSource: viewApiSource,
-    inUse: Boolean(viewApiSource.inUse),
-    lastResponseJson: viewApiSource.lastResponseJson || '',
-    lastPullError: viewApiSource.lastPullError || ''
-  });
+module.exports = function renderApiSourceAddPage(apiSource, message, currentUser, options) {
+  return renderView('data-sources/api-sources/form', buildApiSourceFormViewModel(apiSource, message, currentUser, options, false));
 };

src/web/routes/data-sources/api-sources/edit.js

@@ -1,6 +1,7 @@
-// API source edit page renderer that reuses the add form.
+// API source edit page renderer.
 
-const renderApiSourceFormPage = require('./add');
+const { renderView } = require('../../../view');
+const { buildApiSourceFormViewModel } = require('./form-view-model');
 
 module.exports = function renderApiSourceEditPage(apiSource, data, message, currentUser) {
   const viewData = Object.assign({
@@ -8,7 +9,7 @@ module.exports = function renderApiSourceEditPage(apiSource, data, message, curr
     lastPullError: ''
   }, data || {});
 
-  return renderApiSourceFormPage(Object.assign({}, apiSource, {
+  return renderView('data-sources/api-sources/form', buildApiSourceFormViewModel(Object.assign({}, apiSource, {
     lastResponseJson: viewData.lastResponseJson,
     lastPullError: viewData.lastPullError,
     authMethod: apiSource.auth_method || 'none',
@@ -18,5 +19,5 @@ module.exports = function renderApiSourceEditPage(apiSource, data, message, curr
     authHeaderName: apiSource.auth_header_name || 'X-API-Key',
     authHeaderValue: apiSource.auth_header_value || '',
     itemsPath: apiSource.items_path || ''
-  }), 'edit', message, currentUser);
+  }), message, currentUser, viewData, true));
 };

src/web/routes/data-sources/api-sources/form-view-model.js

@@ -0,0 +1,50 @@
+// Shared API source form view-model builder.
+
+function buildDefaultApiSource() {
+  return {
+    id: null,
+    name: '',
+    apiUrl: '',
+    authMethod: 'none',
+    authUsername: '',
+    authPassword: '',
+    authBearerToken: '',
+    authHeaderName: 'X-API-Key',
+    authHeaderValue: '',
+    itemsPath: '',
+    updateIntervalValue: 60,
+    updateIntervalUnit: 'minutes',
+    lastPulledAt: null,
+    lastPullError: '',
+    lastResponseStatus: null,
+    lastResponseContentType: '',
+    lastResponseJson: ''
+  };
+}
+
+function buildApiSourceFormViewModel(apiSource, message, currentUser, options, isEdit) {
+  const viewApiSource = Object.assign(buildDefaultApiSource(), apiSource || {});
+  const viewOptions = options || {};
+
+  return {
+    title: isEdit ? 'Edit API source' : 'Add API source',
+    active: 'api-sources',
+    message: message,
+    currentUser: currentUser || null,
+    apiSource: viewApiSource,
+    inUse: Boolean(viewApiSource.inUse),
+    lastResponseJson: viewOptions.lastResponseJson || viewApiSource.lastResponseJson || '',
+    lastPullError: viewOptions.lastPullError || viewApiSource.lastPullError || '',
+    isEdit: Boolean(isEdit),
+    showSaveSecondaryActions: Boolean(isEdit),
+    deleteDisabled: !isEdit || Boolean(viewApiSource.inUse),
+    formAction: isEdit && viewApiSource.id ? '/data-sources/api-sources/' + viewApiSource.id : '/data-sources/api-sources',
+    formAttrs: isEdit && viewApiSource.id ? 'data-async-save data-async-save-refresh-target="#api-source-response-panel" data-async-save-refresh-state-url="/data-sources/api-sources/' + viewApiSource.id + '/state" data-async-save-close-url="/data-sources/api-sources" data-async-save-new-url="/data-sources/api-sources/new"' : 'data-async-save data-async-save-close-url="/data-sources/api-sources" data-async-save-new-redirect="response-url" data-async-save-new-url="/data-sources/api-sources/new"',
+    cancelUrl: '/data-sources/api-sources',
+    deleteUrl: isEdit && viewApiSource.id ? '/data-sources/api-sources/' + viewApiSource.id + '/delete' : ''
+  };
+}
+
+module.exports = {
+  buildApiSourceFormViewModel: buildApiSourceFormViewModel
+};

src/web/routes/data-sources/api-sources/routes.js

@@ -0,0 +1,326 @@
+// API source route registration.
+
+const { buildPagination } = require('../../../lib/pagination');
+const renderApiSourcesPage = require('./list');
+const renderApiSourceAddPage = require('./add');
+const renderApiSourceEditPage = require('./edit');
+
+function toIsoTimestamp(value) {
+  if (!value) {
+    return '';
+  }
+
+  const date = new Date(value);
+  return Number.isNaN(date.getTime()) ? '' : date.toISOString();
+}
+
+async function getDataSourceUsageMaps(pool, common) {
+  const [slides] = await pool.query('SELECT content_json FROM c_slides WHERE content_json IS NOT NULL');
+  const apiSourceIds = new Set();
+
+  for (let index = 0; index < slides.length; index += 1) {
+    const content = typeof common.parseJsonSafe === 'function' ? common.parseJsonSafe(slides[index].content_json) : null;
+    if (!content || typeof content !== 'object') {
+      continue;
+    }
+
+    (function walk(value) {
+      if (!value || typeof value !== 'object') {
+        return;
+      }
+
+      if (Array.isArray(value)) {
+        value.forEach(walk);
+        return;
+      }
+
+      if (Object.prototype.hasOwnProperty.call(value, 'source_id')) {
+        const sourceId = Number(value.source_id);
+        if (Number.isFinite(sourceId)) {
+          apiSourceIds.add(sourceId);
+        }
+      }
+
+      Object.keys(value).forEach(function (key) {
+        walk(value[key]);
+      });
+    })(content);
+  }
+
+  return apiSourceIds;
+}
+
+async function isApiSourceInUse(pool, common, apiSourceId) {
+  const usageIds = await getDataSourceUsageMaps(pool, common);
+  return usageIds.has(Number(apiSourceId));
+}
+
+async function sendRefreshTaskState(req, res, dataSourceTasks, sourceId) {
+  const taskId = Number(req.query.refresh_task_id);
+  if (!Number.isFinite(taskId) || taskId <= 0) {
+    return res.status(400).json({ error: 'Missing refresh task id.' });
+  }
+
+  const task = await dataSourceTasks.getTaskStatusById(taskId);
+  const expectedKey = 'api-source-refresh:' + Number(sourceId);
+  if (!task || task.key !== expectedKey) {
+    return res.status(404).json({ error: 'Refresh task not found.' });
+  }
+
+  res.json(task);
+}
+
+module.exports = function registerApiSourceRoutes(app, deps) {
+  const pool = deps.pool;
+  const common = deps.common;
+  const pages = deps.pages;
+  const backgroundTaskQueue = deps.backgroundTaskQueue;
+  const dataSourceTasks = deps.dataSourceTasks;
+  const getAuditUserId = deps.getAuditUserId;
+  const redirectAfterSave = deps.redirectAfterSave;
+  const formatDashboardDate = deps.formatDashboardDate || function (value) {
+    return value ? String(value) : '';
+  };
+  const requirePermission = deps.requirePermission;
+
+  const LIST_PAGE_SIZE = 25;
+
+  app.get('/data-sources/api-sources', requirePermission('api-sources.read'), async function (req, res, next) {
+    try {
+      const page = Math.max(1, Math.floor(Number(req.query.page) || 1));
+      const search = common.getSearchQuery(req);
+      const sort = common.getSortQuery(req);
+      const direction = common.getSortDirectionQuery(req);
+      const data = await common.fetchApiSourcesPage(pool, page, LIST_PAGE_SIZE, search, sort, direction);
+      const usageIds = await getDataSourceUsageMaps(pool, common);
+      const apiSources = (data.apiSources || []).map(function (apiSource) {
+        return Object.assign({}, apiSource, {
+          authMethod: apiSource.auth_method || 'none',
+          authUsername: apiSource.auth_username || '',
+          authPassword: apiSource.auth_password || '',
+          authBearerToken: apiSource.auth_bearer_token || '',
+          authHeaderName: apiSource.auth_header_name || 'X-API-Key',
+          authHeaderValue: apiSource.auth_header_value || '',
+          itemsPath: apiSource.items_path || '',
+          intervalLabel: apiSource.update_interval_unit === 'seconds'
+            ? (Math.max(1, Number(apiSource.update_interval_value) || 0) === 1 ? 'Every second' : `Every ${Math.max(1, Number(apiSource.update_interval_value) || 0)} seconds`)
+            : (Math.max(1, Number(apiSource.update_interval_value) || 0) === 1 ? 'Every minute' : `Every ${Math.max(1, Number(apiSource.update_interval_value) || 0)} minutes`),
+          lastPullLabel: apiSource.last_pulled_at ? formatDashboardDate(apiSource.last_pulled_at) : 'Never',
+          lastPulledAtValue: apiSource.last_pulled_at ? new Date(apiSource.last_pulled_at).toISOString() : '',
+          inUse: usageIds.has(Number(apiSource.id))
+        });
+      });
+
+      res.send(renderApiSourcesPage({
+        apiSources: apiSources,
+        pagination: buildPagination(data.totalItems, data.currentPage, 'page', { search: search, sort: sort, direction: direction }, LIST_PAGE_SIZE, 'API sources', 'API source pages')
+      }, req.query.message ? String(req.query.message) : '', req.currentUser, formatDashboardDate));
+    } catch (error) {
+      next(error);
+    }
+  });
+
+  app.get('/data-sources/api-sources/new', requirePermission('api-sources.create'), function (req, res) {
+    res.send(renderApiSourceAddPage(null, req.query.message ? String(req.query.message) : '', req.currentUser));
+  });
+
+  app.get('/data-sources/api-sources/:id/edit', requirePermission('api-sources.update'), async function (req, res, next) {
+    try {
+      const apiSource = await common.fetchApiSourceById(pool, Number(req.params.id));
+      if (!apiSource) {
+        return res.status(404).send('API source not found');
+      }
+
+      const [slides] = await pool.query('SELECT content_json FROM c_slides WHERE content_json IS NOT NULL');
+      const inUse = (slides || []).some(function (row) {
+        const content = typeof common.parseJsonSafe === 'function' ? common.parseJsonSafe(row.content_json) : null;
+        if (!content || typeof content !== 'object') {
+          return false;
+        }
+
+        const stack = [content];
+        while (stack.length) {
+          const value = stack.pop();
+          if (!value || typeof value !== 'object') {
+            continue;
+          }
+          if (Array.isArray(value)) {
+            value.forEach(function (item) { stack.push(item); });
+            continue;
+          }
+          if (Object.prototype.hasOwnProperty.call(value, 'source_id') && Number(value.source_id) === Number(apiSource.id)) {
+            return true;
+          }
+          Object.keys(value).forEach(function (key) {
+            stack.push(value[key]);
+          });
+        }
+        return false;
+      });
+
+      res.send(renderApiSourceEditPage(Object.assign({}, apiSource, {
+        apiUrl: apiSource.api_url,
+        authMethod: apiSource.auth_method || 'none',
+        authUsername: apiSource.auth_username || '',
+        authPassword: apiSource.auth_password || '',
+        authBearerToken: apiSource.auth_bearer_token || '',
+        authHeaderName: apiSource.auth_header_name || 'X-API-Key',
+        authHeaderValue: apiSource.auth_header_value || '',
+        itemsPath: apiSource.items_path || '',
+        updateIntervalValue: apiSource.update_interval_value,
+        updateIntervalUnit: apiSource.update_interval_unit || 'minutes',
+        lastPulledAtValue: toIsoTimestamp(apiSource.last_pulled_at),
+        lastPulledAtLabel: apiSource.last_pulled_at ? String(apiSource.last_pulled_at) : '',
+        lastPullError: apiSource.last_pull_error || '',
+        lastResponseStatus: apiSource.last_response_status,
+        lastResponseContentType: apiSource.last_response_content_type,
+        lastResponseJson: apiSource.last_response_json || '',
+        inUse: inUse
+      }), {
+        lastResponseJson: apiSource.last_response_json || '',
+        lastPullError: apiSource.last_pull_error || ''
+      }, req.query.message ? String(req.query.message) : '', req.currentUser));
+    } catch (error) {
+      next(error);
+    }
+  });
+
+  app.get('/data-sources/api-sources/:id/state', requirePermission('api-sources.update'), function (req, res, next) {
+    sendRefreshTaskState(req, res, dataSourceTasks, Number(req.params.id)).catch(next);
+  });
+
+  app.post('/data-sources/api-sources', requirePermission('api-sources.create'), async function (req, res, next) {
+    const connection = await pool.getConnection();
+    try {
+      const payload = common.buildApiSourcePayload(req, null);
+      if (await common.fetchDuplicateName(pool, 'i_api_sources', payload.name)) {
+        return res.redirect('/data-sources/api-sources/new?message=' + encodeURIComponent('An API source with that name already exists.'));
+      }
+
+      const actorId = getAuditUserId(req);
+      await connection.beginTransaction();
+      const [result] = await connection.query(
+        'INSERT INTO i_api_sources (name, api_url, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, items_path, update_interval_value, update_interval_unit, last_pulled_at, last_pull_error, last_response_status, last_response_content_type, last_response_json, created_by, modified_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        [payload.name, payload.apiUrl, payload.authMethod, payload.authUsername || null, payload.authPassword || null, payload.authBearerToken || null, payload.authHeaderName || null, payload.authHeaderValue || null, payload.itemsPath || null, payload.updateIntervalValue, payload.updateIntervalUnit, null, null, null, null, null, actorId, actorId]
+      );
+      await connection.commit();
+      dataSourceTasks.registerRecurringRefresh('api-source', result.insertId, payload.name, payload.updateIntervalValue, payload.updateIntervalUnit, function () {
+        return dataSourceTasks.refreshApiSourceInBackground(result.insertId, actorId);
+      });
+      const refreshTask = await backgroundTaskQueue.enqueueTask({
+        key: 'api-source-refresh:' + result.insertId,
+        title: 'API source refresh',
+        category: 'data-source',
+        taskType: 'data-source-refresh',
+        payload: {
+          sourceType: 'api-source',
+          sourceId: result.insertId,
+          sourceName: payload.name,
+          actorId: actorId
+        },
+        metadata: {
+          sourceType: 'api-source',
+          sourceId: result.insertId,
+          sourceName: payload.name
+        }
+      });
+      const message = 'API source created. Refresh is running in the background.';
+      res.redirect('/data-sources/api-sources/' + result.insertId + '/edit?message=' + encodeURIComponent(message) + '&refresh_task_id=' + encodeURIComponent(refreshTask && refreshTask.id ? refreshTask.id : ''));
+    } catch (error) {
+      try {
+        await connection.rollback();
+      } catch (_rollbackError) {
+        // Ignore rollback failures and surface the original error.
+      }
+      next(error);
+    } finally {
+      connection.release();
+    }
+  });
+
+  app.post('/data-sources/api-sources/:id', requirePermission('api-sources.update'), async function (req, res, next) {
+    const connection = await pool.getConnection();
+    try {
+      const apiSource = await common.fetchApiSourceById(pool, Number(req.params.id));
+      if (!apiSource) {
+        return res.status(404).send('API source not found');
+      }
+
+      const payload = common.buildApiSourcePayload(req, apiSource);
+      if (await common.fetchDuplicateName(pool, 'i_api_sources', payload.name, apiSource.id)) {
+        return res.redirect('/data-sources/api-sources/' + apiSource.id + '/edit?message=' + encodeURIComponent('An API source with that name already exists.'));
+      }
+      const actorId = getAuditUserId(req);
+      await connection.beginTransaction();
+      await connection.query(
+        'UPDATE i_api_sources SET name = ?, api_url = ?, auth_method = ?, auth_username = ?, auth_password = ?, auth_bearer_token = ?, auth_header_name = ?, auth_header_value = ?, items_path = ?, update_interval_value = ?, update_interval_unit = ?, last_pulled_at = ?, last_pull_error = ?, last_response_status = ?, last_response_content_type = ?, last_response_json = ?, modified_by = ? WHERE id = ?',
+        [payload.name, payload.apiUrl, payload.authMethod, payload.authUsername || null, payload.authPassword || null, payload.authBearerToken || null, payload.authHeaderName || null, payload.authHeaderValue || null, payload.itemsPath || null, payload.updateIntervalValue, payload.updateIntervalUnit, null, null, null, null, null, actorId, apiSource.id]
+      );
+      await connection.commit();
+      dataSourceTasks.registerRecurringRefresh('api-source', apiSource.id, payload.name, payload.updateIntervalValue, payload.updateIntervalUnit, function () {
+        return dataSourceTasks.refreshApiSourceInBackground(apiSource.id, actorId);
+      });
+      const message = 'API source updated. Refresh is running in the background.';
+      const refreshTask = await backgroundTaskQueue.enqueueTask({
+        key: 'api-source-refresh:' + apiSource.id,
+        title: 'API source refresh',
+        category: 'data-source',
+        taskType: 'data-source-refresh',
+        payload: {
+          sourceType: 'api-source',
+          sourceId: apiSource.id,
+          sourceName: payload.name,
+          actorId: actorId
+        },
+        metadata: {
+          sourceType: 'api-source',
+          sourceId: apiSource.id,
+          sourceName: payload.name
+        }
+      });
+      redirectAfterSave(req, res, '/data-sources/api-sources/' + apiSource.id + '/edit?refresh_task_id=' + encodeURIComponent(refreshTask && refreshTask.id ? refreshTask.id : ''), {
+        closeUrl: '/data-sources/api-sources',
+        newUrl: '/data-sources/api-sources/new',
+        message: message
+      });
+    } catch (error) {
+      try {
+        await connection.rollback();
+      } catch (_rollbackError) {
+        // Ignore rollback failures and surface the original error.
+      }
+      next(error);
+    } finally {
+      connection.release();
+    }
+  });
+
+  app.post('/data-sources/api-sources/:id/delete', requirePermission('api-sources.delete'), async function (req, res, next) {
+    try {
+      const apiSource = await common.fetchApiSourceById(pool, Number(req.params.id));
+      if (!apiSource) {
+        return res.status(404).send('API source not found');
+      }
+
+      if (await isApiSourceInUse(pool, common, apiSource.id)) {
+        return res.redirect('/data-sources/api-sources?message=' + encodeURIComponent('This API source is still used by one or more slides.'));
+      }
+
+      const connection = await pool.getConnection();
+      try {
+        await connection.beginTransaction();
+        await connection.query('DELETE FROM i_api_sources WHERE id = ?', [apiSource.id]);
+        await connection.commit();
+        dataSourceTasks.removeRecurringRefresh('api-source', apiSource.id);
+      } catch (error) {
+        await connection.rollback();
+        throw error;
+      } finally {
+        connection.release();
+      }
+      res.redirect('/data-sources/api-sources?message=' + encodeURIComponent('API source deleted.'));
+    } catch (error) {
+      next(error);
+    }
+  });
+};