From 8ce822809147c7d6af564c4924856cb303e4861f Mon Sep 17 00:00:00 2001 From: Mark Rapson Date: Wed, 21 Oct 2026 00:39:51 +0100 Subject: [PATCH] Release 2.16.1 --- CHANGELOG.md | 10 + build/package.player.json | 2 +- build/package.web.json | 2 +- docs/technical/schema.md | 17 +- docs/user/guide-data-sources.md | 12 +- package-lock.json | 4 +- package.json | 2 +- src/common.js | 1 - src/data/api-source-groups.js | 124 +++++ src/data/api-sources.js | 233 +++++++-- src/data/index.js | 3 +- src/db/index.js | 28 +- src/db/migrations.js | 33 ++ .../tasks-scheduled/data-source-refresh.js | 2 +- .../public/js/data-sources/api-source-form.js | 199 ++++++- .../data-sources/api-sources/duplicate.js | 29 +- .../routes/data-sources/api-sources/edit.js | 27 +- .../api-sources/form-view-model.js | 29 +- .../routes/data-sources/api-sources/routes.js | 488 +++++------------- .../views/data-sources/api-sources/form.hbs | 189 +++---- .../views/data-sources/api-sources/list.hbs | 25 +- test/api-source-groups.test.js | 119 +++++ test/api-source-request.test.js | 306 ++++++++++- test/api-sources-duplicate.test.js | 30 ++ test/schema-update-log.test.js | 6 +- 25 files changed, 1307 insertions(+), 613 deletions(-) create mode 100644 src/data/api-source-groups.js create mode 100644 test/api-source-groups.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e0287d..1741f3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. +## 2.16.1 - 2026-10-21 + +### Added + +- Added API source groups with shared authentication and cached tokens, plus support for unauthenticated public APIs. Existing request IDs and slide bindings are preserved. +- Added individually named request cards with independent URLs, enabled states, refresh schedules, and full-width response trees with Expand all/Collapse all controls. The Connection card provides source actions and Add request, with confirmation before removing a request. +- Added group-level authentication testing and per-request diagnostics using unsaved settings, with dismissible results below the tested section. Diagnostic reports redact credentials and populated tokens while preserving OAuth metadata and null or empty token fields. + ## 2.16.0 - 2026-10-20 ### Added @@ -10,11 +18,13 @@ All notable changes to this project will be documented in this file. - Added slide-specific region geometry overrides that flow through previews, playback, and thumbnails without changing sibling slides. Users with template-edit permission can apply the layout to this slide or save it to the shared template; shared saves confirm the affected-slide count, preserve region IDs and animations, clear only the current slide's geometry overrides, and reject stale templates. - Added a separate slide animation editor with the template's presets and advanced timing controls, plus intro/outro/attention previews on the actual slide canvas. Changes can be applied to this slide or saved to the shared template; per-region and all-region reset actions are available, and shared saves preserve slide content and sibling overrides. - Added consistent +/- sizing controls to HTML region editors, matching WYSIWYG regions, with the sandboxed iframe note alongside the controls. +- Added an API source connection test using unsaved form values, with separate authentication, data request, and JSON/items checks, sanitised response previews, available field paths, and a copyable diagnostic report in a dismissible results card without changing stored responses. ### Changed - Updated browser error pages to use a centered AdminLTE-style layout, with blue 404/other 4xx statuses, warning 403, and red 5xx statuses. - Slide animation overrides are now normalized as JSON objects in slide content, while legacy string values remain supported. +- API source request failures now include the failing stage, sanitised endpoint and provider error, status-specific guidance, and a 15-second timeout per request. ### Fixed diff --git a/build/package.player.json b/build/package.player.json index 714c974..32b0be1 100644 --- a/build/package.player.json +++ b/build/package.player.json @@ -1,6 +1,6 @@ { "name": "pulse-signage-player", - "version": "2.16.0", + "version": "2.16.1", "private": false, "description": "Pulse Signage player application bundle", "engines": { diff --git a/build/package.web.json b/build/package.web.json index 3e6b10b..5552e5f 100644 --- a/build/package.web.json +++ b/build/package.web.json @@ -1,6 +1,6 @@ { "name": "pulse-signage-web", - "version": "2.16.0", + "version": "2.16.1", "private": false, "description": "Pulse Signage web and bridge application bundle", "engines": { diff --git a/docs/technical/schema.md b/docs/technical/schema.md index 9dd0bd6..de032e3 100644 --- a/docs/technical/schema.md +++ b/docs/technical/schema.md @@ -194,7 +194,8 @@ Tables generally use a numeric auto-increment `id` primary key. The relationship - `i_rss_feeds` - RSS feed definitions and refresh cadence. - `i_rss_feed_items` - cached RSS feed items. -- `i_api_sources` - API source definitions and last response snapshot. +- `i_api_source_groups` - API source names and shared authentication settings. +- `i_api_sources` - named requests, independent refresh settings, and last response snapshots. - `i_weather_locations` - configured weather locations and last response snapshot. - `i_timetable_groups` - grouped timetable definitions used by the timetable region. - `i_timetable_entries` - dated entries that belong to a timetable group. @@ -211,9 +212,18 @@ Tables generally use a numeric auto-increment `id` primary key. The relationship - Unique key: - `(rss_feed_id, position)` +### `i_api_source_groups` + +- `id`, `name`, `auth_method`, `auth_username`, `auth_password`, `auth_bearer_token`, `auth_header_name`, `auth_header_value`, `token_url`, `token_request_body_json`, `token_response_path`, `token_refresh_url`, `token_refresh_request_body_json`, `token_refresh_response_path`, `token_header_name`, `token_header_prefix`, `created_at`, `created_by`, `modified_at`, `modified_by` +- Unique group name; authentication applies to every child request. `auth_method = 'none'` supports public endpoints. Token caches are keyed by group and authentication configuration. + ### `i_api_sources` -- `id`, `name`, `api_url`, `auth_method`, `auth_username`, `auth_password`, `auth_bearer_token`, `auth_header_name`, `auth_header_value`, `token_url`, `token_request_body_json`, `token_response_path`, `token_refresh_url`, `token_refresh_request_body_json`, `token_refresh_response_path`, `token_header_name`, `token_header_prefix`, `items_path`, `update_interval_value`, `update_interval_unit`, `enabled`, `last_pulled_at`, `last_pull_error`, `last_response_status`, `last_response_content_type`, `last_response_json`, `created_at`, `created_by`, `modified_at`, `modified_by` +- `id`, `auth_source_id`, `name`, `api_url`, `request_method`, `request_body_json`, `items_path`, `update_interval_value`, `update_interval_unit`, `enabled`, `last_pulled_at`, `last_pull_error`, `last_response_status`, `last_response_content_type`, `last_response_json`, `created_at`, `created_by`, `modified_at`, `modified_by` +- `auth_source_id` references `i_api_source_groups.id` with restricted parent deletion. The web application requires at least one request per group and unique request names within each group. +- Request IDs remain the `source_id` stored in slide regions and the keys used by scheduled refreshes. Removing a request used by a slide is blocked. +- The 2.16.1 migration copies each existing source's authentication into a same-ID parent group and links the original request without changing its ID or response snapshot. Legacy authentication columns remain on request rows for schema compatibility but runtime authentication is loaded from the parent. +- Editor/player request lists exclude authentication credentials. Refreshes resolve current parent authentication at execution time. ### `i_weather_locations` @@ -321,6 +331,8 @@ erDiagram } I_API_SOURCES { } + I_API_SOURCE_GROUPS { + } I_WEATHER_LOCATIONS { } I_TIMETABLE_GROUPS { @@ -343,6 +355,7 @@ erDiagram A_USERS ||--o{ A_SESSIONS : owns A_USERS ||--o{ A_ACCOUNT_TOKENS : has A_USERS o|--o{ O_AUDIT_EVENTS : acts + I_API_SOURCE_GROUPS ||--o{ I_API_SOURCES : contains C_CANVAS_SIZES o|--o{ C_TEMPLATES : used_by C_CANVAS_SIZES o|--o{ C_PLAYLISTS : used_by diff --git a/docs/user/guide-data-sources.md b/docs/user/guide-data-sources.md index 2225cff..76070fc 100644 --- a/docs/user/guide-data-sources.md +++ b/docs/user/guide-data-sources.md @@ -25,7 +25,9 @@ When an RSS feed fails, open the latest-items preview and check the reported pul ## API Sources -An API source connects to a JSON endpoint and can use either GET or POST: +An API source groups one or more named requests under shared authentication. Use **Add request** to create separate cards for endpoints such as event information and donors. Each request has its own URL, method, items path, enabled state, refresh interval, latest response, and **Test Connection** / **Refresh now** actions. Save the source to persist added, edited, or removed requests. A request used by a slide cannot be removed. + +Each request can use either GET or POST: - **GET:** retrieve data directly from the API URL. - **POST:** send a JSON request body with the request. @@ -40,10 +42,18 @@ API authentication supports: - An API key sent in a named header. - A login-then-token flow for services that issue a token from a login request. +Authentication applies to every request in the source. Choose **None** for public endpoints. Token-login requests share one cached token and refresh it as needed. **Test Authentication** checks the shared settings without calling a data endpoint; Basic, Bearer, and API-key settings also need a request test to verify provider acceptance. Slides select a request by its **source / request** name. Existing request IDs and slide bindings are preserved when upgrading. + For a login-then-token source, configure the login URL, JSON login body, token response path, and token header. If the service supports token refresh, configure the refresh URL, refresh-token response path, refresh body, and token prefix as required by that service. Keep credentials and tokens out of slide content and request examples that will be shared with other users. After a pull, review the latest response details, including the last-pulled time, HTTP status, content type, and stored JSON response. Use the response structure to choose the item number, items path, and fields used by an API region. +Use **Test Connection** on the add or edit form to test the current field values without saving them or replacing the stored response. Token-login tests obtain a fresh token without replacing the token used by scheduled refreshes. Results separate authentication, the data request, and JSON/items extraction, with HTTP status, duration, content type, sanitised JSON previews, and a sample of available field paths. Each request has a 15-second timeout. + +An **Authentication: HTTP 404** means the token request received a not-found response before the data endpoint was called. Check the **Login / token URL**, API version, and provider documentation first. For a 401, check credentials; for a 403, check token scopes and application permissions. Token requests currently use POST with a JSON body, so confirm that format is supported by the provider. + +Use **Copy diagnostic report** to share the results. Known credentials and token values are redacted, URL query strings are omitted, and non-JSON response bodies are withheld. Previews are limited to 12,000 characters. Review reports for other private application data before sharing them. + ## Timetables A timetable group contains the events that a timetable region displays. Configure: diff --git a/package-lock.json b/package-lock.json index ede05cb..5fd15ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "pulse-signage", - "version": "2.16.0", + "version": "2.16.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pulse-signage", - "version": "2.16.0", + "version": "2.16.1", "dependencies": { "@sparticuz/chromium": "^149.0.0", "animate.css": "^4.1.1", diff --git a/package.json b/package.json index 6228f1f..5edda0c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pulse-signage", - "version": "2.16.0", + "version": "2.16.1", "private": false, "description": "Pulse Signage application with MySQL and media storage", "engines": { diff --git a/src/common.js b/src/common.js index 9f5854a..2055d0f 100644 --- a/src/common.js +++ b/src/common.js @@ -50,7 +50,6 @@ module.exports = { fetchTimetableEntriesByGroupId: data.fetchTimetableEntriesByGroupId, buildTimetableGroupPayload: data.buildTimetableGroupPayload, fetchApiSourcesData: data.fetchApiSourcesData, - fetchApiSourcesPage: data.fetchApiSourcesPage, fetchApiSourceById: data.fetchApiSourceById, fetchApiSourceResponse: data.fetchApiSourceResponse, buildApiSourcePayload: data.buildApiSourcePayload, diff --git a/src/data/api-source-groups.js b/src/data/api-source-groups.js new file mode 100644 index 0000000..2378f7d --- /dev/null +++ b/src/data/api-source-groups.js @@ -0,0 +1,124 @@ +const { API_AUTH_FIELDS, API_REQUEST_SELECT, buildApiSourcePayload } = require('#src/data/api-sources'); +const { fetchPagedRows, validateMaxLength, parseJsonSafe } = require('#src/data/utils'); + +function toFormFields(value) { + const fields = Object.assign({}, value); + Object.keys(value || {}).forEach(function (key) { + fields[key.replace(/[A-Z]/g, function (letter) { return '_' + letter.toLowerCase(); })] = value[key]; + }); + return fields; +} + +function validationError(message, statusCode) { + const error = new Error(message); + error.statusCode = statusCode || 400; + return error; +} + +function buildApiGroupPayload(body) { + let requests = body.requests_json; + if (typeof requests === 'string') { + try { requests = JSON.parse(requests); } catch (_error) { throw validationError('Requests must contain valid JSON.'); } + } + if (!Array.isArray(requests) || requests.length < 1 || requests.length > 50) { + throw validationError('Add between 1 and 50 API requests.'); + } + const name = validateMaxLength(body.name, 255, 'API source name'); + if (!name) throw validationError('API source name is required.'); + const ids = new Set(); + const names = new Set(); + const parsedRequests = requests.map(function (request) { + if (!request || typeof request !== 'object' || Array.isArray(request)) throw validationError('Invalid API request.'); + const id = request.id === '' || request.id === null || request.id === undefined ? null : Number(request.id); + if (id !== null && (!Number.isSafeInteger(id) || id < 1 || ids.has(id))) throw validationError('Invalid or repeated API request ID.'); + if (id !== null) ids.add(id); + const requestName = validateMaxLength(request.name, 255, 'Request name'); + if (!requestName || names.has(requestName.toLowerCase())) throw validationError('Each request needs a unique name.'); + names.add(requestName.toLowerCase()); + const fields = toFormFields(request); + API_AUTH_FIELDS.forEach(function (field) { fields[field] = body[field]; }); + fields.name = requestName; + const payload = buildApiSourcePayload({ body: fields }); + return Object.assign(payload, { id: id, enabled: request.enabled === false || request.enabled === 0 || request.enabled === '0' ? 0 : 1 }); + }); + const auth = toFormFields(parsedRequests[0]); + return { name: name, auth: auth, requests: parsedRequests }; +} + +async function fetchApiGroupById(pool, id) { + const [groups] = await pool.query('SELECT * FROM i_api_source_groups WHERE id = ?', [id]); + if (!groups.length) return null; + const [requests] = await pool.query(API_REQUEST_SELECT + ' WHERE request.auth_source_id = ? ORDER BY request.id', [id]); + return Object.assign({}, groups[0], { requests: requests.map(function (request) { return Object.assign({}, request, { name: request.request_name }); }) }); +} + +async function fetchApiGroupsPage(pool, page, pageSize, search, sort, direction) { + const data = await fetchPagedRows(pool, { + selectSql: 'SELECT auth.*, (SELECT COUNT(*) FROM i_api_sources request WHERE request.auth_source_id = auth.id) AS request_count, (SELECT MAX(last_pulled_at) FROM i_api_sources request WHERE request.auth_source_id = auth.id) AS last_pulled_at FROM i_api_source_groups auth ORDER BY auth.modified_at DESC, auth.id DESC', + countSql: 'SELECT COUNT(*) AS count FROM i_api_source_groups auth', + searchColumns: ['auth.name', 'auth.token_url'], searchTerm: search, + sortColumns: { name: 'auth.name', requests: 'request_count', last_pulled: 'last_pulled_at' }, + sortKey: sort, sortDirection: direction, page: page, pageSize: pageSize + }); + return Object.assign({ apiSources: data.rows }, data); +} + +async function getApiRequestUsage(pool) { + const [slides] = await pool.query('SELECT content_json FROM c_slides WHERE content_json IS NOT NULL'); + const ids = new Set(); + function walk(value) { + if (!value || typeof value !== 'object') return; + if (value.source_id) ids.add(Number(value.source_id)); + Object.values(value).forEach(walk); + } + slides.forEach(function (slide) { walk(parseJsonSafe(slide.content_json)); }); + return ids; +} + +async function saveApiGroup(pool, id, payload, actorId, canDelete) { + const connection = await pool.getConnection(); + try { + await connection.beginTransaction(); + if (id) { + const [groups] = await connection.query('SELECT id FROM i_api_source_groups WHERE id = ? FOR UPDATE', [id]); + if (!groups.length) throw validationError('API source not found.', 404); + } + const [duplicates] = await connection.query('SELECT id FROM i_api_source_groups WHERE name = ? AND id <> ?', [payload.name, id || 0]); + if (duplicates.length) throw validationError('An API source with that name already exists.'); + const [existing] = id ? await connection.query('SELECT id FROM i_api_sources WHERE auth_source_id = ? FOR UPDATE', [id]) : [[]]; + const existingIds = new Set(existing.map(function (request) { return Number(request.id); })); + const retained = new Set(payload.requests.map(function (request) { return request.id; }).filter(Boolean)); + if (Array.from(retained).some(function (requestId) { return !existingIds.has(requestId); })) throw validationError('A request does not belong to this API source.'); + const removed = Array.from(existingIds).filter(function (requestId) { return !retained.has(requestId); }); + if (removed.length) { + if (!canDelete) throw validationError('You do not have permission to remove API requests.', 403); + const usage = await getApiRequestUsage(connection); + if (removed.some(function (requestId) { return usage.has(requestId); })) throw validationError('A request being removed is still used by one or more slides.'); + } + const values = [payload.name].concat(API_AUTH_FIELDS.map(function (field) { return payload.auth[field] === undefined ? null : payload.auth[field]; })); + if (id) { + await connection.query('UPDATE i_api_source_groups SET name = ?, ' + API_AUTH_FIELDS.map(function (field) { return field + ' = ?'; }).join(', ') + ', modified_by = ? WHERE id = ?', values.concat([actorId, id])); + } else { + const [result] = await connection.query('INSERT INTO i_api_source_groups (name, ' + API_AUTH_FIELDS.join(', ') + ', created_by, modified_by) VALUES (' + Array(values.length + 2).fill('?').join(', ') + ')', values.concat([actorId, actorId])); + id = result.insertId; + } + for (const request of payload.requests) { + const requestValues = [request.name, request.apiUrl, request.requestMethod, request.requestBodyJson || null, request.itemsPath || null, request.enabled, request.updateIntervalValue, request.updateIntervalUnit, actorId]; + if (request.id) { + await connection.query('UPDATE i_api_sources SET name = ?, api_url = ?, request_method = ?, request_body_json = ?, items_path = ?, enabled = ?, update_interval_value = ?, update_interval_unit = ?, modified_by = ? WHERE id = ? AND auth_source_id = ?', requestValues.concat([request.id, id])); + } else { + const [result] = await connection.query('INSERT INTO i_api_sources (name, api_url, request_method, request_body_json, items_path, enabled, update_interval_value, update_interval_unit, modified_by, auth_source_id, created_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', requestValues.concat([id, actorId])); + request.id = result.insertId; + } + } + if (removed.length) await connection.query('DELETE FROM i_api_sources WHERE auth_source_id = ? AND id IN (?)', [id, removed]); + await connection.commit(); + return { id: Number(id), removed: removed, requests: payload.requests }; + } catch (error) { + await connection.rollback(); + if (error.code === 'ER_DUP_ENTRY') throw validationError('An API source with that name already exists.'); + throw error; + } finally { connection.release(); } +} + +module.exports = { buildApiGroupPayload, fetchApiGroupById, fetchApiGroupsPage, getApiRequestUsage, saveApiGroup, toFormFields }; \ No newline at end of file diff --git a/src/data/api-sources.js b/src/data/api-sources.js index 89cec0f..aecf104 100644 --- a/src/data/api-sources.js +++ b/src/data/api-sources.js @@ -2,7 +2,8 @@ const http = require('http'); const https = require('https'); -const { fetchPagedRows, validateMaxLength } = require('#src/data/utils'); +const { randomUUID } = require('crypto'); +const { validateMaxLength } = require('#src/data/utils'); const NAME_MAX_LENGTH = 255; const URL_MAX_LENGTH = 1024; @@ -14,6 +15,121 @@ const TOKEN_RESPONSE_PATH_MAX_LENGTH = 255; const TOKEN_HEADER_PREFIX_MAX_LENGTH = 64; const tokenCache = new Map(); const tokenRequests = new Map(); +const diagnosticContext = Symbol('apiSourceDiagnostics'); +const API_AUTH_FIELDS = ['auth_method', 'auth_username', 'auth_password', 'auth_bearer_token', 'auth_header_name', 'auth_header_value', 'token_url', 'token_request_body_json', 'token_response_path', 'token_refresh_url', 'token_refresh_request_body_json', 'token_refresh_response_path', 'token_header_name', 'token_header_prefix']; +const API_REQUEST_SELECT = 'SELECT request.*, request.name AS request_name, auth.name AS group_name, CONCAT(auth.name, \' / \', request.name) AS name, ' + API_AUTH_FIELDS.map(function (field) { return 'auth.' + field; }).join(', ') + ' FROM i_api_sources request JOIN i_api_source_groups auth ON auth.id = request.auth_source_id'; + +function createDiagnosticSanitizer(source) { + const secrets = []; + const sensitiveKey = /password|secret|token|authorization|cookie|api.?key|authHeaderValue|auth_header_value|authUsername|auth_username/i; + const oauthMetadataKey = /^(scope|grant_type|token_type|tokenType)$/i; + function collect(value, force) { + if (value && typeof value === 'object') { + Object.keys(value).forEach(function (key) { + if (oauthMetadataKey.test(key)) return; + const configuration = /url|path|prefix|header.?name|auth.?method/i.test(key); + collect(value[key], force || (!configuration && sensitiveKey.test(key)) || /body.?json/i.test(key)); + }); + } else if (typeof value === 'string' && value) { + if (force) secrets.push(value); + try { + const url = new URL(value); + if (url.username) secrets.push(url.username); + if (url.password) secrets.push(url.password); + url.searchParams.forEach(function (item) { if (item) secrets.push(item); }); + } catch (_error) {} + try { collect(JSON.parse(value), true); } catch (_error) {} + } + } + collect(source, false); + function redactText(value) { + let text = String(value || ''); + secrets.sort(function (left, right) { return right.length - left.length; }).forEach(function (secret) { + text = text.split(secret).join('[REDACTED]'); + }); + return text.replace(/(Bearer\s+)[^\s"<>]+/gi, '$1[REDACTED]'); + } + function sanitize(value) { + if (Array.isArray(value)) return value.map(sanitize); + if (value && typeof value === 'object') { + return Object.fromEntries(Object.entries(value).map(function ([key, item]) { + const sensitive = sensitiveKey.test(key) && !oauthMetadataKey.test(key); + return [redactText(key), sensitive && item !== null && item !== '' ? '[REDACTED]' : sanitize(item)]; + })); + } + return typeof value === 'string' ? redactText(value) : value; + } + return { + add: function (value) { collect(value, false); }, + secret: function (value) { if (value) secrets.push(String(value)); }, + text: redactText, + body: function (text) { + try { return JSON.stringify(sanitize(JSON.parse(text)), null, 2).slice(0, 12000); } + catch (_error) { return '[Non-JSON response omitted]'; } + } + }; +} + +function diagnosticUrl(value) { + try { + const url = new URL(value); + url.username = ''; + url.password = ''; + url.search = ''; + url.hash = ''; + return url.toString(); + } catch (_error) { return '[Invalid URL]'; } +} + +async function loadUrlText(url, options, source, stage) { + const context = source && source[diagnosticContext]; + const sanitizer = context ? context.sanitizer : createDiagnosticSanitizer(source || {}); + Object.entries(options.headers || {}).forEach(function ([key, value]) { + if (!/^(accept|content-type|user-agent)$/i.test(key)) sanitizer.secret(value); + }); + const step = { stage: stage, method: options.method, url: sanitizer.text(diagnosticUrl(url)) }; + if (context) context.steps.push(step); + const started = Date.now(); + try { + const response = await requestUrlText(url, options); + step.status = response.statusCode; + step.contentType = response.contentType; + try { + const parsed = JSON.parse(response.bodyText); + sanitizer.add(parsed); + if (stage === 'Authentication' || stage === 'Token refresh') { + sanitizer.secret(resolveResponsePath(parsed, source.token_response_path || source.tokenResponsePath || 'access_token')); + sanitizer.secret(resolveResponsePath(parsed, source.token_refresh_response_path || source.tokenRefreshResponsePath || 'refresh_token')); + } + } catch (_error) {} + step.response = sanitizer.body(response.bodyText); + step.ok = response.ok; + if (!response.ok) { + const hints = { + 400: ' Check the request body and grant type.', + 401: ' Check the credentials or token validity.', + 403: ' Check the token scopes and application permissions.', + 404: ' Check the endpoint URL and API version.', + 429: ' The provider is rate limiting requests; reduce the refresh frequency.' + }; + const hint = hints[response.statusCode] || ''; + const error = new Error(stage + ' failed: HTTP ' + response.statusCode + ' at ' + step.url + '.' + hint + ' ' + step.response.slice(0, 1000)); + error.apiDiagnostic = true; + step.error = error.message; + response.diagnosticError = error; + } + return response; + } catch (error) { + step.ok = false; + const code = error.cause && error.cause.code || error.code || error.name; + const safeCode = /^[A-Z_0-9]+$/.test(String(code)) ? ' (' + code + ')' : ''; + step.error = error.apiDiagnostic ? error.message : stage + ' request failed (network, TLS, or timeout)' + safeCode + '.'; + if (error.apiDiagnostic) throw error; + throw new Error(step.error); + } finally { + step.durationMs = Date.now() - started; + } +} function normalizeUpdateIntervalUnit(value) { const unit = String(value || '').trim().toLowerCase(); @@ -60,46 +176,22 @@ function buildAuthHeaders(source) { async function fetchApiSourcesData(pool) { const [apiSources] = await pool.query( - 'SELECT id, name, api_url, request_method, request_body_json, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, token_url, token_request_body_json, token_response_path, token_refresh_url, token_refresh_request_body_json, token_refresh_response_path, token_header_name, token_header_prefix, items_path, enabled, update_interval_value, update_interval_unit, last_pulled_at, last_pull_error, last_response_status, last_response_content_type, last_response_json, created_at, modified_at, created_by, modified_by FROM i_api_sources ORDER BY modified_at DESC, id DESC' + "SELECT request.id, request.auth_source_id, request.name AS request_name, auth.name AS group_name, CONCAT(auth.name, ' / ', request.name) AS name, request.items_path, request.enabled, request.update_interval_value, request.update_interval_unit, request.last_pulled_at, request.last_pull_error, request.last_response_status, request.last_response_content_type, request.last_response_json FROM i_api_sources request JOIN i_api_source_groups auth ON auth.id = request.auth_source_id ORDER BY auth.name, request.id" ); return { apiSources: apiSources }; } -async function fetchApiSourcesPage(pool, page, pageSize, searchTerm, sortKey, sortDirection) { - const paged = await fetchPagedRows(pool, { - selectSql: 'SELECT id, name, api_url, request_method, request_body_json, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, token_url, token_request_body_json, token_response_path, token_refresh_url, token_refresh_request_body_json, token_refresh_response_path, token_header_name, token_header_prefix, items_path, enabled, update_interval_value, update_interval_unit, last_pulled_at, last_pull_error, last_response_status, last_response_content_type, last_response_json, created_at, modified_at, created_by, modified_by FROM i_api_sources ORDER BY modified_at DESC, id DESC', - countSql: 'SELECT COUNT(*) AS count FROM i_api_sources', - searchColumns: ['name', 'api_url', 'last_pull_error'], - searchTerm: searchTerm, - sortColumns: { - name: 'name', - url: 'api_url', - interval: ['update_interval_value', 'update_interval_unit'], - last_pulled: 'last_pulled_at', - last_response: 'last_response_status', - created: 'created_at', - modified: 'modified_at' - }, - sortKey: sortKey, - sortDirection: sortDirection, - page: page, - pageSize: pageSize - }); - - return Object.assign({ apiSources: paged.rows }, paged); -} - async function fetchApiSourceById(pool, id) { const [rows] = await pool.query( - 'SELECT id, name, api_url, request_method, request_body_json, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, token_url, token_request_body_json, token_response_path, token_refresh_url, token_refresh_request_body_json, token_refresh_response_path, token_header_name, token_header_prefix, items_path, enabled, update_interval_value, update_interval_unit, last_pulled_at, last_pull_error, last_response_status, last_response_content_type, last_response_json, created_at, modified_at, created_by, modified_by FROM i_api_sources WHERE id = ?', + API_REQUEST_SELECT + ' WHERE request.id = ?', [id] ); return rows[0] || null; } -async function loadUrlText(urlValue, requestOptions) { +async function requestUrlText(urlValue, requestOptions) { const extraHeaders = requestOptions && requestOptions.headers ? requestOptions.headers : {}; const method = String(requestOptions && requestOptions.method || 'GET').toUpperCase(); const body = requestOptions && requestOptions.body !== undefined ? requestOptions.body : undefined; @@ -108,7 +200,7 @@ async function loadUrlText(urlValue, requestOptions) { 'User-Agent': 'Pulse Signage API Reader' }, extraHeaders); if (typeof fetch === 'function') { - const fetchOptions = { method: method, headers: headers }; + const fetchOptions = { method: method, headers: headers, signal: AbortSignal.timeout(15000) }; if (body !== undefined && method !== 'GET' && method !== 'HEAD') { fetchOptions.body = body; } @@ -149,6 +241,7 @@ async function loadUrlText(urlValue, requestOptions) { request.write(body); } request.end(); + request.setTimeout(15000, function () { request.destroy(new Error('Request timed out')); }); request.on('error', reject); }); } @@ -181,7 +274,7 @@ function resolveResponsePath(value, responsePath) { function getTokenCacheKey(source) { return JSON.stringify([ - source && source.id || '', + source && (source.auth_source_id || source.authSourceId || source.id) || '', source && (source.token_url || source.tokenUrl) || '', source && (source.token_request_body_json || source.tokenRequestBodyJson) || '', source && (source.token_response_path || source.tokenResponsePath) || 'access_token', @@ -252,7 +345,7 @@ function cacheTokenResponse(source, parsed, previousRefreshToken) { const tokenPath = source.token_response_path || source.tokenResponsePath || 'access_token'; const token = resolveResponsePath(parsed, tokenPath); if (token === undefined || token === null || String(token).trim() === '') { - throw new Error('Token response did not contain a token at the configured path.'); + throw new Error('Token response did not contain a token at the configured path: ' + createDiagnosticSanitizer(source).text(tokenPath) + '. Check the token response path.'); } const refreshPath = source.token_refresh_response_path || source.tokenRefreshResponsePath || 'refresh_token'; @@ -293,7 +386,7 @@ async function refreshLoginToken(source, cached) { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(refreshBody) - }); + }, source, 'Token refresh'); return parseTokenResponse(response, source, cached.refreshToken); } @@ -318,10 +411,10 @@ async function fetchLoginTokenUncached(source) { method: 'POST', headers: tokenHeaders, body: tokenBody === undefined ? undefined : JSON.stringify(tokenBody) - }); + }, source, 'Authentication'); const record = await parseTokenResponse(response, source, cached && cached.refreshToken); if (!record) { - throw new Error(`Unable to obtain API token (${response.statusCode}).`); + throw response.diagnosticError || new Error(`Unable to obtain API token (${response.statusCode}).`); } return record; } @@ -363,7 +456,7 @@ async function fetchApiSourceResponse(apiSource) { method: requestMethod, headers: Object.assign({}, await buildRequestHeaders(source), requestBody === undefined ? {} : { 'Content-Type': 'application/json' }), body: requestBody === undefined ? undefined : JSON.stringify(requestBody) - }); + }, source, 'Data request'); if (response.statusCode === 401 && normalizeAuthMethod(source.auth_method || source.authMethod) === 'token_login' && !tokenRetry) { clearCachedToken(source); tokenRetry = true; @@ -372,7 +465,7 @@ async function fetchApiSourceResponse(apiSource) { } } while (true); if (!response.ok) { - throw new Error(`Unable to load API response (${response.statusCode}).`); + throw response.diagnosticError || new Error(`Unable to load API response (${response.statusCode}).`); } const text = String(response.bodyText || '').trim(); @@ -394,6 +487,69 @@ async function fetchApiSourceResponse(apiSource) { }; } +async function runApiSourceDiagnostic(source, run) { + const context = { steps: [], sanitizer: createDiagnosticSanitizer(source) }; + const id = 'diagnostic-' + randomUUID(); + const isolatedSource = Object.assign({}, source, { id: id, auth_source_id: id, authSourceId: null, [diagnosticContext]: context }); + const report = { ok: false, steps: context.steps }; + try { + await run(isolatedSource, context, report); + } catch (error) { + report.error = context.sanitizer.text(error.message); + const lastStep = context.steps[context.steps.length - 1]; + if (lastStep && lastStep.stage === 'Data request' && lastStep.ok) { + context.steps.push({ stage: 'JSON / items extraction', ok: false, error: report.error }); + } else if (lastStep) { + lastStep.ok = false; + lastStep.error = report.error; + } + } finally { + clearCachedToken(isolatedSource); + } + return report; +} + +function testApiSourceAuthentication(source) { + return runApiSourceDiagnostic(source, async function (isolatedSource, context, report) { + await buildRequestHeaders(isolatedSource); + if (!context.steps.length) context.steps.push({ stage: 'Authentication', ok: true, message: normalizeAuthMethod(source.auth_method || source.authMethod) === 'none' ? 'No authentication required.' : 'Authentication headers configured. Test a request to verify provider acceptance.' }); + report.ok = true; + }); +} + +function testApiSourceConnection(source) { + return runApiSourceDiagnostic(source, async function (isolatedSource, context, report) { + if (normalizeAuthMethod(source.auth_method || source.authMethod) !== 'token_login') { + context.steps.push({ stage: 'Authentication', ok: true, skipped: true, message: 'No token exchange required.' }); + } + const result = await fetchApiSourceResponse(isolatedSource); + const parsed = JSON.parse(result.responseJson); + const paths = []; + function walk(value, path, depth) { + if (depth > 6 || paths.length >= 100 || !value || typeof value !== 'object') return; + Object.keys(value).slice(0, Array.isArray(value) ? 1 : 100).forEach(function (key) { + if (paths.length >= 100 || /password|secret|token|authorization|cookie|api.?key/i.test(key)) return; + const nextPath = path ? path + '.' + key : key; + paths.push(context.sanitizer.text(nextPath)); + walk(value[key], nextPath, depth + 1); + }); + } + walk(parsed, '', 0); + report.paths = paths; + report.preview = context.sanitizer.body(result.responseJson); + const itemsPath = String(source.items_path || source.itemsPath || '').trim(); + const items = itemsPath ? resolveResponsePath(parsed, itemsPath) : parsed; + const matched = items !== undefined && items !== null; + context.steps.push({ + stage: 'JSON / items extraction', ok: matched, + message: matched + ? 'Valid JSON. ' + (itemsPath ? 'Items path matched. ' : 'Using response root. ') + (Array.isArray(items) ? items.length + ' items.' : 'Selected value is ' + typeof items + '.') + : 'Items path did not match the response: ' + context.sanitizer.text(itemsPath) + }); + report.ok = matched; + }); +} + function buildApiSourcePayload(req, existingApiSource) { const fallback = existingApiSource || {}; const body = req && req.body ? req.body : {}; @@ -543,9 +699,12 @@ function buildApiSourcePayload(req, existingApiSource) { } module.exports = { + API_AUTH_FIELDS, + API_REQUEST_SELECT, fetchApiSourcesData: fetchApiSourcesData, - fetchApiSourcesPage: fetchApiSourcesPage, fetchApiSourceById: fetchApiSourceById, fetchApiSourceResponse: fetchApiSourceResponse, + testApiSourceConnection: testApiSourceConnection, + testApiSourceAuthentication, buildApiSourcePayload: buildApiSourcePayload }; \ No newline at end of file diff --git a/src/data/index.js b/src/data/index.js index 4afba27..89b9188 100644 --- a/src/data/index.js +++ b/src/data/index.js @@ -5,7 +5,7 @@ const { ANNOUNCEMENT_TYPES, ANNOUNCEMENT_COLORS, ANNOUNCEMENT_ICONS, DEFAULT_ANN const { ANNOUNCEMENT_ICON_OPTIONS, ANNOUNCEMENT_ICON_LABELS } = require('#src/data/announcement-icons'); const { fetchPlaylistById } = require('#src/data/playlists'); const { normalizeDisplayMode, fetchTimetablesData, fetchTimetableGroupsPage, fetchTimetableGroupById, fetchTimetableEntriesByGroupId, buildTimetableGroupPayload } = require('#src/data/timetables'); -const { fetchApiSourcesData, fetchApiSourcesPage, fetchApiSourceById, fetchApiSourceResponse, buildApiSourcePayload } = require('#src/data/api-sources'); +const { fetchApiSourcesData, fetchApiSourceById, fetchApiSourceResponse, buildApiSourcePayload } = require('#src/data/api-sources'); const { fetchRssFeedsData, fetchRssFeedsPage, fetchRssFeedById, fetchRssFeedItemsByFeedId, normalizeRssFeedItem, buildRssFeedPayload, fetchRssFeedItems, replaceRssFeedItems } = require('#src/data/rss-feeds'); const { fetchWeatherLocationsData, fetchWeatherLocationsPage, fetchWeatherLocationById, fetchWeatherLocationSuggestions, fetchWeatherLocationForecast, buildWeatherLocationPayload } = require('#src/data/weather'); const { slugify, uniqueScreenSlug, fetchScreenById, fetchScreenEditData, fetchScreenPlayerUrls, fetchPlayerPublicBaseUrl, fetchScreenPlayerRecord, fetchPlayerRecordByIdentifier } = require('#src/data/screens'); @@ -48,7 +48,6 @@ module.exports = { fetchTimetableEntriesByGroupId, buildTimetableGroupPayload, fetchApiSourcesData, - fetchApiSourcesPage, fetchApiSourceById, fetchApiSourceResponse, buildApiSourcePayload, diff --git a/src/db/index.js b/src/db/index.js index c655820..9c44f43 100644 --- a/src/db/index.js +++ b/src/db/index.js @@ -259,9 +259,34 @@ async function ensureSchema(pool, options) { ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `); + await pool.query(` + CREATE TABLE IF NOT EXISTS i_api_source_groups ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL UNIQUE, + auth_method VARCHAR(32) NOT NULL DEFAULT 'none', + auth_username VARCHAR(255) NULL, + auth_password MEDIUMTEXT NULL, + auth_bearer_token MEDIUMTEXT NULL, + auth_header_name VARCHAR(255) NULL, + auth_header_value MEDIUMTEXT NULL, + token_url VARCHAR(1024) NULL, + token_request_body_json MEDIUMTEXT NULL, + token_response_path VARCHAR(255) NULL DEFAULT 'access_token', + token_refresh_url VARCHAR(1024) NULL, + token_refresh_request_body_json MEDIUMTEXT NULL, + token_refresh_response_path VARCHAR(255) NULL DEFAULT 'refresh_token', + token_header_name VARCHAR(255) NULL DEFAULT 'Authorization', + token_header_prefix VARCHAR(64) NULL DEFAULT 'Bearer', + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + created_by INT NULL, + modified_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + modified_by INT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + `); await pool.query(` CREATE TABLE IF NOT EXISTS i_api_sources ( id INT AUTO_INCREMENT PRIMARY KEY, + auth_source_id INT NULL, name VARCHAR(255) NOT NULL, api_url VARCHAR(1024) NOT NULL, request_method VARCHAR(10) NOT NULL DEFAULT 'GET', @@ -292,7 +317,8 @@ async function ensureSchema(pool, options) { created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, created_by INT NULL, modified_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - modified_by INT NULL + modified_by INT NULL, + CONSTRAINT fk_api_sources_auth_source FOREIGN KEY (auth_source_id) REFERENCES i_api_source_groups(id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `); diff --git a/src/db/migrations.js b/src/db/migrations.js index a9c7cdf..3adb618 100644 --- a/src/db/migrations.js +++ b/src/db/migrations.js @@ -684,6 +684,39 @@ const VERSIONED_MIGRATIONS = [ run: async function (pool) { await ensureColumn(pool, 'c_canvas_sizes', 'enabled', 'TINYINT(1) NOT NULL DEFAULT 1', 'height'); } + }, + { + version: '2.16.1', + label: 'v2.16.1 shared API authentication and request groups', + run: async function (pool) { + await pool.query(`CREATE TABLE IF NOT EXISTS i_api_source_groups ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL UNIQUE, + auth_method VARCHAR(32) NOT NULL DEFAULT 'none', + auth_username VARCHAR(255) NULL, + auth_password MEDIUMTEXT NULL, + auth_bearer_token MEDIUMTEXT NULL, + auth_header_name VARCHAR(255) NULL, + auth_header_value MEDIUMTEXT NULL, + token_url VARCHAR(1024) NULL, + token_request_body_json MEDIUMTEXT NULL, + token_response_path VARCHAR(255) NULL DEFAULT 'access_token', + token_refresh_url VARCHAR(1024) NULL, + token_refresh_request_body_json MEDIUMTEXT NULL, + token_refresh_response_path VARCHAR(255) NULL DEFAULT 'refresh_token', + token_header_name VARCHAR(255) NULL DEFAULT 'Authorization', + token_header_prefix VARCHAR(64) NULL DEFAULT 'Bearer', + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + created_by INT NULL, + modified_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + modified_by INT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`); + await ensureColumn(pool, 'i_api_sources', 'auth_source_id', 'INT NULL', 'id'); + const fields = 'name, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, token_url, token_request_body_json, token_response_path, token_refresh_url, token_refresh_request_body_json, token_refresh_response_path, token_header_name, token_header_prefix, created_at, created_by, modified_at, modified_by'; + await pool.query('INSERT IGNORE INTO i_api_source_groups (id, ' + fields + ') SELECT id, ' + fields + ' FROM i_api_sources WHERE auth_source_id IS NULL'); + await pool.query('UPDATE i_api_sources SET auth_source_id = id WHERE auth_source_id IS NULL'); + await ensureForeignKey(pool, 'i_api_sources', 'fk_api_sources_auth_source', 'auth_source_id', 'i_api_source_groups', 'id', 'RESTRICT'); + } } ]; diff --git a/src/web/lib/background-tasks/tasks-scheduled/data-source-refresh.js b/src/web/lib/background-tasks/tasks-scheduled/data-source-refresh.js index e4ab784..0420be3 100644 --- a/src/web/lib/background-tasks/tasks-scheduled/data-source-refresh.js +++ b/src/web/lib/background-tasks/tasks-scheduled/data-source-refresh.js @@ -35,7 +35,7 @@ function registerRecurringDataSourceRefreshes(options) { sourceName: apiSource.name }, run: function () { - return refreshApiSource(pool, common, apiSource, null, options.notifyPlayerScreens); + return refreshApiSource(pool, common, apiSource.id, null, options.notifyPlayerScreens); } }); }); diff --git a/src/web/public/js/data-sources/api-source-form.js b/src/web/public/js/data-sources/api-source-form.js index 601a4a8..ff07c57 100644 --- a/src/web/public/js/data-sources/api-source-form.js +++ b/src/web/public/js/data-sources/api-source-form.js @@ -12,9 +12,188 @@ var panels = Array.prototype.slice.call(form.querySelectorAll('[data-api-source-auth-panel]')); var bearerTokenInput = form.querySelector('[data-api-source-bearer-token-input]'); var bearerTokenToggle = form.querySelector('[data-api-source-bearer-token-toggle]'); - var requestMethodSelect = form.querySelector('[data-api-source-request-method]'); - var requestSection = form.querySelector('[data-api-source-request-section]'); - var requestBodyInput = form.querySelector('[data-api-source-request-body]'); + var testButton = form.querySelector('[data-api-auth-test-url]') || form.querySelector('[data-api-source-test-url]'); + var lastTestButton = testButton; + var isTesting = false; + var requestList = form.querySelector('[data-api-request-list]'); + var requestTemplate = form.querySelector('[data-api-request-template]'); + var addRequestButton = form.querySelector('[data-api-request-add]'); + var diagnostics = form.querySelector('[data-api-source-diagnostics]'); + var testStatus = form.querySelector('[data-api-source-test-status]'); + var testResults = form.querySelector('[data-api-source-test-results]'); + var copyReportButton = form.querySelector('[data-api-source-copy-report]'); + var dismissDiagnosticsButton = form.querySelector('[data-api-source-dismiss-diagnostics]'); + var diagnosticReport = null; + + function addText(parent, tag, text, className) { + var element = document.createElement(tag); + element.textContent = text; + if (className) element.className = className; + parent.appendChild(element); + return element; + } + + function addPreview(parent, title, text) { + var details = document.createElement('details'); + details.className = 'mb-3'; + addText(details, 'summary', title); + var preview = addText(details, 'pre', text, 'small bg-body-tertiary border rounded p-2 mt-2 mb-0'); + preview.style.cssText = 'white-space:pre-wrap;overflow-wrap:anywhere;max-height:24rem;overflow:auto;'; + parent.appendChild(details); + } + + function showReport(report) { + testResults.replaceChildren(); + testStatus.textContent = report.ok ? 'Connection test passed. No changes saved.' : 'Connection test failed. No changes saved.'; + testStatus.className = report.ok ? 'mb-2 text-success' : 'mb-2 text-danger'; + if (report.error) addText(testResults, 'p', report.error, 'text-break'); + (report.steps || []).forEach(function (step) { + var block = document.createElement('div'); + block.className = 'border-top py-2'; + addText(block, 'h4', step.stage + ': ' + (step.skipped ? 'Not required' : step.ok ? 'Passed' : 'Failed'), 'h6'); + var metadata = [step.method, step.url, step.status ? 'HTTP ' + step.status : '', step.contentType, step.durationMs !== undefined ? step.durationMs + ' ms' : ''].filter(Boolean); + if (metadata.length) addText(block, 'p', metadata.join(' | '), 'small text-break mb-1'); + if (step.error || step.message) addText(block, 'p', step.error || step.message, 'text-break mb-1'); + if (step.response) addPreview(block, 'Sanitised response', step.response); + testResults.appendChild(block); + }); + if (report.preview) addPreview(testResults, 'JSON preview (up to 12,000 characters)', report.preview); + if (report.paths && report.paths.length) { + addPreview(testResults, 'Available field paths (sample)', report.paths.join('\n')); + } + } + + function readRequest(card) { + var request = { id: card.getAttribute('data-request-id') || null }; + Array.prototype.forEach.call(card.querySelectorAll('[data-request-field]'), function (input) { + request[input.getAttribute('data-request-field')] = input.type === 'checkbox' ? input.checked : input.value; + }); + return request; + } + + function serializeRequests() { + return Array.prototype.map.call(requestList.querySelectorAll('[data-api-request-card]'), readRequest); + } + + function updateRequestCard(card) { + var fields = readRequest(card); + card.querySelector('[data-api-request-title]').textContent = 'Request - ' + (fields.name || 'Unnamed request'); + card.querySelector('[data-api-request-body-section]').hidden = fields.request_method !== 'POST'; + } + + function updateRequestFromEvent(event) { + var card = event.target.closest('[data-api-request-card]'); + if (card) updateRequestCard(card); + } + + if (requestList) { + form.addEventListener('formdata', function (event) { + event.formData.set('requests_json', JSON.stringify(serializeRequests())); + }); + requestList.addEventListener('input', updateRequestFromEvent); + requestList.addEventListener('change', updateRequestFromEvent); + requestList.addEventListener('click', function (event) { + var remove = event.target.closest('[data-api-request-remove]'); + if (remove && !remove.disabled) { + var card = remove.closest('[data-api-request-card]'); + if (requestList.querySelectorAll('[data-api-request-card]').length === 1) return; + var requestName = card.querySelector('[data-request-field="name"]').value || 'Unnamed request'; + if (!window.confirm('Delete request "' + requestName + '"? This takes effect when you save the API source.')) return; + if (lastTestButton && lastTestButton.closest && lastTestButton.closest('[data-api-request-card]') === card) { + diagnostics.hidden = true; + form.appendChild(diagnostics); + lastTestButton = testButton; + } + card.remove(); + addRequestButton.focus(); + } + var button = event.target.closest('[data-api-source-test-url]'); + if (button && button !== testButton) testConnection({ currentTarget: button }); + updateRemoveButtons(); + }); + addRequestButton.addEventListener('click', function () { + if (requestList.querySelectorAll('[data-api-request-card]').length >= 50) return; + requestList.appendChild(requestTemplate.content.cloneNode(true)); + var cards = requestList.querySelectorAll('[data-api-request-card]'); + var input = cards[cards.length - 1].querySelector('[data-request-field="name"]'); + var names = Array.prototype.map.call(cards, function (card) { return card.querySelector('[data-request-field="name"]').value.toLowerCase(); }); + var suffix = cards.length; + while (names.indexOf('request ' + suffix) !== -1) suffix += 1; + input.value = 'Request ' + suffix; + updateRequestCard(cards[cards.length - 1]); + updateRemoveButtons(); + input.focus(); + input.select(); + }); + updateRemoveButtons(); + } + + function updateRemoveButtons() { + if (!requestList) return; + var cards = requestList.querySelectorAll('[data-api-request-card]'); + addRequestButton.disabled = cards.length >= 50; + Array.prototype.forEach.call(cards, function (card) { + var button = card.querySelector('[data-api-request-remove]'); + if (!button.hasAttribute('data-request-protected')) { + button.disabled = cards.length <= 1; + } + }); + } + + async function testConnection(event) { + if (isTesting) return; + isTesting = true; + var activeButton = event && event.currentTarget || testButton; + lastTestButton = activeButton; + var card = activeButton.closest ? activeButton.closest('[data-api-request-card]') : null; + var anchor = card || (authCard && !authCard.hidden ? authCard : activeButton.closest && activeButton.closest('.card')); + if (anchor) anchor.after(diagnostics); + diagnostics.hidden = false; + diagnostics.setAttribute('aria-busy', 'true'); + activeButton.disabled = true; + copyReportButton.disabled = true; + diagnosticReport = null; + testResults.replaceChildren(); + testStatus.className = 'mb-2'; + testStatus.textContent = 'Testing connection...'; + try { + var payload = Object.fromEntries(new FormData(form)); + if (card) Object.assign(payload, readRequest(card)); + var response = await fetch(activeButton.getAttribute('data-api-auth-test-url') || activeButton.getAttribute('data-api-source-test-url'), { + method: 'POST', credentials: 'same-origin', + headers: { 'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest' }, + body: JSON.stringify(payload) + }); + if (!(response.headers.get('content-type') || '').includes('application/json')) { + throw new Error('Connection test unavailable (HTTP ' + response.status + '). Check your session and permissions.'); + } + diagnosticReport = await response.json(); + showReport(diagnosticReport); + copyReportButton.disabled = false; + } catch (error) { + testStatus.className = 'mb-2 text-danger'; + testStatus.textContent = error.message || 'Unable to run connection test.'; + } finally { + activeButton.disabled = false; + isTesting = false; + diagnostics.setAttribute('aria-busy', 'false'); + } + } + + if (testButton) testButton.addEventListener('click', testConnection); + if (dismissDiagnosticsButton) dismissDiagnosticsButton.addEventListener('click', function () { + diagnostics.hidden = true; + if (lastTestButton && lastTestButton.isConnected !== false) lastTestButton.focus(); + }); + if (copyReportButton) copyReportButton.addEventListener('click', async function () { + if (!diagnosticReport) return; + try { + await navigator.clipboard.writeText(JSON.stringify(diagnosticReport, null, 2)); + testStatus.textContent = 'Diagnostic report copied.'; + } catch (_error) { + testStatus.textContent = 'Clipboard unavailable. Select the diagnostic results to copy them.'; + } + }); function updateBearerTokenToggle() { if (!bearerTokenInput || !bearerTokenToggle) { @@ -61,30 +240,16 @@ }); } - function updateRequestBodyVisibility() { - if (!requestMethodSelect || !requestSection || !requestBodyInput) { - return; - } - - var isPost = String(requestMethodSelect.value || 'GET').toUpperCase() === 'POST'; - requestSection.hidden = !isPost; - } - if (methodSelect) { methodSelect.addEventListener('change', function () { updatePanels(true); }); } - if (requestMethodSelect) { - requestMethodSelect.addEventListener('change', updateRequestBodyVisibility); - } - if (bearerTokenToggle && bearerTokenInput) { bearerTokenToggle.addEventListener('click', toggleBearerTokenVisibility); updateBearerTokenToggle(); } updatePanels(false); - updateRequestBodyVisibility(); }()); \ No newline at end of file diff --git a/src/web/routes/data-sources/api-sources/duplicate.js b/src/web/routes/data-sources/api-sources/duplicate.js index c365dcc..e9c16b8 100644 --- a/src/web/routes/data-sources/api-sources/duplicate.js +++ b/src/web/routes/data-sources/api-sources/duplicate.js @@ -1,4 +1,5 @@ // API source duplication helpers. +const { normalizeApiSourceForForm } = require('#src/web/routes/data-sources/api-sources/form-view-model'); function buildDuplicateApiSourceName(sourceName) { const baseName = 'Copy of ' + String(sourceName || '').trim(); @@ -6,33 +7,21 @@ function buildDuplicateApiSourceName(sourceName) { } function buildDuplicateApiSource(apiSource, duplicateName) { - return Object.assign({}, apiSource, { + const duplicate = Object.assign(normalizeApiSourceForForm(apiSource), { id: null, name: duplicateName, - apiUrl: apiSource.api_url || '', lastPulledAt: null, lastPullError: '', lastResponseStatus: null, lastResponseContentType: '', - lastResponseJson: '', - authMethod: apiSource.auth_method || 'none', - authUsername: apiSource.auth_username || '', - authPassword: apiSource.auth_password || '', - authBearerToken: apiSource.auth_bearer_token || '', - authHeaderName: apiSource.auth_header_name || 'X-API-Key', - authHeaderValue: apiSource.auth_header_value || '', - requestMethod: apiSource.request_method || 'GET', - requestBodyJson: apiSource.request_body_json || '', - tokenUrl: apiSource.token_url || '', - tokenRequestBodyJson: apiSource.token_request_body_json || '', - tokenResponsePath: apiSource.token_response_path || 'access_token', - tokenRefreshUrl: apiSource.token_refresh_url || '', - tokenRefreshRequestBodyJson: apiSource.token_refresh_request_body_json || '', - tokenRefreshResponsePath: apiSource.token_refresh_response_path || 'refresh_token', - tokenHeaderName: apiSource.token_header_name || 'Authorization', - tokenHeaderPrefix: apiSource.token_header_prefix || 'Bearer', - itemsPath: apiSource.items_path || '' + lastResponseJson: '' }); + if (Array.isArray(apiSource.requests)) { + duplicate.requests = apiSource.requests.map(function (request) { + return { name: request.name, api_url: request.api_url, request_method: request.request_method, request_body_json: request.request_body_json, items_path: request.items_path, enabled: request.enabled, update_interval_value: request.update_interval_value, update_interval_unit: request.update_interval_unit }; + }); + } + return duplicate; } module.exports = { diff --git a/src/web/routes/data-sources/api-sources/edit.js b/src/web/routes/data-sources/api-sources/edit.js index 1268ef4..41ccf38 100644 --- a/src/web/routes/data-sources/api-sources/edit.js +++ b/src/web/routes/data-sources/api-sources/edit.js @@ -4,30 +4,5 @@ const { renderView } = require('#src/web/view'); const { buildApiSourceFormViewModel } = require('#src/web/routes/data-sources/api-sources/form-view-model'); module.exports = function renderApiSourceEditPage(apiSource, data, message, currentUser) { - const viewData = Object.assign({ - lastResponseJson: '', - lastPullError: '' - }, data || {}); - - return renderView('data-sources/api-sources/form', buildApiSourceFormViewModel(Object.assign({}, apiSource, { - lastResponseJson: viewData.lastResponseJson, - lastPullError: viewData.lastPullError, - authMethod: apiSource.auth_method || 'none', - authUsername: apiSource.auth_username || '', - authPassword: apiSource.auth_password || '', - authBearerToken: apiSource.auth_bearer_token || '', - authHeaderName: apiSource.auth_header_name || 'X-API-Key', - authHeaderValue: apiSource.auth_header_value || '', - tokenUrl: apiSource.token_url || '', - tokenRequestBodyJson: apiSource.token_request_body_json || '', - tokenResponsePath: apiSource.token_response_path || 'access_token', - tokenRefreshUrl: apiSource.token_refresh_url || '', - tokenRefreshRequestBodyJson: apiSource.token_refresh_request_body_json || '', - tokenRefreshResponsePath: apiSource.token_refresh_response_path || 'refresh_token', - tokenHeaderName: apiSource.token_header_name || 'Authorization', - tokenHeaderPrefix: apiSource.token_header_prefix || 'Bearer', - itemsPath: apiSource.items_path || '', - requestMethod: apiSource.request_method || 'GET', - requestBodyJson: apiSource.request_body_json || '' - }), message, currentUser, viewData, true)); + return renderView('data-sources/api-sources/form', buildApiSourceFormViewModel(apiSource, message, currentUser, data, true)); }; \ No newline at end of file diff --git a/src/web/routes/data-sources/api-sources/form-view-model.js b/src/web/routes/data-sources/api-sources/form-view-model.js index 2035ccc..9f3f0c0 100644 --- a/src/web/routes/data-sources/api-sources/form-view-model.js +++ b/src/web/routes/data-sources/api-sources/form-view-model.js @@ -33,9 +33,28 @@ function buildDefaultApiSource() { }; } +function normalizeApiSourceForForm(apiSource) { + const source = apiSource || {}; + const defaults = buildDefaultApiSource(); + const values = Object.assign({}, source); + Object.keys(defaults).forEach(function (key) { + const field = key.replace(/[A-Z]/g, function (letter) { return '_' + letter.toLowerCase(); }); + const value = source[key] !== undefined ? source[key] : source[field]; + values[key] = value === undefined || value === null ? defaults[key] : value; + }); + return values; +} + function buildApiSourceFormViewModel(apiSource, message, currentUser, options, isEdit) { - const viewApiSource = Object.assign(buildDefaultApiSource(), apiSource || {}); + const viewApiSource = normalizeApiSourceForForm(apiSource); const viewOptions = options || {}; + function requestModel(request) { + return Object.assign({ name: 'Request 1', api_url: '', request_method: 'GET', request_body_json: '', items_path: '', enabled: 1, update_interval_value: viewApiSource.updateIntervalValue, update_interval_unit: viewApiSource.updateIntervalUnit }, request, { + lastResponseJson: typeof request.last_response_json === 'object' && request.last_response_json ? JSON.stringify(request.last_response_json, null, 2) : request.last_response_json || '', + lastPulledAtValue: request.last_pulled_at ? new Date(request.last_pulled_at).toISOString() : '' + }); + } + const requests = (viewApiSource.requests || [{ id: viewApiSource.id, name: 'Request 1', api_url: viewApiSource.apiUrl, request_method: viewApiSource.requestMethod, request_body_json: viewApiSource.requestBodyJson, items_path: viewApiSource.itemsPath }]).map(requestModel); return { title: isEdit ? 'Edit API source' : 'Add API source', @@ -43,6 +62,9 @@ function buildApiSourceFormViewModel(apiSource, message, currentUser, options, i message: message, currentUser: currentUser || null, apiSource: viewApiSource, + apiRequests: requests, + newRequest: requestModel({ name: 'New request' }), + requestsJson: JSON.stringify(requests), inUse: Boolean(viewApiSource.inUse), lastResponseJson: viewOptions.lastResponseJson || viewApiSource.lastResponseJson || '', lastPullError: viewOptions.lastPullError || viewApiSource.lastPullError || '', @@ -51,12 +73,13 @@ function buildApiSourceFormViewModel(apiSource, message, currentUser, options, i showSaveSecondaryActions: true, deleteDisabled: !isEdit || Boolean(viewApiSource.inUse), formAction: isEdit && viewApiSource.id ? '/data-sources/api-sources/' + viewApiSource.id : '/data-sources/api-sources', - formAttrs: isEdit && viewApiSource.id ? 'data-async-save data-async-save-refresh-target="#api-source-response-panel" data-async-save-refresh-state-url="/data-sources/api-sources/' + viewApiSource.id + '/state" data-async-save-close-url="/data-sources/api-sources" data-async-save-new-url="/data-sources/api-sources/new"' : 'data-async-save data-async-save-close-url="/data-sources/api-sources" data-async-save-new-redirect="response-url" data-async-save-new-url="/data-sources/api-sources/new"', + formAttrs: '', cancelUrl: '/data-sources/api-sources', deleteUrl: isEdit && viewApiSource.id ? '/data-sources/api-sources/' + viewApiSource.id + '/delete' : '' }; } module.exports = { - buildApiSourceFormViewModel: buildApiSourceFormViewModel + buildApiSourceFormViewModel: buildApiSourceFormViewModel, + normalizeApiSourceForForm: normalizeApiSourceForForm }; \ No newline at end of file diff --git a/src/web/routes/data-sources/api-sources/routes.js b/src/web/routes/data-sources/api-sources/routes.js index 59745ae..b858c41 100644 --- a/src/web/routes/data-sources/api-sources/routes.js +++ b/src/web/routes/data-sources/api-sources/routes.js @@ -1,385 +1,151 @@ -// API source route registration. - const { buildPagination } = require('#src/web/lib/pagination'); const renderApiSourcesPage = require('#src/web/routes/data-sources/api-sources/list'); const renderApiSourceAddPage = require('#src/web/routes/data-sources/api-sources/add'); const renderApiSourceEditPage = require('#src/web/routes/data-sources/api-sources/edit'); const { buildDuplicateApiSourceName, buildDuplicateApiSource } = require('#src/web/routes/data-sources/api-sources/duplicate'); const { fetchAppSettings } = require('#src/data/app-settings'); - -function toIsoTimestamp(value) { - if (!value) { - return ''; - } - - const date = new Date(value); - return Number.isNaN(date.getTime()) ? '' : date.toISOString(); -} - -async function getDataSourceUsageMaps(pool, common) { - const [slides] = await pool.query('SELECT content_json FROM c_slides WHERE content_json IS NOT NULL'); - const apiSourceIds = new Set(); - - for (let index = 0; index < slides.length; index += 1) { - const content = typeof common.parseJsonSafe === 'function' ? common.parseJsonSafe(slides[index].content_json) : null; - if (!content || typeof content !== 'object') { - continue; - } - - (function walk(value) { - if (!value || typeof value !== 'object') { - return; - } - - if (Array.isArray(value)) { - value.forEach(walk); - return; - } - - if (Object.prototype.hasOwnProperty.call(value, 'source_id')) { - const sourceId = Number(value.source_id); - if (Number.isFinite(sourceId)) { - apiSourceIds.add(sourceId); - } - } - - Object.keys(value).forEach(function (key) { - walk(value[key]); - }); - })(content); - } - - return apiSourceIds; -} - -async function isApiSourceInUse(pool, common, apiSourceId) { - const usageIds = await getDataSourceUsageMaps(pool, common); - return usageIds.has(Number(apiSourceId)); -} - -async function sendRefreshTaskState(req, res, dataSourceTasks, sourceId) { - const taskId = Number(req.query.refresh_task_id); - if (!Number.isFinite(taskId) || taskId <= 0) { - return res.status(400).json({ error: 'Missing refresh task id.' }); - } - - const task = await dataSourceTasks.getTaskStatusById(taskId); - const expectedKey = 'api-source-refresh:' + Number(sourceId); - if (!task || task.key !== expectedKey) { - return res.status(404).json({ error: 'Refresh task not found.' }); - } - - res.json(task); -} +const { buildApiSourcePayload, testApiSourceConnection, testApiSourceAuthentication } = require('#src/data/api-sources'); +const { buildApiGroupPayload, fetchApiGroupById, fetchApiGroupsPage, getApiRequestUsage, saveApiGroup } = require('#src/data/api-source-groups'); +const { hasPermission } = require('#src/rbac'); module.exports = function registerApiSourceRoutes(app, deps) { - const pool = deps.pool; - const common = deps.common; - const pages = deps.pages; - const backgroundTaskQueue = deps.backgroundTaskQueue; - const dataSourceTasks = deps.dataSourceTasks; - const getAuditUserId = deps.getAuditUserId; - const redirectAfterSave = deps.redirectAfterSave; - const formatDashboardDate = deps.formatDashboardDate || function (value) { - return value ? String(value) : ''; - }; - const requirePermission = deps.requirePermission; + const { pool, common, backgroundTaskQueue, dataSourceTasks, getAuditUserId, redirectAfterSave, requirePermission } = deps; + const formatDate = deps.formatDashboardDate || String; + const base = '/data-sources/api-sources'; - const LIST_PAGE_SIZE = 25; + function syncSchedules(group, removed, actorId) { + (removed || []).forEach(function (id) { dataSourceTasks.removeRecurringRefresh('api-source', id); }); + group.requests.forEach(function (request) { + if (!request.enabled) return dataSourceTasks.removeRecurringRefresh('api-source', request.id); + dataSourceTasks.registerRecurringRefresh('api-source', request.id, group.name + ' / ' + request.name, request.update_interval_value || request.updateIntervalValue, request.update_interval_unit || request.updateIntervalUnit, function () { + return dataSourceTasks.refreshApiSourceInBackground(request.id, actorId); + }); + }); + } - app.get('/data-sources/api-sources', requirePermission('api-sources.read'), async function (req, res, next) { + async function enqueue(request, group, actorId) { + return backgroundTaskQueue.enqueueTask({ + key: 'api-source-refresh:' + request.id, title: 'API request refresh', category: 'data-source', taskType: 'data-source-refresh', + payload: { sourceType: 'api-source', sourceId: request.id, sourceName: group.name + ' / ' + request.name, actorId: actorId }, + metadata: { sourceType: 'api-source', sourceId: request.id, sourceName: group.name + ' / ' + request.name } + }); + } + + async function testConnection(req, res) { + res.set('Cache-Control', 'no-store'); + try { + const body = Object.assign({}, req.body, { name: 'Connection test' }); + const authOnly = String(req.path || '').endsWith('/auth-test'); + if (authOnly) body.api_url = 'https://connection.test'; + const payload = buildApiSourcePayload({ body: body }); + res.json(await (authOnly ? testApiSourceAuthentication(payload) : testApiSourceConnection(payload))); + } catch (error) { + res.status(error.statusCode === 400 ? 400 : 500).json({ ok: false, error: error.statusCode === 400 ? error.message : 'Unable to test API connection.', steps: [] }); + } + } + app.post(base + '/test', requirePermission('api-sources.create'), testConnection); + app.post(base + '/auth-test', requirePermission('api-sources.create'), testConnection); + app.post(base + '/:id/test', requirePermission('api-sources.update'), testConnection); + app.post(base + '/:id/auth-test', requirePermission('api-sources.update'), testConnection); + + app.get(base, requirePermission('api-sources.read'), async function (req, res, next) { try { - const page = Math.max(1, Math.floor(Number(req.query.page) || 1)); const search = common.getSearchQuery(req); const sort = common.getSortQuery(req); const direction = common.getSortDirectionQuery(req); - const data = await common.fetchApiSourcesPage(pool, page, LIST_PAGE_SIZE, search, sort, direction); - const usageIds = await getDataSourceUsageMaps(pool, common); - const apiSources = (data.apiSources || []).map(function (apiSource) { - return Object.assign({}, apiSource, { - authMethod: apiSource.auth_method || 'none', - authUsername: apiSource.auth_username || '', - authPassword: apiSource.auth_password || '', - authBearerToken: apiSource.auth_bearer_token || '', - authHeaderName: apiSource.auth_header_name || 'X-API-Key', - authHeaderValue: apiSource.auth_header_value || '', - tokenUrl: apiSource.token_url || '', - tokenResponsePath: apiSource.token_response_path || 'access_token', - tokenRefreshUrl: apiSource.token_refresh_url || '', - tokenRefreshRequestBodyJson: apiSource.token_refresh_request_body_json || '', - tokenRefreshResponsePath: apiSource.token_refresh_response_path || 'refresh_token', - itemsPath: apiSource.items_path || '', - intervalLabel: apiSource.update_interval_unit === 'seconds' - ? (Math.max(1, Number(apiSource.update_interval_value) || 0) === 1 ? 'Every second' : `Every ${Math.max(1, Number(apiSource.update_interval_value) || 0)} seconds`) - : apiSource.update_interval_unit === 'hours' - ? (Math.max(1, Number(apiSource.update_interval_value) || 0) === 1 ? 'Every hour' : `Every ${Math.max(1, Number(apiSource.update_interval_value) || 0)} hours`) - : (Math.max(1, Number(apiSource.update_interval_value) || 0) === 1 ? 'Every minute' : `Every ${Math.max(1, Number(apiSource.update_interval_value) || 0)} minutes`), - lastPullLabel: apiSource.last_pulled_at ? formatDashboardDate(apiSource.last_pulled_at) : 'Never', - lastPulledAtValue: apiSource.last_pulled_at ? new Date(apiSource.last_pulled_at).toISOString() : '', - inUse: usageIds.has(Number(apiSource.id)) - }); - }); - - res.send(renderApiSourcesPage({ - apiSources: apiSources, - pagination: buildPagination(data.totalItems, data.currentPage, 'page', { search: search, sort: sort, direction: direction }, LIST_PAGE_SIZE, 'API sources', 'API source pages') - }, req.query.message ? String(req.query.message) : '', req.currentUser, formatDashboardDate)); - } catch (error) { - next(error); - } - }); - - app.get('/data-sources/api-sources/new', requirePermission('api-sources.create'), async function (req, res, next) { - try { - const settings = await fetchAppSettings(pool); - res.send(renderApiSourceAddPage({ - updateIntervalValue: settings['data-sources.api_default_interval_value'], - updateIntervalUnit: settings['data-sources.api_default_interval_unit'] - }, req.query.message ? String(req.query.message) : '', req.currentUser)); - } catch (error) { - next(error); - } - }); - - app.get('/data-sources/api-sources/:id/edit', requirePermission('api-sources.update'), async function (req, res, next) { - try { - const apiSource = await common.fetchApiSourceById(pool, Number(req.params.id)); - if (!apiSource) { - return res.status(404).send('API source not found'); - } - - const [slides] = await pool.query('SELECT content_json FROM c_slides WHERE content_json IS NOT NULL'); - const inUse = (slides || []).some(function (row) { - const content = typeof common.parseJsonSafe === 'function' ? common.parseJsonSafe(row.content_json) : null; - if (!content || typeof content !== 'object') { - return false; - } - - const stack = [content]; - while (stack.length) { - const value = stack.pop(); - if (!value || typeof value !== 'object') { - continue; - } - if (Array.isArray(value)) { - value.forEach(function (item) { stack.push(item); }); - continue; - } - if (Object.prototype.hasOwnProperty.call(value, 'source_id') && Number(value.source_id) === Number(apiSource.id)) { - return true; - } - Object.keys(value).forEach(function (key) { - stack.push(value[key]); - }); - } - return false; - }); - - res.send(renderApiSourceEditPage(Object.assign({}, apiSource, { - apiUrl: apiSource.api_url, - authMethod: apiSource.auth_method || 'none', - authUsername: apiSource.auth_username || '', - authPassword: apiSource.auth_password || '', - authBearerToken: apiSource.auth_bearer_token || '', - authHeaderName: apiSource.auth_header_name || 'X-API-Key', - authHeaderValue: apiSource.auth_header_value || '', - tokenUrl: apiSource.token_url || '', - tokenRequestBodyJson: apiSource.token_request_body_json || '', - tokenResponsePath: apiSource.token_response_path || 'access_token', - tokenRefreshUrl: apiSource.token_refresh_url || '', - tokenRefreshRequestBodyJson: apiSource.token_refresh_request_body_json || '', - tokenRefreshResponsePath: apiSource.token_refresh_response_path || 'refresh_token', - tokenHeaderName: apiSource.token_header_name || 'Authorization', - tokenHeaderPrefix: apiSource.token_header_prefix || 'Bearer', - itemsPath: apiSource.items_path || '', - updateIntervalValue: apiSource.update_interval_value, - updateIntervalUnit: apiSource.update_interval_unit || 'minutes', - lastPulledAtValue: toIsoTimestamp(apiSource.last_pulled_at), - lastPulledAtLabel: apiSource.last_pulled_at ? String(apiSource.last_pulled_at) : '', - lastPullError: apiSource.last_pull_error || '', - lastResponseStatus: apiSource.last_response_status, - lastResponseContentType: apiSource.last_response_content_type, - lastResponseJson: apiSource.last_response_json || '', - inUse: inUse - }), { - lastResponseJson: apiSource.last_response_json || '', - lastPullError: apiSource.last_pull_error || '' - }, req.query.message ? String(req.query.message) : '', req.currentUser)); - } catch (error) { - next(error); - } - }); - - app.get('/data-sources/api-sources/:id/duplicate', requirePermission('api-sources.read'), requirePermission('api-sources.create'), async function (req, res, next) { - try { - const apiSource = await common.fetchApiSourceById(pool, Number(req.params.id)); - if (!apiSource) { - return res.status(404).send('API source not found'); - } - - let duplicateName = buildDuplicateApiSourceName(apiSource.name); - let duplicateIndex = 2; - while (await common.fetchDuplicateName(pool, 'i_api_sources', duplicateName)) { - duplicateName = buildDuplicateApiSourceName(apiSource.name) + ' (' + duplicateIndex + ')'; - duplicateIndex += 1; - } - - res.send(renderApiSourceAddPage(buildDuplicateApiSource(apiSource, duplicateName), req.query.message ? String(req.query.message) : 'Review the copied values and save when ready.', req.currentUser, { - messageVariant: 'info', - showSaveSecondaryActions: true - })); - } catch (error) { - next(error); - } - }); - - app.get('/data-sources/api-sources/:id/state', requirePermission('api-sources.update'), function (req, res, next) { - sendRefreshTaskState(req, res, dataSourceTasks, Number(req.params.id)).catch(next); - }); - - app.post('/data-sources/api-sources', requirePermission('api-sources.create'), async function (req, res, next) { - const connection = await pool.getConnection(); - try { - const payload = common.buildApiSourcePayload(req, null); - if (await common.fetchDuplicateName(pool, 'i_api_sources', payload.name)) { - return res.redirect('/data-sources/api-sources/new?message=' + encodeURIComponent('An API source with that name already exists.')); - } - - const actorId = getAuditUserId(req); - await connection.beginTransaction(); - const [result] = await connection.query( - 'INSERT INTO i_api_sources (name, api_url, request_method, request_body_json, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, token_url, token_request_body_json, token_response_path, token_refresh_url, token_refresh_request_body_json, token_refresh_response_path, token_header_name, token_header_prefix, items_path, update_interval_value, update_interval_unit, last_pulled_at, last_pull_error, last_response_status, last_response_content_type, last_response_json, created_by, modified_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', - [payload.name, payload.apiUrl, payload.requestMethod, payload.requestBodyJson || null, payload.authMethod, payload.authUsername || null, payload.authPassword || null, payload.authBearerToken || null, payload.authHeaderName || null, payload.authHeaderValue || null, payload.tokenUrl || null, payload.tokenRequestBodyJson || null, payload.tokenResponsePath || null, payload.tokenRefreshUrl || null, payload.tokenRefreshRequestBodyJson || null, payload.tokenRefreshResponsePath || null, payload.tokenHeaderName || null, payload.tokenHeaderPrefix || null, payload.itemsPath || null, payload.updateIntervalValue, payload.updateIntervalUnit, null, null, null, null, null, actorId, actorId] - ); - await connection.commit(); - dataSourceTasks.registerRecurringRefresh('api-source', result.insertId, payload.name, payload.updateIntervalValue, payload.updateIntervalUnit, function () { - return dataSourceTasks.refreshApiSourceInBackground(result.insertId, actorId); - }); - const refreshTask = await backgroundTaskQueue.enqueueTask({ - key: 'api-source-refresh:' + result.insertId, - title: 'API source refresh', - category: 'data-source', - taskType: 'data-source-refresh', - payload: { - sourceType: 'api-source', - sourceId: result.insertId, - sourceName: payload.name, - actorId: actorId - }, - metadata: { - sourceType: 'api-source', - sourceId: result.insertId, - sourceName: payload.name - } - }); - const message = 'API source created. Refresh is running in the background.'; - res.redirect('/data-sources/api-sources/' + result.insertId + '/edit?message=' + encodeURIComponent(message) + '&refresh_task_id=' + encodeURIComponent(refreshTask && refreshTask.id ? refreshTask.id : '')); - } catch (error) { - try { - await connection.rollback(); - } catch (_rollbackError) { - // Ignore rollback failures and surface the original error. - } - next(error); - } finally { - connection.release(); - } - }); - - app.post('/data-sources/api-sources/:id', requirePermission('api-sources.update'), async function (req, res, next) { - const connection = await pool.getConnection(); - try { - const apiSource = await common.fetchApiSourceById(pool, Number(req.params.id)); - if (!apiSource) { - return res.status(404).send('API source not found'); - } - - if (String(req.body && req.body.data_source_action || '').trim() === 'toggle') { - const enabled = apiSource.enabled === 0 || apiSource.enabled === false ? 1 : 0; - await connection.query('UPDATE i_api_sources SET enabled = ?, modified_by = ? WHERE id = ?', [enabled, getAuditUserId(req), apiSource.id]); - if (enabled) { - dataSourceTasks.registerRecurringRefresh('api-source', apiSource.id, apiSource.name, apiSource.update_interval_value, apiSource.update_interval_unit, function () { - return dataSourceTasks.refreshApiSourceInBackground(apiSource.id, getAuditUserId(req)); - }); - } else { - dataSourceTasks.removeRecurringRefresh('api-source', apiSource.id); - } - return res.redirect('/data-sources/api-sources/' + apiSource.id + '/edit?message=' + encodeURIComponent(enabled ? 'API source enabled.' : 'API source disabled.')); - } - - const payload = common.buildApiSourcePayload(req, apiSource); - if (await common.fetchDuplicateName(pool, 'i_api_sources', payload.name, apiSource.id)) { - return res.redirect('/data-sources/api-sources/' + apiSource.id + '/edit?message=' + encodeURIComponent('An API source with that name already exists.')); - } - const actorId = getAuditUserId(req); - await connection.beginTransaction(); - await connection.query( - 'UPDATE i_api_sources SET name = ?, api_url = ?, request_method = ?, request_body_json = ?, auth_method = ?, auth_username = ?, auth_password = ?, auth_bearer_token = ?, auth_header_name = ?, auth_header_value = ?, token_url = ?, token_request_body_json = ?, token_response_path = ?, token_refresh_url = ?, token_refresh_request_body_json = ?, token_refresh_response_path = ?, token_header_name = ?, token_header_prefix = ?, items_path = ?, update_interval_value = ?, update_interval_unit = ?, modified_by = ? WHERE id = ?', - [payload.name, payload.apiUrl, payload.requestMethod, payload.requestBodyJson || null, payload.authMethod, payload.authUsername || null, payload.authPassword || null, payload.authBearerToken || null, payload.authHeaderName || null, payload.authHeaderValue || null, payload.tokenUrl || null, payload.tokenRequestBodyJson || null, payload.tokenResponsePath || null, payload.tokenRefreshUrl || null, payload.tokenRefreshRequestBodyJson || null, payload.tokenRefreshResponsePath || null, payload.tokenHeaderName || null, payload.tokenHeaderPrefix || null, payload.itemsPath || null, payload.updateIntervalValue, payload.updateIntervalUnit, actorId, apiSource.id] - ); - await connection.commit(); - if (apiSource.enabled === 0 || apiSource.enabled === false) { - dataSourceTasks.removeRecurringRefresh('api-source', apiSource.id); - } else { - dataSourceTasks.registerRecurringRefresh('api-source', apiSource.id, payload.name, payload.updateIntervalValue, payload.updateIntervalUnit, function () { - return dataSourceTasks.refreshApiSourceInBackground(apiSource.id, actorId); - }); - } - redirectAfterSave(req, res, '/data-sources/api-sources/' + apiSource.id + '/edit', { - closeUrl: '/data-sources/api-sources', - newUrl: '/data-sources/api-sources/new', - message: 'API source updated.' - }); - } catch (error) { - try { - await connection.rollback(); - } catch (_rollbackError) { - // Ignore rollback failures and surface the original error. - } - next(error); - } finally { - connection.release(); - } - }); - - app.post('/data-sources/api-sources/:id/refresh', requirePermission('api-sources.allow'), async function (req, res, next) { - try { - const apiSource = await common.fetchApiSourceById(pool, Number(req.params.id)); - if (!apiSource) return res.status(404).send('API source not found'); - if (apiSource.enabled === 0 || apiSource.enabled === false) { - return res.redirect('/data-sources/api-sources/' + apiSource.id + '/edit?message=' + encodeURIComponent('API source is disabled.')); - } - await backgroundTaskQueue.enqueueTask({ key: 'api-source-refresh:' + apiSource.id, title: 'API source refresh', category: 'data-source', taskType: 'data-source-refresh', payload: { sourceType: 'api-source', sourceId: apiSource.id, sourceName: apiSource.name, actorId: getAuditUserId(req) }, metadata: { sourceType: 'api-source', sourceId: apiSource.id, sourceName: apiSource.name } }); - res.redirect('/data-sources/api-sources/' + apiSource.id + '/edit?message=' + encodeURIComponent('API source refresh queued.')); + const data = await fetchApiGroupsPage(pool, Math.max(1, Number(req.query.page) || 1), 25, search, sort, direction); + res.send(renderApiSourcesPage({ apiSources: data.apiSources.map(function (group) { + return Object.assign({}, group, { lastPullLabel: group.last_pulled_at ? formatDate(group.last_pulled_at) : 'Never', lastPulledAtValue: group.last_pulled_at ? new Date(group.last_pulled_at).toISOString() : '' }); + }), pagination: buildPagination(data.totalItems, data.currentPage, 'page', { search, sort, direction }, 25, 'API sources', 'API source pages') }, req.query.message || '', req.currentUser, formatDate)); } catch (error) { next(error); } }); - app.post('/data-sources/api-sources/:id/delete', requirePermission('api-sources.delete'), async function (req, res, next) { + app.get(base + '/new', requirePermission('api-sources.create'), async function (req, res, next) { try { - const apiSource = await common.fetchApiSourceById(pool, Number(req.params.id)); - if (!apiSource) { - return res.status(404).send('API source not found'); - } + const settings = await fetchAppSettings(pool); + res.send(renderApiSourceAddPage({ updateIntervalValue: settings['data-sources.api_default_interval_value'], updateIntervalUnit: settings['data-sources.api_default_interval_unit'] }, req.query.message || '', req.currentUser)); + } catch (error) { next(error); } + }); - if (await isApiSourceInUse(pool, common, apiSource.id)) { - return res.redirect('/data-sources/api-sources?message=' + encodeURIComponent('This API source is still used by one or more slides.')); - } + app.get(base + '/:id/edit', requirePermission('api-sources.update'), async function (req, res, next) { + try { + const group = await fetchApiGroupById(pool, Number(req.params.id)); + if (!group) return res.status(404).send('API source not found'); + const usage = await getApiRequestUsage(pool); + group.requests = group.requests.map(function (request) { return Object.assign(request, { inUse: usage.has(Number(request.id)) }); }); + group.inUse = group.requests.some(function (request) { return request.inUse; }); + res.send(renderApiSourceEditPage(group, {}, req.query.message || '', req.currentUser)); + } catch (error) { next(error); } + }); - const connection = await pool.getConnection(); - try { - await connection.beginTransaction(); - await connection.query('DELETE FROM i_api_sources WHERE id = ?', [apiSource.id]); - await connection.commit(); - dataSourceTasks.removeRecurringRefresh('api-source', apiSource.id); - } catch (error) { + app.get(base + '/:id/duplicate', requirePermission('api-sources.read'), requirePermission('api-sources.create'), async function (req, res, next) { + try { + const group = await fetchApiGroupById(pool, Number(req.params.id)); + if (!group) return res.status(404).send('API source not found'); + const initialName = buildDuplicateApiSourceName(group.name); + let name = initialName; + let suffix = 2; + while (await common.fetchDuplicateName(pool, 'i_api_source_groups', name)) name = initialName + ' (' + suffix++ + ')'; + const duplicate = buildDuplicateApiSource(group, name); + res.send(renderApiSourceAddPage(duplicate, 'Review the copied values and save when ready.', req.currentUser, { messageVariant: 'info' })); + } catch (error) { next(error); } + }); + + async function save(req, res, next) { + try { + const id = req.params.id ? Number(req.params.id) : null; + if (req.body.data_source_action === 'toggle') { + const group = await fetchApiGroupById(pool, id); + if (!group) return res.status(404).send('API source not found'); + const enabled = group.requests.some(function (request) { return request.enabled; }) ? 0 : 1; + await pool.query('UPDATE i_api_sources SET enabled = ?, modified_by = ? WHERE auth_source_id = ?', [enabled, getAuditUserId(req), id]); + group.requests.forEach(function (request) { request.enabled = enabled; }); + syncSchedules(group, [], getAuditUserId(req)); + return res.redirect(base + '/' + id + '/edit?message=' + encodeURIComponent(enabled ? 'API requests enabled.' : 'API requests disabled.')); + } + const payload = buildApiGroupPayload(req.body); + const result = await saveApiGroup(pool, id, payload, getAuditUserId(req), hasPermission(req.currentUser, 'api-sources.delete')); + syncSchedules({ name: payload.name, requests: result.requests }, result.removed, getAuditUserId(req)); + if (!id) { + for (const request of result.requests) { if (request.enabled) await enqueue(request, payload, getAuditUserId(req)); } + } + redirectAfterSave(req, res, base + '/' + result.id + '/edit', { closeUrl: base, newUrl: base + '/new', message: id ? 'API source updated.' : 'API source created. Initial requests queued.' }); + } catch (error) { next(error); } + } + app.post(base, requirePermission('api-sources.create'), save); + app.post(base + '/:id', requirePermission('api-sources.update'), save); + + async function refresh(req, res, next) { + try { + const group = await fetchApiGroupById(pool, Number(req.params.id)); + if (!group) return res.status(404).send('API source not found'); + const requests = req.params.requestId ? group.requests.filter(function (request) { return Number(request.id) === Number(req.params.requestId); }) : group.requests; + if (!requests.length) return res.status(404).send('API request not found'); + for (const request of requests) { if (request.enabled) await enqueue(request, group, getAuditUserId(req)); } + res.redirect(base + '/' + group.id + '/edit?message=' + encodeURIComponent('Enabled API request refreshes queued.')); + } catch (error) { next(error); } + } + app.post(base + '/:id/refresh', requirePermission('api-sources.allow'), refresh); + app.post(base + '/:id/requests/:requestId/refresh', requirePermission('api-sources.allow'), refresh); + + app.post(base + '/:id/delete', requirePermission('api-sources.delete'), async function (req, res, next) { + const connection = await pool.getConnection(); + try { + await connection.beginTransaction(); + const [groups] = await connection.query('SELECT id FROM i_api_source_groups WHERE id = ? FOR UPDATE', [Number(req.params.id)]); + if (!groups.length) { await connection.rollback(); return res.status(404).send('API source not found'); } + const [requests] = await connection.query('SELECT id FROM i_api_sources WHERE auth_source_id = ? FOR UPDATE', [Number(req.params.id)]); + const usage = await getApiRequestUsage(connection); + if (requests.some(function (request) { return usage.has(Number(request.id)); })) { await connection.rollback(); - throw error; - } finally { - connection.release(); + return res.redirect(base + '?message=' + encodeURIComponent('This API source has requests used by slides.')); } - res.redirect('/data-sources/api-sources?message=' + encodeURIComponent('API source deleted.')); - } catch (error) { - next(error); - } + await connection.query('DELETE FROM i_api_sources WHERE auth_source_id = ?', [Number(req.params.id)]); + await connection.query('DELETE FROM i_api_source_groups WHERE id = ?', [Number(req.params.id)]); + await connection.commit(); + requests.forEach(function (request) { dataSourceTasks.removeRecurringRefresh('api-source', request.id); }); + res.redirect(base + '?message=' + encodeURIComponent('API source deleted.')); + } catch (error) { await connection.rollback(); next(error); } + finally { connection.release(); } }); }; \ No newline at end of file diff --git a/src/web/views/data-sources/api-sources/form.hbs b/src/web/views/data-sources/api-sources/form.hbs index 5201e1f..8d88bca 100644 --- a/src/web/views/data-sources/api-sources/form.hbs +++ b/src/web/views/data-sources/api-sources/form.hbs @@ -1,50 +1,67 @@ +{{#*inline "api-request-card"}} +
+
+

Request - {{name}}

+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+ {{#if id}} +
+
+

Latest response{{#if last_response_status}} HTTP {{last_response_status}}{{/if}}

+ {{#if lastResponseJson}} +
+ + +
+ {{/if}} +
+ {{#if last_pull_error}}
{{last_pull_error}}
{{/if}} +
Last pulled
{{#if lastPulledAtValue}}{{else}}Never{{/if}}
Content type
{{last_response_content_type}}
+ {{#if lastResponseJson}}
{{lastResponseJson}}
{{else}}

No JSON response is stored yet.

{{/if}} +
+ {{/if}} +
+
+ + {{#if id}}{{#if (hasPermission currentUser 'api-sources.allow')}}{{/if}}{{/if}} +
+
+{{/inline}}

{{#if isEdit}}Edit API source - {{apiSource.name}}{{else}}Add API source{{/if}}

-

{{#if isEdit}}Adjust the endpoint or refresh cadence and review the latest JSON response.{{else}}Adjust the endpoint or refresh cadence for this API source.{{/if}}

+

{{#if isEdit}}Manage shared authentication, request URLs, and latest responses.{{else}}Configure shared authentication and the requests for this API source.{{/if}}

-{{#if isEdit}}
{{/if}} -
+

Connection

-
+
-
- - -
-
- - -
-
-
-

Request

-
-
-
POST requests use JSON with the existing authentication settings.
- -
-
-
-
-

Access and parsing

-
-
+
-
- - -
-
-
-
-
Optional. Use dot notation to change the default base for placeholders.
-
-
-

Refresh

-
-
- - -
-
- - -
+
+
+ + +
+
+ {{{saveActionButtons formId="api-source-form" saveUrl=formAction cancelUrl=cancelUrl deleteUrl=deleteUrl deleteDisabled=deleteDisabled showSaveAndClose=showSaveSecondaryActions showSaveAndNew=showSaveSecondaryActions}}}
-
-
{{#if isEdit}}{{#if (hasPermission currentUser "api-sources.update")}}{{/if}} {{#if (hasPermission currentUser "api-sources.allow")}}{{/if}}{{/if}}
- {{{saveActionButtons formId="api-source-form" saveUrl=formAction cancelUrl=cancelUrl deleteUrl=deleteUrl deleteDisabled=deleteDisabled showSaveAndClose=showSaveSecondaryActions showSaveAndNew=showSaveSecondaryActions}}} -
-
-