424 lines
18 KiB
JavaScript
424 lines
18 KiB
JavaScript
// API source data access, pagination, and remote fetch helpers.
|
|
|
|
const http = require('http');
|
|
const https = require('https');
|
|
const { fetchPagedRows, validateMaxLength } = require('./utils');
|
|
|
|
const NAME_MAX_LENGTH = 255;
|
|
const URL_MAX_LENGTH = 1024;
|
|
const AUTH_MAX_LENGTH = 255;
|
|
const ITEMS_PATH_MAX_LENGTH = 255;
|
|
const REQUEST_BODY_MAX_LENGTH = 1000000;
|
|
const TOKEN_URL_MAX_LENGTH = 1024;
|
|
const TOKEN_RESPONSE_PATH_MAX_LENGTH = 255;
|
|
const TOKEN_HEADER_PREFIX_MAX_LENGTH = 64;
|
|
const tokenCache = new Map();
|
|
|
|
function normalizeUpdateIntervalUnit(value) {
|
|
const unit = String(value || '').trim().toLowerCase();
|
|
if (unit === 'seconds' || unit === 'minutes' || unit === 'hours') {
|
|
return unit;
|
|
}
|
|
return 'minutes';
|
|
}
|
|
|
|
function normalizeAuthMethod(value) {
|
|
const method = String(value || '').trim().toLowerCase();
|
|
return ['basic', 'bearer', 'api_key_header', 'token_login'].includes(method) ? method : 'none';
|
|
}
|
|
|
|
function normalizeRequestMethod(value) {
|
|
return String(value || '').trim().toUpperCase() === 'POST' ? 'POST' : 'GET';
|
|
}
|
|
|
|
function buildAuthHeaders(source) {
|
|
const method = normalizeAuthMethod(source && (source.auth_method || source.authMethod));
|
|
const headers = {};
|
|
|
|
if (method === 'basic') {
|
|
const username = String(source && (source.auth_username || source.authUsername) || '').trim();
|
|
const password = String(source && (source.auth_password || source.authPassword) || '');
|
|
if (username || password) {
|
|
headers.Authorization = 'Basic ' + Buffer.from(username + ':' + password, 'utf8').toString('base64');
|
|
}
|
|
} else if (method === 'bearer') {
|
|
const token = String(source && (source.auth_bearer_token || source.authBearerToken) || '').trim();
|
|
if (token) {
|
|
headers.Authorization = 'Bearer ' + token;
|
|
}
|
|
} else if (method === 'api_key_header') {
|
|
const headerName = String(source && (source.auth_header_name || source.authHeaderName) || 'X-API-Key').trim() || 'X-API-Key';
|
|
const headerValue = String(source && (source.auth_header_value || source.authHeaderValue) || '').trim();
|
|
if (headerValue) {
|
|
headers[headerName] = headerValue;
|
|
}
|
|
}
|
|
|
|
return headers;
|
|
}
|
|
|
|
async function fetchApiSourcesData(pool) {
|
|
const [apiSources] = await pool.query(
|
|
'SELECT id, name, api_url, request_method, request_body_json, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, token_url, token_request_body_json, token_response_path, token_header_name, token_header_prefix, items_path, enabled, update_interval_value, update_interval_unit, last_pulled_at, last_pull_error, last_response_status, last_response_content_type, last_response_json, created_at, modified_at, created_by, modified_by FROM i_api_sources ORDER BY modified_at DESC, id DESC'
|
|
);
|
|
|
|
return { apiSources: apiSources };
|
|
}
|
|
|
|
async function fetchApiSourcesPage(pool, page, pageSize, searchTerm, sortKey, sortDirection) {
|
|
const paged = await fetchPagedRows(pool, {
|
|
selectSql: 'SELECT id, name, api_url, request_method, request_body_json, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, token_url, token_request_body_json, token_response_path, token_header_name, token_header_prefix, items_path, enabled, update_interval_value, update_interval_unit, last_pulled_at, last_pull_error, last_response_status, last_response_content_type, last_response_json, created_at, modified_at, created_by, modified_by FROM i_api_sources ORDER BY modified_at DESC, id DESC',
|
|
countSql: 'SELECT COUNT(*) AS count FROM i_api_sources',
|
|
searchColumns: ['name', 'api_url', 'last_pull_error'],
|
|
searchTerm: searchTerm,
|
|
sortColumns: {
|
|
name: 'name',
|
|
url: 'api_url',
|
|
interval: ['update_interval_value', 'update_interval_unit'],
|
|
last_pulled: 'last_pulled_at',
|
|
last_response: 'last_response_status',
|
|
created: 'created_at',
|
|
modified: 'modified_at'
|
|
},
|
|
sortKey: sortKey,
|
|
sortDirection: sortDirection,
|
|
page: page,
|
|
pageSize: pageSize
|
|
});
|
|
|
|
return Object.assign({ apiSources: paged.rows }, paged);
|
|
}
|
|
|
|
async function fetchApiSourceById(pool, id) {
|
|
const [rows] = await pool.query(
|
|
'SELECT id, name, api_url, request_method, request_body_json, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, token_url, token_request_body_json, token_response_path, token_header_name, token_header_prefix, items_path, enabled, update_interval_value, update_interval_unit, last_pulled_at, last_pull_error, last_response_status, last_response_content_type, last_response_json, created_at, modified_at, created_by, modified_by FROM i_api_sources WHERE id = ?',
|
|
[id]
|
|
);
|
|
|
|
return rows[0] || null;
|
|
}
|
|
|
|
async function loadUrlText(urlValue, requestOptions) {
|
|
const extraHeaders = requestOptions && requestOptions.headers ? requestOptions.headers : {};
|
|
const method = String(requestOptions && requestOptions.method || 'GET').toUpperCase();
|
|
const body = requestOptions && requestOptions.body !== undefined ? requestOptions.body : undefined;
|
|
const headers = Object.assign({
|
|
Accept: 'application/json, text/plain;q=0.9, */*;q=0.8',
|
|
'User-Agent': 'Pulse Signage API Reader'
|
|
}, extraHeaders);
|
|
if (typeof fetch === 'function') {
|
|
const fetchOptions = { method: method, headers: headers };
|
|
if (body !== undefined && method !== 'GET' && method !== 'HEAD') {
|
|
fetchOptions.body = body;
|
|
}
|
|
const response = await fetch(urlValue, fetchOptions);
|
|
|
|
return {
|
|
statusCode: response.status,
|
|
contentType: response.headers.get('content-type') || '',
|
|
bodyText: await response.text(),
|
|
ok: response.ok
|
|
};
|
|
}
|
|
|
|
return await new Promise(function (resolve, reject) {
|
|
const url = new URL(urlValue);
|
|
const transport = url.protocol === 'https:' ? https : http;
|
|
const request = transport.request(url, Object.assign({}, requestOptions || {}, {
|
|
method: method,
|
|
headers: headers
|
|
}), function (response) {
|
|
response.setEncoding('utf8');
|
|
let body = '';
|
|
response.on('data', function (chunk) {
|
|
body += chunk;
|
|
});
|
|
response.on('end', function () {
|
|
resolve({
|
|
statusCode: response.statusCode || 0,
|
|
contentType: String(response.headers['content-type'] || ''),
|
|
bodyText: body,
|
|
ok: !response.statusCode || response.statusCode < 400
|
|
});
|
|
});
|
|
response.on('error', reject);
|
|
});
|
|
|
|
if (body !== undefined && method !== 'GET' && method !== 'HEAD') {
|
|
request.write(body);
|
|
}
|
|
request.end();
|
|
request.on('error', reject);
|
|
});
|
|
}
|
|
|
|
function parseJsonRequestBody(value, fieldName) {
|
|
const text = String(value || '').trim();
|
|
if (!text) {
|
|
return undefined;
|
|
}
|
|
try {
|
|
return JSON.parse(text);
|
|
} catch (_error) {
|
|
const error = new Error(fieldName + ' must contain valid JSON.');
|
|
error.statusCode = 400;
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function resolveResponsePath(value, responsePath) {
|
|
let current = value;
|
|
String(responsePath || '').split('.').forEach(function (segment) {
|
|
if (current === undefined || current === null) {
|
|
current = undefined;
|
|
return;
|
|
}
|
|
current = current[segment];
|
|
});
|
|
return current;
|
|
}
|
|
|
|
function getTokenCacheKey(source) {
|
|
return JSON.stringify([
|
|
source && source.id || '',
|
|
source && (source.token_url || source.tokenUrl) || '',
|
|
source && (source.token_request_body_json || source.tokenRequestBodyJson) || '',
|
|
source && (source.token_response_path || source.tokenResponsePath) || 'access_token',
|
|
source && (source.token_header_name || source.tokenHeaderName) || 'Authorization',
|
|
source && (source.token_header_prefix || source.tokenHeaderPrefix) || 'Bearer'
|
|
]);
|
|
}
|
|
|
|
function clearCachedToken(source) {
|
|
tokenCache.delete(getTokenCacheKey(source));
|
|
}
|
|
|
|
async function fetchLoginToken(source) {
|
|
const cacheKey = getTokenCacheKey(source);
|
|
const cached = tokenCache.get(cacheKey);
|
|
if (cached && cached.expiresAt > Date.now()) {
|
|
return cached.value;
|
|
}
|
|
|
|
const tokenUrl = source.token_url || source.tokenUrl;
|
|
const tokenBody = parseJsonRequestBody(source.token_request_body_json || source.tokenRequestBodyJson, 'Login request body');
|
|
const tokenHeaders = { 'Content-Type': 'application/json' };
|
|
const response = await loadUrlText(tokenUrl, {
|
|
method: 'POST',
|
|
headers: tokenHeaders,
|
|
body: tokenBody === undefined ? undefined : JSON.stringify(tokenBody)
|
|
});
|
|
if (!response.ok) {
|
|
throw new Error(`Unable to obtain API token (${response.statusCode}).`);
|
|
}
|
|
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(String(response.bodyText || '').trim());
|
|
} catch (_error) {
|
|
throw new Error('Token response was not valid JSON.');
|
|
}
|
|
|
|
const tokenPath = source.token_response_path || source.tokenResponsePath || 'access_token';
|
|
const token = resolveResponsePath(parsed, tokenPath);
|
|
if (token === undefined || token === null || String(token).trim() === '') {
|
|
throw new Error('Token response did not contain a token at the configured path.');
|
|
}
|
|
|
|
const expiresIn = Number(parsed && (parsed.expires_in || parsed.expiresIn));
|
|
const lifetimeMs = Number.isFinite(expiresIn) && expiresIn > 0
|
|
? Math.max(30000, expiresIn * 1000 - 60000)
|
|
: 300000;
|
|
tokenCache.set(cacheKey, { value: String(token), expiresAt: Date.now() + lifetimeMs });
|
|
return String(token);
|
|
}
|
|
|
|
async function buildRequestHeaders(source) {
|
|
const method = normalizeAuthMethod(source && (source.auth_method || source.authMethod));
|
|
if (method !== 'token_login') {
|
|
return buildAuthHeaders(source);
|
|
}
|
|
|
|
const token = await fetchLoginToken(source);
|
|
const headerName = String(source.token_header_name || source.tokenHeaderName || 'Authorization').trim() || 'Authorization';
|
|
const prefix = String(source.token_header_prefix || source.tokenHeaderPrefix || 'Bearer').trim();
|
|
return { [headerName]: prefix ? prefix + ' ' + token : token };
|
|
}
|
|
|
|
async function fetchApiSourceResponse(apiSource) {
|
|
const source = apiSource && typeof apiSource === 'object' ? apiSource : { api_url: apiSource };
|
|
const requestMethod = normalizeRequestMethod(source.request_method || source.requestMethod);
|
|
const requestBody = parseJsonRequestBody(source.request_body_json || source.requestBodyJson, 'API request body');
|
|
let response;
|
|
let tokenRetry = false;
|
|
do {
|
|
response = await loadUrlText(source.api_url || source.apiUrl, {
|
|
method: requestMethod,
|
|
headers: Object.assign({}, await buildRequestHeaders(source), requestBody === undefined ? {} : { 'Content-Type': 'application/json' }),
|
|
body: requestBody === undefined ? undefined : JSON.stringify(requestBody)
|
|
});
|
|
if (response.statusCode === 401 && normalizeAuthMethod(source.auth_method || source.authMethod) === 'token_login' && !tokenRetry) {
|
|
clearCachedToken(source);
|
|
tokenRetry = true;
|
|
} else {
|
|
break;
|
|
}
|
|
} while (true);
|
|
if (!response.ok) {
|
|
throw new Error(`Unable to load API response (${response.statusCode}).`);
|
|
}
|
|
|
|
const text = String(response.bodyText || '').trim();
|
|
if (!text) {
|
|
throw new Error('API response did not return JSON.');
|
|
}
|
|
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(text);
|
|
} catch (_error) {
|
|
throw new Error('API response was not valid JSON.');
|
|
}
|
|
|
|
return {
|
|
responseJson: JSON.stringify(parsed, null, 2),
|
|
responseStatus: response.statusCode,
|
|
responseContentType: response.contentType
|
|
};
|
|
}
|
|
|
|
function buildApiSourcePayload(req, existingApiSource) {
|
|
const fallback = existingApiSource || {};
|
|
const body = req && req.body ? req.body : {};
|
|
function readBodyValue(fieldName, fallbackValue) {
|
|
if (Object.prototype.hasOwnProperty.call(body, fieldName)) {
|
|
return body[fieldName];
|
|
}
|
|
return fallbackValue;
|
|
}
|
|
|
|
const name = validateMaxLength(req.body.name || fallback.name || '', NAME_MAX_LENGTH, 'API source name');
|
|
const apiUrl = validateMaxLength(req.body.api_url || req.body.apiUrl || fallback.api_url || '', URL_MAX_LENGTH, 'API source URL');
|
|
const authMethod = normalizeAuthMethod(readBodyValue('auth_method', readBodyValue('authMethod', fallback.auth_method || 'none')));
|
|
const requestMethod = normalizeRequestMethod(readBodyValue('request_method', readBodyValue('requestMethod', fallback.request_method || 'GET')));
|
|
const requestBodyJson = validateMaxLength(readBodyValue('request_body_json', readBodyValue('requestBodyJson', fallback.request_body_json || '')) || '', REQUEST_BODY_MAX_LENGTH, 'API request body');
|
|
const authUsername = validateMaxLength(readBodyValue('auth_username', readBodyValue('authUsername', fallback.auth_username || '')) || '', AUTH_MAX_LENGTH, 'API source username');
|
|
const authPassword = validateMaxLength(readBodyValue('auth_password', readBodyValue('authPassword', fallback.auth_password || '')) || '', AUTH_MAX_LENGTH, 'API source password');
|
|
const authBearerToken = validateMaxLength(readBodyValue('auth_bearer_token', readBodyValue('authBearerToken', fallback.auth_bearer_token || '')) || '', AUTH_MAX_LENGTH, 'API source bearer token');
|
|
const authHeaderName = validateMaxLength(readBodyValue('auth_header_name', readBodyValue('authHeaderName', fallback.auth_header_name || 'X-API-Key')) || 'X-API-Key', AUTH_MAX_LENGTH, 'API source header name') || 'X-API-Key';
|
|
const authHeaderValue = validateMaxLength(readBodyValue('auth_header_value', readBodyValue('authHeaderValue', fallback.auth_header_value || '')) || '', AUTH_MAX_LENGTH, 'API source header value');
|
|
const tokenUrl = validateMaxLength(readBodyValue('token_url', readBodyValue('tokenUrl', fallback.token_url || '')) || '', TOKEN_URL_MAX_LENGTH, 'API token URL');
|
|
const tokenRequestBodyJson = validateMaxLength(readBodyValue('token_request_body_json', readBodyValue('tokenRequestBodyJson', fallback.token_request_body_json || '')) || '', REQUEST_BODY_MAX_LENGTH, 'Login request body');
|
|
const tokenResponsePath = validateMaxLength(readBodyValue('token_response_path', readBodyValue('tokenResponsePath', fallback.token_response_path || 'access_token')) || 'access_token', TOKEN_RESPONSE_PATH_MAX_LENGTH, 'Token response path');
|
|
const tokenHeaderName = validateMaxLength(readBodyValue('token_header_name', readBodyValue('tokenHeaderName', fallback.token_header_name || 'Authorization')) || 'Authorization', AUTH_MAX_LENGTH, 'Token header name');
|
|
const tokenHeaderPrefix = validateMaxLength(readBodyValue('token_header_prefix', readBodyValue('tokenHeaderPrefix', fallback.token_header_prefix || 'Bearer')) || '', TOKEN_HEADER_PREFIX_MAX_LENGTH, 'Token prefix');
|
|
const itemsPath = validateMaxLength(readBodyValue('items_path', readBodyValue('itemsPath', fallback.items_path || '')) || '', ITEMS_PATH_MAX_LENGTH, 'API source items path');
|
|
const updateIntervalValue = Math.max(1, Number(req.body.update_interval_value || req.body.updateIntervalValue || fallback.update_interval_value || 60));
|
|
const updateIntervalUnit = normalizeUpdateIntervalUnit(req.body.update_interval_unit || req.body.updateIntervalUnit || fallback.update_interval_unit || 'minutes');
|
|
|
|
if (!name) {
|
|
const error = new Error('API source name is required.');
|
|
error.statusCode = 400;
|
|
throw error;
|
|
}
|
|
|
|
if (!apiUrl) {
|
|
const error = new Error('API source URL is required.');
|
|
error.statusCode = 400;
|
|
throw error;
|
|
}
|
|
|
|
let parsedUrl;
|
|
try {
|
|
parsedUrl = new URL(apiUrl);
|
|
} catch (_error) {
|
|
const error = new Error('Enter a valid API URL.');
|
|
error.statusCode = 400;
|
|
throw error;
|
|
}
|
|
|
|
if (parsedUrl.protocol !== 'http:' && parsedUrl.protocol !== 'https:') {
|
|
const error = new Error('API URL must start with http or https.');
|
|
error.statusCode = 400;
|
|
throw error;
|
|
}
|
|
|
|
if (!Number.isFinite(updateIntervalValue)) {
|
|
const error = new Error('Update interval must be a number.');
|
|
error.statusCode = 400;
|
|
throw error;
|
|
}
|
|
|
|
if (authMethod === 'basic' && !authUsername && !authPassword) {
|
|
const error = new Error('Basic auth requires a username or password.');
|
|
error.statusCode = 400;
|
|
throw error;
|
|
}
|
|
|
|
if (authMethod === 'bearer' && !authBearerToken) {
|
|
const error = new Error('Bearer auth requires a token.');
|
|
error.statusCode = 400;
|
|
throw error;
|
|
}
|
|
|
|
if (authMethod === 'api_key_header' && !authHeaderValue) {
|
|
const error = new Error('API key auth requires a header value.');
|
|
error.statusCode = 400;
|
|
throw error;
|
|
}
|
|
|
|
parseJsonRequestBody(requestBodyJson, 'API request body');
|
|
parseJsonRequestBody(tokenRequestBodyJson, 'Login request body');
|
|
|
|
if (authMethod === 'token_login') {
|
|
if (!tokenUrl) {
|
|
const error = new Error('Token login requires a login URL.');
|
|
error.statusCode = 400;
|
|
throw error;
|
|
}
|
|
try {
|
|
const tokenParsedUrl = new URL(tokenUrl);
|
|
if (tokenParsedUrl.protocol !== 'http:' && tokenParsedUrl.protocol !== 'https:') {
|
|
throw new Error('invalid protocol');
|
|
}
|
|
} catch (_error) {
|
|
const error = new Error('Enter a valid API token URL.');
|
|
error.statusCode = 400;
|
|
throw error;
|
|
}
|
|
if (!tokenRequestBodyJson) {
|
|
const error = new Error('Token login requires a JSON request body.');
|
|
error.statusCode = 400;
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
return {
|
|
name: name,
|
|
apiUrl: parsedUrl.toString(),
|
|
requestMethod: requestMethod,
|
|
requestBodyJson: requestBodyJson,
|
|
authMethod: authMethod,
|
|
authUsername: authUsername,
|
|
authPassword: authPassword,
|
|
authBearerToken: authBearerToken,
|
|
authHeaderName: authHeaderName,
|
|
authHeaderValue: authHeaderValue,
|
|
tokenUrl: tokenUrl,
|
|
tokenRequestBodyJson: tokenRequestBodyJson,
|
|
tokenResponsePath: tokenResponsePath,
|
|
tokenHeaderName: tokenHeaderName,
|
|
tokenHeaderPrefix: tokenHeaderPrefix,
|
|
itemsPath: itemsPath,
|
|
updateIntervalValue: Math.floor(updateIntervalValue),
|
|
updateIntervalUnit: updateIntervalUnit
|
|
};
|
|
}
|
|
|
|
module.exports = {
|
|
fetchApiSourcesData: fetchApiSourcesData,
|
|
fetchApiSourcesPage: fetchApiSourcesPage,
|
|
fetchApiSourceById: fetchApiSourceById,
|
|
fetchApiSourceResponse: fetchApiSourceResponse,
|
|
buildApiSourcePayload: buildApiSourcePayload
|
|
}; |