Release v2.11.1

2026-09-04 15:43:55 +01:00
parent 2c150b5b2e
commit 98f969ca0f
104 changed files with 3559 additions and 447 deletions
@@ -13,6 +13,7 @@ const TOKEN_URL_MAX_LENGTH = 1024;
 const TOKEN_RESPONSE_PATH_MAX_LENGTH = 255;
 const TOKEN_HEADER_PREFIX_MAX_LENGTH = 64;
 const tokenCache = new Map();
+const tokenRequests = new Map();

 function normalizeUpdateIntervalUnit(value) {
  const unit = String(value || '').trim().toLowerCase();
@@ -59,7 +60,7 @@ function buildAuthHeaders(source) {

 async function fetchApiSourcesData(pool) {
  const [apiSources] = await pool.query(
-    'SELECT id, name, api_url, request_method, request_body_json, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, token_url, token_request_body_json, token_response_path, token_header_name, token_header_prefix, items_path, enabled, update_interval_value, update_interval_unit, last_pulled_at, last_pull_error, last_response_status, last_response_content_type, last_response_json, created_at, modified_at, created_by, modified_by FROM i_api_sources ORDER BY modified_at DESC, id DESC'
+    'SELECT id, name, api_url, request_method, request_body_json, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, token_url, token_request_body_json, token_response_path, token_refresh_url, token_refresh_request_body_json, token_refresh_response_path, token_header_name, token_header_prefix, items_path, enabled, update_interval_value, update_interval_unit, last_pulled_at, last_pull_error, last_response_status, last_response_content_type, last_response_json, created_at, modified_at, created_by, modified_by FROM i_api_sources ORDER BY modified_at DESC, id DESC'
  );

  return { apiSources: apiSources };
@@ -67,7 +68,7 @@ async function fetchApiSourcesData(pool) {

 async function fetchApiSourcesPage(pool, page, pageSize, searchTerm, sortKey, sortDirection) {
  const paged = await fetchPagedRows(pool, {
-    selectSql: 'SELECT id, name, api_url, request_method, request_body_json, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, token_url, token_request_body_json, token_response_path, token_header_name, token_header_prefix, items_path, enabled, update_interval_value, update_interval_unit, last_pulled_at, last_pull_error, last_response_status, last_response_content_type, last_response_json, created_at, modified_at, created_by, modified_by FROM i_api_sources ORDER BY modified_at DESC, id DESC',
+    selectSql: 'SELECT id, name, api_url, request_method, request_body_json, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, token_url, token_request_body_json, token_response_path, token_refresh_url, token_refresh_request_body_json, token_refresh_response_path, token_header_name, token_header_prefix, items_path, enabled, update_interval_value, update_interval_unit, last_pulled_at, last_pull_error, last_response_status, last_response_content_type, last_response_json, created_at, modified_at, created_by, modified_by FROM i_api_sources ORDER BY modified_at DESC, id DESC',
    countSql: 'SELECT COUNT(*) AS count FROM i_api_sources',
    searchColumns: ['name', 'api_url', 'last_pull_error'],
    searchTerm: searchTerm,
@@ -91,7 +92,7 @@ async function fetchApiSourcesPage(pool, page, pageSize, searchTerm, sortKey, so

 async function fetchApiSourceById(pool, id) {
  const [rows] = await pool.query(
-    'SELECT id, name, api_url, request_method, request_body_json, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, token_url, token_request_body_json, token_response_path, token_header_name, token_header_prefix, items_path, enabled, update_interval_value, update_interval_unit, last_pulled_at, last_pull_error, last_response_status, last_response_content_type, last_response_json, created_at, modified_at, created_by, modified_by FROM i_api_sources WHERE id = ?',
+    'SELECT id, name, api_url, request_method, request_body_json, auth_method, auth_username, auth_password, auth_bearer_token, auth_header_name, auth_header_value, token_url, token_request_body_json, token_response_path, token_refresh_url, token_refresh_request_body_json, token_refresh_response_path, token_header_name, token_header_prefix, items_path, enabled, update_interval_value, update_interval_unit, last_pulled_at, last_pull_error, last_response_status, last_response_content_type, last_response_json, created_at, modified_at, created_by, modified_by FROM i_api_sources WHERE id = ?',
    [id]
  );

@@ -184,6 +185,9 @@ function getTokenCacheKey(source) {
    source && (source.token_url || source.tokenUrl) || '',
    source && (source.token_request_body_json || source.tokenRequestBodyJson) || '',
    source && (source.token_response_path || source.tokenResponsePath) || 'access_token',
+    source && (source.token_refresh_url || source.tokenRefreshUrl) || '',
+    source && (source.token_refresh_request_body_json || source.tokenRefreshRequestBodyJson) || '',
+    source && (source.token_refresh_response_path || source.tokenRefreshResponsePath) || 'refresh_token',
    source && (source.token_header_name || source.tokenHeaderName) || 'Authorization',
    source && (source.token_header_prefix || source.tokenHeaderPrefix) || 'Bearer'
  ]);
@@ -193,11 +197,118 @@ function clearCachedToken(source) {
  tokenCache.delete(getTokenCacheKey(source));
 }

-async function fetchLoginToken(source) {
+function replaceRefreshToken(value, refreshToken) {
+  if (typeof value === 'string') {
+    return value.split('{{refresh_token}}').join(refreshToken);
+  }
+  if (Array.isArray(value)) {
+    return value.map(function (item) {
+      return replaceRefreshToken(item, refreshToken);
+    });
+  }
+  if (value && typeof value === 'object') {
+    return Object.keys(value).reduce(function (result, key) {
+      result[key] = replaceRefreshToken(value[key], refreshToken);
+      return result;
+    }, {});
+  }
+  return value;
+}
+
+function resolveTokenExpiry(parsed, token) {
+  const expiresIn = Number(parsed && (parsed.expires_in || parsed.expiresIn));
+  if (Number.isFinite(expiresIn) && expiresIn > 0) {
+    return { lifetimeMs: expiresIn * 1000 };
+  }
+
+  const explicitExpiry = parsed && (parsed.expires_at || parsed.expiresAt);
+  if (explicitExpiry !== undefined && explicitExpiry !== null) {
+    const expiryNumber = Number(explicitExpiry);
+    const expiryMs = Number.isFinite(expiryNumber)
+      ? (expiryNumber < 100000000000 ? expiryNumber * 1000 : expiryNumber)
+      : Date.parse(String(explicitExpiry));
+    if (Number.isFinite(expiryMs) && expiryMs > Date.now()) {
+      return { expiresAt: expiryMs };
+    }
+  }
+
+  const tokenParts = String(token).split('.');
+  if (tokenParts.length === 3) {
+    try {
+      const payload = JSON.parse(Buffer.from(tokenParts[1], 'base64url').toString('utf8'));
+      const expiryMs = Number(payload.exp) * 1000;
+      if (Number.isFinite(expiryMs) && expiryMs > Date.now()) {
+        return { expiresAt: expiryMs };
+      }
+    } catch (_error) {
+      // Opaque tokens do not contain a readable JWT expiry.
+    }
+  }
+
+  return { lifetimeMs: 300000 };
+}
+
+function cacheTokenResponse(source, parsed, previousRefreshToken) {
+  const tokenPath = source.token_response_path || source.tokenResponsePath || 'access_token';
+  const token = resolveResponsePath(parsed, tokenPath);
+  if (token === undefined || token === null || String(token).trim() === '') {
+    throw new Error('Token response did not contain a token at the configured path.');
+  }
+
+  const refreshPath = source.token_refresh_response_path || source.tokenRefreshResponsePath || 'refresh_token';
+  const responseRefreshToken = resolveResponsePath(parsed, refreshPath);
+  const refreshToken = responseRefreshToken === undefined || responseRefreshToken === null || String(responseRefreshToken).trim() === ''
+    ? previousRefreshToken
+    : String(responseRefreshToken);
+  const expiry = resolveTokenExpiry(parsed, token);
+  const expiresAt = expiry.expiresAt || Date.now() + Math.max(1000, expiry.lifetimeMs - Math.min(60000, expiry.lifetimeMs * 0.1));
+  const record = { value: String(token), refreshToken: refreshToken, expiresAt: expiresAt };
+  tokenCache.set(getTokenCacheKey(source), record);
+  return record;
+}
+
+async function parseTokenResponse(response, source, previousRefreshToken) {
+  if (!response.ok) {
+    return null;
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(String(response.bodyText || '').trim());
+  } catch (_error) {
+    throw new Error('Token response was not valid JSON.');
+  }
+  return cacheTokenResponse(source, parsed, previousRefreshToken);
+}
+
+async function refreshLoginToken(source, cached) {
+  const refreshUrl = source.token_refresh_url || source.tokenRefreshUrl || source.token_url || source.tokenUrl;
+  if (!cached || !cached.refreshToken) {
+    return null;
+  }
+  const configuredBody = parseJsonRequestBody(source.token_refresh_request_body_json || source.tokenRefreshRequestBodyJson, 'Refresh request body');
+  const refreshBody = configuredBody === undefined
+    ? { grant_type: 'refresh_token', refresh_token: cached.refreshToken }
+    : replaceRefreshToken(configuredBody, cached.refreshToken);
+  const response = await loadUrlText(refreshUrl, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(refreshBody)
+  });
+  return parseTokenResponse(response, source, cached.refreshToken);
+}
+
+async function fetchLoginTokenUncached(source) {
  const cacheKey = getTokenCacheKey(source);
  const cached = tokenCache.get(cacheKey);
  if (cached && cached.expiresAt > Date.now()) {
-    return cached.value;
+    return cached;
+  }
+
+  if (cached && cached.refreshToken) {
+    const refreshed = await refreshLoginToken(source, cached);
+    if (refreshed) {
+      return refreshed;
+    }
  }

  const tokenUrl = source.token_url || source.tokenUrl;
@@ -208,29 +319,25 @@ async function fetchLoginToken(source) {
    headers: tokenHeaders,
    body: tokenBody === undefined ? undefined : JSON.stringify(tokenBody)
  });
-  if (!response.ok) {
+  const record = await parseTokenResponse(response, source, cached && cached.refreshToken);
+  if (!record) {
    throw new Error(`Unable to obtain API token (${response.statusCode}).`);
  }
+  return record;
+}

-  let parsed;
-  try {
-    parsed = JSON.parse(String(response.bodyText || '').trim());
-  } catch (_error) {
-    throw new Error('Token response was not valid JSON.');
+async function fetchLoginToken(source) {
+  const cacheKey = getTokenCacheKey(source);
+  const cached = tokenCache.get(cacheKey);
+  if (cached && cached.expiresAt > Date.now()) {
+    return cached.value;
  }
-
-  const tokenPath = source.token_response_path || source.tokenResponsePath || 'access_token';
-  const token = resolveResponsePath(parsed, tokenPath);
-  if (token === undefined || token === null || String(token).trim() === '') {
-    throw new Error('Token response did not contain a token at the configured path.');
+  if (!tokenRequests.has(cacheKey)) {
+    tokenRequests.set(cacheKey, fetchLoginTokenUncached(source).finally(function () {
+      tokenRequests.delete(cacheKey);
+    }));
  }
-
-  const expiresIn = Number(parsed && (parsed.expires_in || parsed.expiresIn));
-  const lifetimeMs = Number.isFinite(expiresIn) && expiresIn > 0
-    ? Math.max(30000, expiresIn * 1000 - 60000)
-    : 300000;
-  tokenCache.set(cacheKey, { value: String(token), expiresAt: Date.now() + lifetimeMs });
-  return String(token);
+  return (await tokenRequests.get(cacheKey)).value;
 }

 async function buildRequestHeaders(source) {
@@ -310,6 +417,9 @@ function buildApiSourcePayload(req, existingApiSource) {
  const tokenUrl = validateMaxLength(readBodyValue('token_url', readBodyValue('tokenUrl', fallback.token_url || '')) || '', TOKEN_URL_MAX_LENGTH, 'API token URL');
  const tokenRequestBodyJson = validateMaxLength(readBodyValue('token_request_body_json', readBodyValue('tokenRequestBodyJson', fallback.token_request_body_json || '')) || '', REQUEST_BODY_MAX_LENGTH, 'Login request body');
  const tokenResponsePath = validateMaxLength(readBodyValue('token_response_path', readBodyValue('tokenResponsePath', fallback.token_response_path || 'access_token')) || 'access_token', TOKEN_RESPONSE_PATH_MAX_LENGTH, 'Token response path');
+  const tokenRefreshUrl = validateMaxLength(readBodyValue('token_refresh_url', readBodyValue('tokenRefreshUrl', fallback.token_refresh_url || '')) || '', TOKEN_URL_MAX_LENGTH, 'API refresh URL');
+  const tokenRefreshRequestBodyJson = validateMaxLength(readBodyValue('token_refresh_request_body_json', readBodyValue('tokenRefreshRequestBodyJson', fallback.token_refresh_request_body_json || '')) || '', REQUEST_BODY_MAX_LENGTH, 'Refresh request body');
+  const tokenRefreshResponsePath = validateMaxLength(readBodyValue('token_refresh_response_path', readBodyValue('tokenRefreshResponsePath', fallback.token_refresh_response_path || 'refresh_token')) || 'refresh_token', TOKEN_RESPONSE_PATH_MAX_LENGTH, 'Refresh token response path');
  const tokenHeaderName = validateMaxLength(readBodyValue('token_header_name', readBodyValue('tokenHeaderName', fallback.token_header_name || 'Authorization')) || 'Authorization', AUTH_MAX_LENGTH, 'Token header name');
  const tokenHeaderPrefix = validateMaxLength(readBodyValue('token_header_prefix', readBodyValue('tokenHeaderPrefix', fallback.token_header_prefix || 'Bearer')) || '', TOKEN_HEADER_PREFIX_MAX_LENGTH, 'Token prefix');
  const itemsPath = validateMaxLength(readBodyValue('items_path', readBodyValue('itemsPath', fallback.items_path || '')) || '', ITEMS_PATH_MAX_LENGTH, 'API source items path');
@@ -369,6 +479,7 @@ function buildApiSourcePayload(req, existingApiSource) {

  parseJsonRequestBody(requestBodyJson, 'API request body');
  parseJsonRequestBody(tokenRequestBodyJson, 'Login request body');
+  parseJsonRequestBody(tokenRefreshRequestBodyJson, 'Refresh request body');

  if (authMethod === 'token_login') {
    if (!tokenUrl) {
@@ -393,6 +504,19 @@ function buildApiSourcePayload(req, existingApiSource) {
    }
  }

+  if (tokenRefreshUrl) {
+    try {
+      const refreshParsedUrl = new URL(tokenRefreshUrl);
+      if (refreshParsedUrl.protocol !== 'http:' && refreshParsedUrl.protocol !== 'https:') {
+        throw new Error('invalid protocol');
+      }
+    } catch (_error) {
+      const error = new Error('Enter a valid API refresh URL.');
+      error.statusCode = 400;
+      throw error;
+    }
+  }
+
  return {
    name: name,
    apiUrl: parsedUrl.toString(),
@@ -407,6 +531,9 @@ function buildApiSourcePayload(req, existingApiSource) {
    tokenUrl: tokenUrl,
    tokenRequestBodyJson: tokenRequestBodyJson,
    tokenResponsePath: tokenResponsePath,
+    tokenRefreshUrl: tokenRefreshUrl,
+    tokenRefreshRequestBodyJson: tokenRefreshRequestBodyJson,
+    tokenRefreshResponsePath: tokenRefreshResponsePath,
    tokenHeaderName: tokenHeaderName,
    tokenHeaderPrefix: tokenHeaderPrefix,
    itemsPath: itemsPath,