Initial Sheet Relay application

2026-09-17 18:54:22 +01:00
commit 854da073d4
@@ -0,0 +1,100 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { randomBytes } from 'node:crypto';
+import path from 'node:path';
+
+const dataDirectory = path.join(process.cwd(), 'data');
+const configPath = path.join(dataDirectory, 'config.json');
+
+const defaultConfig = () => ({
+  adminToken: randomBytes(24).toString('base64url'),
+  settings: {
+    corsOrigins: [],
+    trustedProxyHops: 0
+  },
+  credentials: [],
+  documents: [],
+  publications: []
+});
+
+function migrateConfig(config) {
+  if (config.credentials) {
+    return {
+      ...config,
+      settings: {
+        ...config.settings,
+        corsOrigins: config.settings?.corsOrigins || [],
+        trustedProxyHops: Number(config.settings?.trustedProxyHops) || 0
+      }
+    };
+  }
+  const legacyCredential = config.google?.apiKey || config.google?.serviceAccountJson ? {
+    id: crypto.randomUUID(),
+    name: 'Imported Google connection',
+    mode: config.google.mode,
+    apiKey: config.google.apiKey,
+    serviceAccountJson: config.google.serviceAccountJson,
+    requestsPerMinute: config.google.requestsPerMinute || 45
+  } : null;
+  const documents = (config.publications || []).map((publication) => ({
+    id: crypto.randomUUID(),
+    name: publication.name,
+    spreadsheetId: publication.spreadsheetId,
+    credentialId: legacyCredential?.id || '',
+    sheets: publication.sheets || []
+  }));
+  return {
+    adminToken: config.adminToken,
+    settings: { corsOrigins: [], trustedProxyHops: 0 },
+    credentials: legacyCredential ? [legacyCredential] : [],
+    documents,
+    publications: (config.publications || []).map((publication, index) => ({
+      id: publication.id,
+      name: publication.name,
+      documentId: documents[index].id,
+      sheets: publication.sheets || [],
+      apiToken: publication.apiToken,
+      cacheSeconds: config.google?.cacheSeconds || 30
+    }))
+  };
+}
+
+export async function loadConfig() {
+  try {
+    const config = migrateConfig(JSON.parse(await readFile(configPath, 'utf8')));
+    await saveConfig(config);
+    return config;
+  } catch (error) {
+    if (error.code !== 'ENOENT') throw error;
+    const config = defaultConfig();
+    await saveConfig(config);
+    console.log(`Created admin token: ${config.adminToken}`);
+    return config;
+  }
+}
+
+export async function saveConfig(config) {
+  await mkdir(dataDirectory, { recursive: true });
+  await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 });
+}
+
+export function publicConfig(config) {
+  return {
+    settings: config.settings,
+    credentials: config.credentials.map(({ apiKey, serviceAccountJson, ...credential }) => ({
+      ...credential,
+      configured: credential.mode === 'apiKey' ? Boolean(apiKey) : Boolean(serviceAccountJson)
+    })),
+    documents: config.documents,
+    publications: config.publications.map(({ apiToken, ...publication }) => ({
+      ...publication,
+      dataFormat: publication.dataFormat || 'rows',
+      range: publication.range || '',
+      omitBlankRows: Boolean(publication.omitBlankRows),
+      maxRows: Math.max(1, Math.min(100_000, Number(publication.maxRows) || 10_000)),
+      consumerRequestsPerMinute: Math.max(1, Number(publication.consumerRequestsPerMinute) || 120),
+      scheduledRefresh: Boolean(publication.scheduledRefresh),
+      enabled: publication.enabled !== false,
+      tokenConfigured: Boolean(apiToken)
+    }))
+  };
+}
@@ -0,0 +1,100 @@
+import { google } from 'googleapis';
+
+export class RateLimiter {
+  constructor() {
+    this.nextAllowedAt = 0;
+  }
+
+  async run(requestsPerMinute, operation) {
+    for (let attempt = 0; ; attempt += 1) {
+      const interval = Math.ceil(60_000 / Math.max(1, Number(requestsPerMinute) || 45));
+      const scheduledAt = Math.max(Date.now(), this.nextAllowedAt);
+      this.nextAllowedAt = scheduledAt + interval;
+      const wait = scheduledAt - Date.now();
+      if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait));
+      try {
+        return await operation();
+      } catch (error) {
+        const status = error.code || error.response?.status;
+        if (attempt >= 2 || (status !== 429 && (status < 500 || status > 599))) throw error;
+        const retryAfter = retryAfterMilliseconds(error);
+        const backoff = retryAfter ?? 1_000 * 2 ** attempt;
+        this.nextAllowedAt = Math.max(this.nextAllowedAt, Date.now() + backoff);
+      }
+    }
+  }
+}
+
+function retryAfterMilliseconds(error) {
+  const value = error.response?.headers?.get?.('retry-after') || error.response?.headers?.['retry-after'];
+  if (!value) return null;
+  const seconds = Number(value);
+  if (Number.isFinite(seconds)) return Math.max(0, seconds * 1_000);
+  const date = Date.parse(value);
+  return Number.isNaN(date) ? null : Math.max(0, date - Date.now());
+}
+
+export function createSheetsClient(googleConfig) {
+  if (googleConfig.mode === 'serviceAccount') {
+    const credentials = JSON.parse(googleConfig.serviceAccountJson);
+    const auth = new google.auth.GoogleAuth({
+      credentials,
+      scopes: ['https://www.googleapis.com/auth/spreadsheets.readonly']
+    });
+    return google.sheets({ version: 'v4', auth });
+  }
+
+  if (!googleConfig.apiKey) throw new Error('A Google API key is required.');
+  return google.sheets({ version: 'v4', auth: googleConfig.apiKey });
+}
+
+export async function listTabs(client, spreadsheetId, limiter, requestsPerMinute) {
+  const response = await limiter.run(requestsPerMinute, () => client.spreadsheets.get({
+    spreadsheetId,
+    fields: 'properties.title,sheets.properties(sheetId,title)'
+  }));
+  return {
+    title: response.data.properties?.title ?? spreadsheetId,
+    sheets: (response.data.sheets ?? []).map(({ properties }) => ({
+      id: properties.sheetId,
+      title: properties.title
+    }))
+  };
+}
+
+export async function readTabs(client, publication, limiter, requestsPerMinute) {
+  const maxRows = Math.max(1, Math.min(100_000, Number(publication.maxRows) || 10_000));
+  const ranges = publication.sheets.map((sheet) => {
+    const title = `'${sheet.title.replaceAll("'", "''")}'`;
+    return publication.range?.trim() ? `${title}!${publication.range.trim()}` : `${title}!1:${maxRows}`;
+  });
+  const response = await limiter.run(requestsPerMinute, () => client.spreadsheets.values.batchGet({
+    spreadsheetId: publication.spreadsheetId,
+    ranges,
+    majorDimension: 'ROWS'
+  }));
+
+  return (response.data.valueRanges ?? []).map((valueRange, index) => formatSheet(
+    publication.sheets[index].title,
+    valueRange.values ?? [],
+    publication
+  ));
+}
+
+function formatSheet(sheet, values, publication) {
+  const maxRows = Math.max(1, Math.min(100_000, Number(publication.maxRows) || 10_000));
+  const limitedValues = values.slice(0, maxRows);
+  const rows = publication.omitBlankRows
+    ? limitedValues.filter((row) => row.some((cell) => String(cell).trim() !== ''))
+    : limitedValues;
+  if (publication.dataFormat !== 'objects') return { sheet, values: rows };
+  const [headers = [], ...records] = rows;
+  const normalizedHeaders = headers.map((header) => String(header).trim());
+  if (normalizedHeaders.some((header) => !header) || new Set(normalizedHeaders).size !== normalizedHeaders.length) {
+    throw new Error(`Object output for "${sheet}" requires non-empty, unique header names.`);
+  }
+  return {
+    sheet,
+    rows: records.map((row) => Object.fromEntries(normalizedHeaders.map((header, index) => [header, row[index] ?? ''])))
+  };
+}
@@ -0,0 +1,30 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+
+const logPath = path.join(process.cwd(), 'data', 'request-log.json');
+const maximumEntries = 500;
+
+async function loadEntries() {
+  try {
+    return JSON.parse(await readFile(logPath, 'utf8'));
+  } catch (error) {
+    if (error.code === 'ENOENT') return [];
+    throw error;
+  }
+}
+
+export async function recordRequest(entry) {
+  const entries = await loadEntries();
+  entries.push(entry);
+  await mkdir(path.dirname(logPath), { recursive: true });
+  await writeFile(logPath, `${JSON.stringify(entries.slice(-maximumEntries), null, 2)}\n`, { mode: 0o600 });
+}
+
+export async function recentRequests(limit = 50) {
+  return (await loadEntries()).slice(-limit).reverse();
+}
+
+export async function clearRequests() {
+  await mkdir(path.dirname(logPath), { recursive: true });
+  await writeFile(logPath, '[]\n', { mode: 0o600 });
+}