From abf08b6334a0ef5bdb3e6e2b73a6a8eb76bdb813 Mon Sep 17 00:00:00 2001 From: Mark Rapson Date: Sun, 5 Jul 2026 14:59:48 +0100 Subject: [PATCH] Version 2 Rewrite --- .gitattributes | 2 - .gitignore | 35 +- GUIDE_API_KEY.md | 60 ++ GUIDE_SERVICE_ACCOUNT.md | 61 ++ config-example.json | 37 -- config.example.json | 30 + index.js | 428 +++++++++++--- lib/auth.js | 199 +++++++ lib/config.js | 48 ++ lib/rate.js | 45 ++ package-lock.json | 1159 ++++++++++++++++++++++++++++++++++++++ package.json | 12 +- readme.md | 80 +-- 13 files changed, 2037 insertions(+), 159 deletions(-) delete mode 100644 .gitattributes create mode 100644 GUIDE_API_KEY.md create mode 100644 GUIDE_SERVICE_ACCOUNT.md delete mode 100644 config-example.json create mode 100644 config.example.json create mode 100644 lib/auth.js create mode 100644 lib/config.js create mode 100644 lib/rate.js create mode 100644 package-lock.json diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index dfe0770..0000000 --- a/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -# Auto detect text files and perform LF normalization -* text=auto diff --git a/.gitignore b/.gitignore index ebf53a8..38f6bbb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,32 @@ -.vscode -node_modules -output -package-lock.json +# Node modules +node_modules/ + +# Project outputs +output/ +coverage/ +dist/ + +# Credentials and config (do NOT commit secrets) +credentials.json +credentials-*.json config.json +config-*.json +.env +.env.* + +# Logs +logs/ +*.log +npm-debug.log +yarn-error.log + +# Editor directories +.vscode/ +.idea/ + +# OS files +.DS_Store +Thumbs.db + +# Misc +.cache/ diff --git a/GUIDE_API_KEY.md b/GUIDE_API_KEY.md new file mode 100644 index 0000000..5a9b65e --- /dev/null +++ b/GUIDE_API_KEY.md @@ -0,0 +1,60 @@ +# API Key Setup Guide + +This guide explains how to create and configure API keys for use with the Google Sheets Rate Assistant. The application expects an `apiKeys` array in `config.json` (even a single key should be provided as a single-element array). + +## Overview + +- The app supports using API keys instead of a service account by providing an `apiKeys` array in `config.json`. +- Each entry in `apiKeys` may be a string (the key) or an object with `{ key, rateLimitPerMinute }`. + +## Create an API key + +1. Go to the Google Cloud Console: https://console.cloud.google.com/ +2. Select or create a project. +3. Enable the **Google Sheets API** (and **Drive API** if you need it) under "APIs & Services > Library". +4. Go to "APIs & Services > Credentials" and click **Create credentials > API key**. Copy the key value. + +## Secure the key + +- Click **Restrict key** on the credentials page and set: + - Application restrictions (IP addresses or HTTP referrers) where possible. + - API restrictions: select **Google Sheets API** only. +- Do not leave keys unrestricted in production. + +## Example `config.json` snippet + +To configure one key: + +```json +{ + "apiKeys": [ + "YOUR_API_KEY_1" + ] +} +``` + +To configure multiple keys with per-key rate limits: + +```json +{ + "apiKeys": [ + { "key": "API_KEY_1", "rateLimitPerMinute": 60 }, + { "key": "API_KEY_2", "rateLimitPerMinute": 30 } + ] +} +``` + +The app will round-robin requests across configured keys and respect `rateLimitPerMinute` values. + +## Storing keys safely + +- Do not commit `config.json` with keys to version control. Add `config.json` to `.gitignore`. +- Alternatively, keep a `config.json.example` in the repo and place the real keys in a local `config.json`. +- If you prefer to use environment variables, you can have a small loader script that reads an env var and writes `config.json` before running the app. + +## Troubleshooting + +- If you get quota or 403 errors, check API restrictions, project quotas, and whether the key is restricted to the wrong IP/referrer. +- Use multiple keys to spread requests if you hit per-key quotas. + +*** End of guide *** diff --git a/GUIDE_SERVICE_ACCOUNT.md b/GUIDE_SERVICE_ACCOUNT.md new file mode 100644 index 0000000..41a9be0 --- /dev/null +++ b/GUIDE_SERVICE_ACCOUNT.md @@ -0,0 +1,61 @@ +# Google Service Account Setup for Google Sheets Rate Assistant + +This guide explains how to create a Google service account, enable the Sheets API, and configure the project to use service account credentials. + +## 1. Enable the Google Sheets API + +1. Open the Google Cloud Console: https://console.cloud.google.com/ +2. Select an existing project or create a new one. +3. In the left menu, open **APIs & Services > Library**. +4. Search for **Google Sheets API** and click it. +5. Click **Enable**. + +## 2. Create a service account + +1. In the Cloud Console, open **IAM & Admin > Service Accounts**. +2. Click **Create Service Account**. +3. Enter a name and description. +4. Click **Create and continue**. +5. Skip granting optional roles or add a minimal role if required. +6. Click **Done**. + +## 3. Create and download credentials + +1. Find the service account you created in the list. +2. Click the service account name. +3. Open the **Keys** tab. +4. Click **Add Key > Create new key**. +5. Choose **JSON** and click **Create**. +6. Save the downloaded JSON file securely in your project folder, for example `credentials.json`. + +## 4. Share your Google Sheets with the service account + +1. Open the Google Sheet you want to export. +2. Click **Share**. +3. Add the service account email address, which looks like `...@...iam.gserviceaccount.com`. +4. Give it **Viewer** access. + +## 5. Configure the project + +In `config.json`, set the `credentialsPath` field to the JSON file path, and remove any `apiKey` or `apiKeys` entries if you are using service account auth. + +Example: + +```json +{ + "credentialsPath": "credentials.json", + "rateLimitPerMinute": 50, + "documents": [ + { + "documentId": "YOUR_SPREADSHEET_ID", + "outputDir": "output/your-document", + "sheets": [ + { + "name": "Sheet1", + "outputFilename": "sheet1.csv" + } + ] + } + ] +} +``` \ No newline at end of file diff --git a/config-example.json b/config-example.json deleted file mode 100644 index a28b1d8..0000000 --- a/config-example.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "outputFolder": "output", - "apis": [ - { - "apiKey": "--- API key ---", - "documents": [ - { - "googleDocId": "--- Google Sheet ID ---", - "sheets": [ - "-- Sheet Name ---", - "-- 2nd Sheet Name ---" - ], - "pollRate": 1500 - }, - { - "googleDocId": "--- 2nd Google Sheet ID ---", - "sheets": [ - "-- Sheet Name ---" - ], - "pollRate": 10000 - } - ] - }, - { - "apiKey": "--- 2nd API Key ---", - "documents": [ - { - "googleDocId": "--- 3rd Google Sheet ID ---", - "sheets": [ - "-- Sheet Name ---" - ], - "pollRate": 5000 - } - ] - } - ] -} \ No newline at end of file diff --git a/config.example.json b/config.example.json new file mode 100644 index 0000000..6015870 --- /dev/null +++ b/config.example.json @@ -0,0 +1,30 @@ +{ + "apiKeys": [ + { + "key": "YOUR_API_KEY_1", + "rateLimitPerMinute": 50 + }, + { + "key": "YOUR_API_KEY_2", + "rateLimitPerMinute": 50 + } + ], + "serviceAccounts": [ + { + "path": "credentials.json", + "rateLimitPerMinute": 50 + } + ], + "documents": [ + { + "documentId": "YOUR_SPREADSHEET_ID", + "outputDir": "output/your-document", + "sheets": [ + { + "name": "Sheet1", + "outputFilename": "sheet1.csv" + } + ] + } + ] +} diff --git a/index.js b/index.js index 773ebbe..dc1ba50 100644 --- a/index.js +++ b/index.js @@ -1,84 +1,358 @@ -const { GoogleSpreadsheet } = require('google-spreadsheet'); +// Google Sheets Rate Assistant +// Orchestrates periodic fetches of Google Sheets and writes CSV outputs. +// Uses `./lib/config`, `./lib/rate`, and `./lib/auth` for configuration, +// rate-limiting, and authentication/fetch helpers respectively. +// === Imports & constants === const fs = require('fs'); -const readline = require('readline').createInterface({ - input: process.stdin, - output: process.stdout, -}); -const config = require('./config.json'); +const path = require('path'); +const {google} = require('googleapis'); +const stringifyModule = require('csv-stringify/sync'); +const stringify = typeof stringifyModule === 'function' + ? stringifyModule + : stringifyModule.default || stringifyModule.stringify || stringifyModule; +const {loadConfig, resolveOutputPath, validateConfig} = require('./lib/config'); +const {buildThrottle, applyRateLimitBuffer, getTotalRateLimitPerMinute} = require('./lib/rate'); +const authHelpers = require('./lib/auth'); +const {createApiKeyBatchFetcher, createServiceAccountBatchFetcher, getFirstApiKey, getFirstServiceAccountAuthClient, loadAuth} = authHelpers; -const check = new Promise(function (resolve) { - var count = 0, - position = {}; - config.apis.forEach(function (document, d) { - var totalRate = 0; - document.documents.forEach(function (item, i) { - item.sheets.forEach(function (sh, s) { - count += 1; - position[d + "" + i + "" + s] = count; - }); - totalRate += (60 / (item.pollRate / 1000)); - //config.apis[d].documents[i].sheets.itemNo = count; - }); - config.position = position; - var stdin = process.openStdin(); - if (totalRate > 50) { - console.log('\n---------- RATE LIMIT WARNING ----------'); - readline.question(`\nYour poll rate will be ${totalRate} per minute, this is above the recommendation of 50 per minute.\nIf the GoogleAPI limit is reached (60 per min on free) you will receive no updates until a break period has passed.\n(API: ${document.apiKey})\n\nAre you sure you want to continue? [y/n]: `, answer => { - if (answer == 'y') { - readline.close(); - resolve(); - } else { - process.exit(); - } - }); - } else if (config.apis.length - 1 == d) { - resolve(); +// === Status rendering / console display === +// Renders per-sheet status lines when running in a TTY, updating in-place. + +function mapDocumentToFilename(documentConfig, sheetName) { + const safeName = sheetName.replace(/[\\/:*?"<>|]/g, '_'); + return `${documentConfig.documentId}-${safeName}.csv`; +} + +const statusManager = { + keys: [], + statuses: new Map(), + lastRenderLines: 0, + enabled: process.stdout && process.stdout.isTTY, +}; + +// === Document grouping & mapping === +// Map `config.documents` entries to internal groups with resolved output paths. + +function renderStatusBlock() { + if (!statusManager.enabled || statusManager.keys.length === 0) { + return; + } + + if (statusManager.lastRenderLines > 0) { + process.stdout.write(`\x1B[${statusManager.lastRenderLines}F`); + } + + for (const key of statusManager.keys) { + const statusText = statusManager.statuses.get(key) || `${key} - pending`; + process.stdout.clearLine(0); + process.stdout.cursorTo(0); + process.stdout.write(statusText + '\n'); + } + + statusManager.lastRenderLines = statusManager.keys.length; +} + +function setTaskStatus(taskId, statusText) { + statusManager.statuses.set(taskId, statusText); + renderStatusBlock(); +} + +function initializeTaskStatuses(tasks) { + statusManager.keys = tasks.map(task => `${task.documentId}:${task.sheetName}`); + statusManager.statuses.clear(); + statusManager.lastRenderLines = 0; + + for (const key of statusManager.keys) { + statusManager.statuses.set(key, `${key} - pending`); + } + + renderStatusBlock(); +} + +// Authentication helpers: see `./lib/auth` for credential loaders and +// per-credential, throttled batch fetchers (API key and service account modes). + +async function fetchSpreadsheetTitle(context, documentId) { + const sheetOptions = {version: 'v4'}; + if (context.authClient) { + sheetOptions.auth = context.authClient; + } + + const sheets = google.sheets(sheetOptions); + const request = { + spreadsheetId: documentId, + fields: 'properties/title', + }; + + if (context.apiKey) { + request.key = context.apiKey; + } + + const response = await sheets.spreadsheets.get(request); + return response.data.properties?.title || documentId; +} + +function buildDocumentGroups(config) { + const groups = []; + + if (!Array.isArray(config.documents)) { + throw new Error('`documents` must be an array in config.json'); + } + + for (const documentConfig of config.documents) { + if (!documentConfig.documentId || !Array.isArray(documentConfig.sheets)) { + throw new Error('Each document must include documentId and sheets array'); + } + + const documentOutputDir = documentConfig.outputDir || './output'; + const sheets = []; + + for (const sheetConfig of documentConfig.sheets) { + const name = typeof sheetConfig === 'string' ? sheetConfig : sheetConfig.name; + const outputFilename = sheetConfig.outputFilename || mapDocumentToFilename(documentConfig, name); + const outputPath = resolveOutputPath(documentOutputDir, outputFilename); + + sheets.push({ + sheetName: name, + outputPath, + }); + } + + groups.push({ + documentId: documentConfig.documentId, + sheets, + }); + } + + return groups; +} + +// === Batch fetching (fallback) === +// `index.js` provides a simple `batchFetchDocumentSheets` fallback used when +// no per-credential batch fetcher is configured by `./lib/auth`. +async function batchFetchDocumentSheets(context, documentId, sheetNames) { + const sheetOptions = {version: 'v4'}; + if (context.authClient) { + sheetOptions.auth = context.authClient; + } + + const sheets = google.sheets(sheetOptions); + const request = { + spreadsheetId: documentId, + ranges: sheetNames, + }; + + if (context.apiKey) { + request.key = context.apiKey; + } + + const response = await sheets.spreadsheets.values.batchGet(request); + const valueRanges = response.data.valueRanges || []; + const rowsBySheet = {}; + + for (let i = 0; i < sheetNames.length; i += 1) { + const sheetName = sheetNames[i]; + rowsBySheet[sheetName] = valueRanges[i]?.values || []; + } + + return rowsBySheet; +} + +function formatStatusTimestamp(date = new Date()) { + const pad = value => String(value).padStart(2, '0'); + const year = date.getFullYear(); + const month = pad(date.getMonth() + 1); + const day = pad(date.getDate()); + const hours = pad(date.getHours()); + const minutes = pad(date.getMinutes()); + const seconds = pad(date.getSeconds()); + return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`; +} + +// === Execution / saving === +// Functions that execute per-document fetches and write CSV outputs. + +function writeCsv(outputPath, rows) { + const csv = stringify(rows, { + header: false, + quoted: true, + }); + fs.writeFileSync(outputPath, csv, 'utf8'); +} + +async function executeDocumentGroup(group, fetcher) { + const sheetNames = group.sheets.map(sheet => sheet.sheetName); + + for (const sheet of group.sheets) { + const displayName = `${group.documentDisplayName || group.documentId}:${sheet.sheetName}`; + setTaskStatus(`${group.documentId}:${sheet.sheetName}`, `${displayName} - pulling...`); + } + + const rowsBySheet = await fetcher(group.documentId, sheetNames); + + for (const sheet of group.sheets) { + const rows = rowsBySheet[sheet.sheetName] || []; + writeCsv(sheet.outputPath, rows); + const relativePath = path.relative(process.cwd(), sheet.outputPath) || sheet.outputPath; + const timestamp = formatStatusTimestamp(); + const displayName = `${group.documentDisplayName || group.documentId}:${sheet.sheetName}`; + setTaskStatus( + `${group.documentId}:${sheet.sheetName}`, + `${displayName} - saved to ${relativePath} (updated ${timestamp})` + ); + } +} + +function formatGoogleError(err) { + if (err && err.response && err.response.data) { + return `${err.message} (${JSON.stringify(err.response.data)})`; + } + return err.message || String(err); +} + +function interpretGoogleSheetsError(err, documentId, sheetName) { + const message = formatGoogleError(err); + const notFound = err && err.response && err.response.status === 404; + const requestedEntity = message.includes('Requested entity was not found'); + + if (notFound || requestedEntity) { + let detail = `Spreadsheet not found or inaccessible: ${documentId}`; + if (sheetName) { + detail += `, sheet: ${sheetName}`; + } + detail += '. Check that the spreadsheet ID is correct, that the sheet exists, and that the document is shared with the service account or is publicly readable when using an API key.'; + return detail; + } + + return message; +} + +function sleep(milliseconds) { + return new Promise(resolve => setTimeout(resolve, milliseconds)); +} + +// === Error formatting / interpretation === + +// === Main scheduler === +// Orchestrates startup, selects the appropriate fetcher (API keys, +// service accounts, or single credentials), and runs the periodic loop. +async function scheduleRuns() { + const config = loadConfig(); + try { + validateConfig(config); + } catch (e) { + console.error('Configuration error:', e.message); + process.exit(1); + } + const apiBatchFetcher = createApiKeyBatchFetcher(config); + const svcBatchFetcher = createServiceAccountBatchFetcher(config); + let fetcher; + + if (apiBatchFetcher) { + fetcher = apiBatchFetcher; + } else if (svcBatchFetcher) { + fetcher = svcBatchFetcher; + } else { + const auth = await loadAuth(config); + const baseRate = config.rateLimitPerMinute || 50; + const bufferedRate = applyRateLimitBuffer(baseRate); + const throttle = buildThrottle(bufferedRate); + fetcher = throttle((documentId, sheetNames) => batchFetchDocumentSheets({authClient: auth}, documentId, sheetNames)); + } + + const groups = buildDocumentGroups(config); + if (groups.length === 0) { + throw new Error('No sheets configured to pull in config.json'); + } + + const firstApiKey = getFirstApiKey(config); + const titleMap = {}; + + // If running in single-auth mode, try to warm the token + if (!apiBatchFetcher && !svcBatchFetcher) { + try { + const singleAuth = await loadAuth(config); + if (singleAuth) { + try { + const at = await singleAuth.getAccessToken(); + console.log('Service account token acquired (len=', (at && at.token) ? at.token.length : 'no', ')'); + } catch (e) { + console.error('Failed to acquire initial access token from service account:', e && e.message); } - }); -}) + } + } catch (_) { + // loadAuth will throw if missing credentials; continue and let title fetch handle errors + } + } -check.then(function () { - console.clear(); - process.stdout.cursorTo(0, 1); - console.log(`Ctrl+C to kill the application.`); - config.apis.forEach(function (api, d) { - api.documents.forEach(function (docs, i) { - const doc = new GoogleSpreadsheet(docs.googleDocId); - doc.useApiKey(api.apiKey); + // build a context for fetching titles: prefer API key, then first service account, then single-auth if present + let titleContext = null; + if (firstApiKey) { + titleContext = {apiKey: firstApiKey}; + } else if (svcBatchFetcher) { + const firstSvcAuth = getFirstServiceAccountAuthClient(config); + if (firstSvcAuth) titleContext = {authClient: firstSvcAuth}; + } else { + try { + const singleAuth = await loadAuth(config); + if (singleAuth) titleContext = {authClient: singleAuth}; + } catch (_) { + // ignore + } + } - setInterval(function () { - (async function () { - await doc.loadInfo(); - var titleClean = doc.title.replace(/[^a-zA-Z0-9 ]/g, ''); - if (!fs.existsSync(`${config.outputFolder}/${titleClean}/`)) { - fs.mkdirSync(`${config.outputFolder}/${titleClean}/`, { recursive: true }); - } - docs.sheets.forEach(function (sheet, s) { - (async function () { - downloadCSV = await doc.sheetsByTitle[sheet].downloadAsCSV(); - fs.writeFile(`${config.outputFolder}/${titleClean}/${sheet.replace(/[^a-zA-Z0-9 ]/g)}.csv`, downloadCSV, function (err) { - if (err) { - return console.log(err); - } - const date = new Date(); - process.stdout.cursorTo(0, config.position[d + "" + i + "" + s] + 2); - process.stdout.clearLine(); - console.log(`"${doc.title} - ${sheet}" last updated at ${pad(date.getHours(), 2)}:${pad(date.getMinutes(), 2)}:${pad(date.getSeconds(), 2)}`); - process.stdout.cursorTo(31, 1); - }); - }()); - }); - }()); - }, docs.pollRate) - }); + for (const group of groups) { + try { + titleMap[group.documentId] = await fetchSpreadsheetTitle(titleContext || {}, group.documentId); + } catch (err) { + titleMap[group.documentId] = group.documentId; + console.error(`Failed to load title for ${group.documentId}: ${formatGoogleError(err)}`); + } + } - }); -}, function (err) { - console.log(err); -}) + const tasks = groups.flatMap(group => + group.sheets.map(sheet => ({ + documentId: group.documentId, + sheetName: sheet.sheetName, + })) + ); -function pad(n, width, z) { - z = z || '0'; - n = n + ''; - return n.length >= width ? n : new Array(width - n.length + 1).join(z) + n; -} \ No newline at end of file + tasks.forEach(task => { + task.documentDisplayName = titleMap[task.documentId] || task.documentId; + }); + + groups.forEach(group => { + group.documentDisplayName = titleMap[group.documentId] || group.documentId; + }); + + const totalRateLimit = getTotalRateLimitPerMinute(config); + const intervalMs = Math.max(1, Math.ceil(60000 / totalRateLimit)); + console.log(`Configured for ${totalRateLimit} requests per minute across ${tasks.length} tasks (${intervalMs}ms interval)`); + initializeTaskStatuses(tasks); + + let currentIndex = 0; + while (true) { + const group = groups[currentIndex]; + currentIndex = (currentIndex + 1) % groups.length; + + try { + await executeDocumentGroup(group, fetcher); + } catch (err) { + for (const sheet of group.sheets) { + setTaskStatus( + `${group.documentId}:${sheet.sheetName}`, + `${group.documentId}:${sheet.sheetName} - failed: ${interpretGoogleSheetsError(err, group.documentId, sheet.sheetName)}` + ); + } + } + + await sleep(intervalMs); + } +} + +if (require.main === module) { + scheduleRuns().catch(err => { + console.error('Application error:', err.message || err); + process.exit(1); + }); +} diff --git a/lib/auth.js b/lib/auth.js new file mode 100644 index 0000000..b3895d6 --- /dev/null +++ b/lib/auth.js @@ -0,0 +1,199 @@ +const fs = require('fs'); +const path = require('path'); +const {google} = require('googleapis'); +const {applyRateLimitBuffer} = require('./rate'); +const pThrottleModule = require('p-throttle'); +const pThrottle = pThrottleModule.default || pThrottleModule; + +function loadApiKeys(config) { + const apiKeys = []; + const defaultRateLimit = config.rateLimitPerMinute || 50; + + if (Array.isArray(config.apiKeys)) { + for (const entry of config.apiKeys) { + if (typeof entry === 'string') { + apiKeys.push({key: entry, rateLimitPerMinute: defaultRateLimit}); + } else if (entry && typeof entry === 'object') { + if (!entry.key) { + throw new Error('Each entry in `apiKeys` must include `key`'); + } + apiKeys.push({ + key: entry.key, + rateLimitPerMinute: entry.rateLimitPerMinute || defaultRateLimit, + }); + } else { + throw new Error('`apiKeys` entries must be strings or objects'); + } + } + } + + return apiKeys; +} + +function loadServiceAccounts(config) { + const accounts = []; + const defaultRateLimit = config.rateLimitPerMinute || 50; + + if (!Array.isArray(config.serviceAccounts)) { + return accounts; + } + + for (const entry of config.serviceAccounts) { + if (typeof entry === 'string') { + accounts.push({path: entry, rateLimitPerMinute: defaultRateLimit}); + } else if (entry && typeof entry === 'object') { + if (!entry.path && !entry.credentialsPath) { + throw new Error('Each entry in `serviceAccounts` must include `path` (or `credentialsPath`)'); + } + const p = entry.path || entry.credentialsPath; + accounts.push({path: p, rateLimitPerMinute: entry.rateLimitPerMinute || defaultRateLimit}); + } else { + throw new Error('`serviceAccounts` entries must be strings or objects'); + } + } + + return accounts; +} + +function createServiceAccountBatchFetcher(config) { + const accounts = loadServiceAccounts(config); + if (accounts.length === 0) return null; + + const fetchers = accounts.map(({path: credentialsPath, rateLimitPerMinute}) => { + const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath); + if (!fs.existsSync(resolvedPath)) { + throw new Error(`Service account credentials not found: ${resolvedPath}`); + } + + let credentials; + try { + credentials = JSON.parse(fs.readFileSync(resolvedPath, 'utf8')); + } catch (e) { + throw new Error(`Failed to read/parse service account credentials at ${resolvedPath}: ${e.message}`); + } + + let authClient; + try { + authClient = google.auth.fromJSON(credentials); + } catch (e) { + throw new Error(`Failed to initialize auth client from credentials at ${resolvedPath}: ${e.message}`); + } + authClient.scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly']; + + const bufferedRateLimit = applyRateLimitBuffer(rateLimitPerMinute); + const interval = Math.ceil(60000 / bufferedRateLimit); + const throttled = pThrottle({limit: 1, interval})(async (documentId, sheetNames) => { + const sheets = google.sheets({version: 'v4', auth: authClient}); + const request = {spreadsheetId: documentId, ranges: sheetNames}; + const response = await sheets.spreadsheets.values.batchGet(request); + const valueRanges = response.data.valueRanges || []; + const rowsBySheet = {}; + for (let i = 0; i < sheetNames.length; i += 1) { + const sheetName = sheetNames[i]; + rowsBySheet[sheetName] = valueRanges[i]?.values || []; + } + return rowsBySheet; + }); + + return {authClient, fetch: throttled}; + }); + + let currentIndex = 0; + return async (documentId, sheetNames) => { + const worker = fetchers[currentIndex]; + currentIndex = (currentIndex + 1) % fetchers.length; + return worker.fetch(documentId, sheetNames); + }; +} + +function createApiKeyBatchFetcher(config) { + const apiKeys = loadApiKeys(config); + if (apiKeys.length === 0) { + return null; + } + + const fetchers = apiKeys.map(({key, rateLimitPerMinute}) => { + const bufferedRateLimit = applyRateLimitBuffer(rateLimitPerMinute); + const interval = Math.ceil(60000 / bufferedRateLimit); + const throttled = pThrottle({limit: 1, interval})(async (documentId, sheetNames) => { + const sheets = google.sheets({version: 'v4'}); + const request = {spreadsheetId: documentId, ranges: sheetNames, key}; + const response = await sheets.spreadsheets.values.batchGet(request); + const valueRanges = response.data.valueRanges || []; + const rowsBySheet = {}; + for (let i = 0; i < sheetNames.length; i += 1) { + const sheetName = sheetNames[i]; + rowsBySheet[sheetName] = valueRanges[i]?.values || []; + } + return rowsBySheet; + }); + return {key, fetch: throttled}; + }); + + let currentIndex = 0; + return async (documentId, sheetNames) => { + const worker = fetchers[currentIndex]; + currentIndex = (currentIndex + 1) % fetchers.length; + return worker.fetch(documentId, sheetNames); + }; +} + +function getFirstApiKey(config) { + if (Array.isArray(config.apiKeys) && config.apiKeys.length > 0) { + const firstEntry = config.apiKeys[0]; + return typeof firstEntry === 'string' ? firstEntry : firstEntry.key; + } + + return null; +} + +function getFirstServiceAccountAuthClient(config) { + if (!Array.isArray(config.serviceAccounts) || config.serviceAccounts.length === 0) return null; + const firstEntry = config.serviceAccounts[0]; + const entryObj = typeof firstEntry === 'string' ? {path: firstEntry} : firstEntry; + const credentialsPath = entryObj.path || entryObj.credentialsPath; + if (!credentialsPath) return null; + + const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath); + if (!fs.existsSync(resolvedPath)) return null; + + try { + const raw = fs.readFileSync(resolvedPath, 'utf8'); + const credentials = JSON.parse(raw); + const authClient = google.auth.fromJSON(credentials); + authClient.scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly']; + return authClient; + } catch (e) { + console.error(`Failed to load first service account from ${resolvedPath}: ${e.message}`); + return null; + } +} + +function loadAuth(config) { + if (!config || typeof config !== 'object') return null; + + if (config.credentialsPath) { + const credentialsPath = config.credentialsPath; + const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath); + if (!fs.existsSync(resolvedPath)) { + throw new Error(`Credentials file not found: ${resolvedPath}`); + } + const raw = fs.readFileSync(resolvedPath, 'utf8'); + const credentials = JSON.parse(raw); + const authClient = google.auth.fromJSON(credentials); + authClient.scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly']; + return authClient; + } + + return getFirstServiceAccountAuthClient(config); +} + +module.exports = { + loadApiKeys, + loadServiceAccounts, + createServiceAccountBatchFetcher, + createApiKeyBatchFetcher, + getFirstApiKey, + getFirstServiceAccountAuthClient + , loadAuth +}; diff --git a/lib/config.js b/lib/config.js new file mode 100644 index 0000000..7eea2ea --- /dev/null +++ b/lib/config.js @@ -0,0 +1,48 @@ +const fs = require('fs'); +const path = require('path'); + +const CONFIG_PATH = path.resolve(__dirname, '..', 'config.json'); + +function loadConfig() { + if (!fs.existsSync(CONFIG_PATH)) { + throw new Error(`Missing config file at ${CONFIG_PATH}`); + } + const json = fs.readFileSync(CONFIG_PATH, 'utf8'); + return JSON.parse(json); +} + +function resolveOutputPath(outputDir, filename) { + if (!path.isAbsolute(outputDir)) { + outputDir = path.resolve(process.cwd(), outputDir); + } + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, {recursive: true}); + } + return path.join(outputDir, filename); +} + +function validateConfig(config) { + if (!config || typeof config !== 'object') { + throw new Error('Missing or invalid configuration object'); + } + + if (!Array.isArray(config.documents) || config.documents.length === 0) { + throw new Error('`documents` must be a non-empty array in config.json'); + } + + const hasApiKeys = Array.isArray(config.apiKeys) && config.apiKeys.length > 0; + const hasSvcAccounts = Array.isArray(config.serviceAccounts) && config.serviceAccounts.length > 0; + const hasCredPath = !!config.credentialsPath; + + if (!hasApiKeys && !hasSvcAccounts && !hasCredPath) { + throw new Error('Configuration must include at least one of: `apiKeys`, `serviceAccounts`, or `credentialsPath`'); + } + + // Basic validation of documents/sheets structure + for (const doc of config.documents) { + if (!doc.documentId) throw new Error('Each document must include a `documentId`'); + if (!Array.isArray(doc.sheets) || doc.sheets.length === 0) throw new Error(`Document ${doc.documentId} must include a non-empty 'sheets' array`); + } +} + +module.exports = {loadConfig, resolveOutputPath, validateConfig}; diff --git a/lib/rate.js b/lib/rate.js new file mode 100644 index 0000000..a4c88bd --- /dev/null +++ b/lib/rate.js @@ -0,0 +1,45 @@ +const pThrottleModule = require('p-throttle'); +const pThrottle = pThrottleModule.default || pThrottleModule; + +function buildThrottle(maxRequestsPerMinute) { + const limit = Math.max(1, Math.floor(maxRequestsPerMinute)); + const interval = Math.ceil(60000 / limit); + return pThrottle({limit: 1, interval}); +} + +function applyRateLimitBuffer(rateLimitPerMinute) { + return Math.max(1, Math.floor(rateLimitPerMinute * 0.95)); +} + +function getTotalRateLimitPerMinute(config) { + const defaultRateLimit = config.rateLimitPerMinute || 50; + let totalRateLimit = 0; + + if (Array.isArray(config.apiKeys) && config.apiKeys.length > 0) { + for (const entry of config.apiKeys) { + if (typeof entry === 'string') { + totalRateLimit += applyRateLimitBuffer(defaultRateLimit); + } else if (entry && typeof entry === 'object') { + totalRateLimit += applyRateLimitBuffer(entry.rateLimitPerMinute || defaultRateLimit); + } + } + } + + if (Array.isArray(config.serviceAccounts) && config.serviceAccounts.length > 0) { + for (const entry of config.serviceAccounts) { + if (typeof entry === 'string') { + totalRateLimit += applyRateLimitBuffer(defaultRateLimit); + } else if (entry && typeof entry === 'object') { + totalRateLimit += applyRateLimitBuffer(entry.rateLimitPerMinute || defaultRateLimit); + } + } + } + + if (totalRateLimit === 0) { + totalRateLimit = applyRateLimitBuffer(defaultRateLimit); + } + + return Math.max(1, totalRateLimit); +} + +module.exports = {buildThrottle, applyRateLimitBuffer, getTotalRateLimitPerMinute}; diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..9a1a93f --- /dev/null +++ b/package-lock.json @@ -0,0 +1,1159 @@ +{ + "name": "google-sheets-rate-assistant", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "google-sheets-rate-assistant", + "version": "1.0.0", + "dependencies": { + "csv-stringify": "^6.0.0", + "googleapis": "^173.0.0", + "p-throttle": "^8.1.0" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/brace-expansion": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.1.tgz", + "integrity": "sha512-WR1cURNjuvBLMZBMbqM0UoE+WAfdUcEV1ccD8PVBVOI+Z3ND4+SZbN8RsfT2bMuG1qwz5RFvPukSZm5fF2D5eA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/csv-stringify": { + "version": "6.8.1", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.8.1.tgz", + "integrity": "sha512-tZ6X6TKQyQgCo5OptXcyAbfN1pwmoxEqELPQ7KFazNErx7kiVsDK8o+VYRXhfMl4N9vvOOLXuioquR2MeP847A==", + "license": "MIT" + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT" + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gaxios": { + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.5.tgz", + "integrity": "sha512-5FZy72Rh8LhtjmvDrKkI+lVhrsQrVKVsItxMoDm5mNQE+xR0WVIIs+jzPSJgBvKVsLi24fZhXJIsNI0bihDzFg==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/google-auth-library": { + "version": "10.9.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.9.0.tgz", + "integrity": "sha512-xtvUqvINPhTaBm7nXqlYPcrMHJPm1lCNdSovxnKKhTm+4JsvQ+KGVYJViLoH9Yxu8w+T0Qv5HubzYT9BLrppJg==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/googleapis": { + "version": "173.0.0", + "resolved": "https://registry.npmjs.org/googleapis/-/googleapis-173.0.0.tgz", + "integrity": "sha512-xEJJYLZ4qeenVyfzispNfRjCe9bsv7CzBv5zYFLvScOze9snJ8S9W6hjQ729CWPQt5mvn/JrcRaCHzQiukt0ng==", + "license": "Apache-2.0", + "dependencies": { + "google-auth-library": "^10.2.0", + "googleapis-common": "^8.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/googleapis-common": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/googleapis-common/-/googleapis-common-8.0.2.tgz", + "integrity": "sha512-5MXeQzIZaqCH7B+HJWqhQm946VARpZep6acbWSr/fcgF2cQANq7allgX+i/G0EqF0WyUxB277gtWMzRYHMl9tg==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "gaxios": "7.1.3", + "google-auth-library": "10.5.0", + "google-logging-utils": "1.1.3", + "qs": "^6.7.0", + "url-template": "^2.0.8" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/googleapis-common/node_modules/gaxios": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.3.tgz", + "integrity": "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2", + "rimraf": "^5.0.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/googleapis-common/node_modules/google-auth-library": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.5.0.tgz", + "integrity": "sha512-7ABviyMOlX5hIVD60YOfHw4/CxOfBhyduaYB+wbFWCWoni4N7SLcV46hrVRktuBbZjFC9ONyqamZITN7q3n32w==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.0.0", + "gcp-metadata": "^8.0.0", + "google-logging-utils": "^1.0.0", + "gtoken": "^8.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gtoken": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-8.0.0.tgz", + "integrity": "sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==", + "license": "MIT", + "dependencies": { + "gaxios": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/p-throttle": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-8.1.0.tgz", + "integrity": "sha512-c1wmXavsHZIC4g1OLhOsafK6jZSAeMo0Ap3yivj59PUcCkpacy5YgWdgIp/dB4vp1JZrfBSsPCR0YuADB+ENLQ==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/rimraf": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz", + "integrity": "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==", + "license": "ISC", + "dependencies": { + "glob": "^10.3.7" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/url-template": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", + "integrity": "sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==", + "license": "BSD" + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + } + } +} diff --git a/package.json b/package.json index f0b444b..f0875e9 100644 --- a/package.json +++ b/package.json @@ -1,14 +1,14 @@ { "name": "google-sheets-rate-assistant", - "version": "0.0.1", + "version": "2.0.0", + "description": "Pull Google Sheets data and export to CSV with configurable schedules and rate limiting.", "main": "index.js", "scripts": { - "run": "node index.js" + "start": "node index.js" }, - "author": "", - "license": "ISC", - "description": "Assists in keeping requests under the free rate limit for applications such as vMix.", "dependencies": { - "google-spreadsheet": "^3.3.0" + "csv-stringify": "^6.0.0", + "googleapis": "^173.0.0", + "p-throttle": "^8.1.0" } } diff --git a/readme.md b/readme.md index 32884fe..e186540 100644 --- a/readme.md +++ b/readme.md @@ -1,45 +1,59 @@ -## Google Sheets Rate Assistant +# Google Sheets Rate Assistant -### Instructions +This Node.js application pulls configured Google Sheets documents and exports them as CSV files. -First fill out the config.json with the required information. +## Setup +1. Copy `config.example.json` to `config.json`. +2. Configure authentication in `config.json` — choose one of: -### Run -*Node must be installed on your system* +- Service account: place your service account JSON in `credentials.json` and set `credentialsPath`, or add one or more entries to the `serviceAccounts` array (each entry may be a path string or an object with `path`/`credentialsPath` and optional `rateLimitPerMinute`). +- API keys: add one or more API keys to the `apiKeys` array in `config.json` (each entry may be a string or an object with `key` and optional `rateLimitPerMinute`). -Download the .zip of the repository and extract into a folder. +3. Install dependencies: -1. Open Powershell in the directory -2. run `npm install` -3. run `node index` + npm install -### Config Extract +4. Run the application: -``` + node index.js + +## Setup Guides + +For detailed instructions on authentication methods, please refer to the following guides: + +- [Service Account Setup](./GUIDE_SERVICE_ACCOUNT.md) +- [API Key Setup Guide](./GUIDE_API_KEY.md) + +### Sample `config.json` + +Here is an example `config.json` matching the repository's `config.example.json`: + +```json { - "apiKey": "--- API key ---", - "documents": [ - { - "googleDocId": "--- Google Sheet ID ---", - "sheets": [ - "-- Sheet Name ---", - "-- 2nd Sheet Name ---" - ], - "pollRate": 1500 - }, - { - "googleDocId": "--- 2nd Google Sheet ID ---", - "sheets": [ - "-- Sheet Name ---" - ], - "pollRate": 10000 - } - ] + "apiKeys": [ + { "key": "YOUR_API_KEY_1", "rateLimitPerMinute": 50 }, + { "key": "YOUR_API_KEY_2", "rateLimitPerMinute": 50 } + ], + "serviceAccounts": [ + { "path": "credentials.json", "rateLimitPerMinute": 50 } + ], + "documents": [ + { + "documentId": "YOUR_SPREADSHEET_ID", + "outputDir": "output/your-document", + "sheets": [ + { "name": "Sheet1", "outputFilename": "sheet1.csv" } + ] + } + ] } ``` -+ Lowest PollRate for a single API key is recommended at 1200 to keep these under the free tier of 60 per minute. -+ Each API key should be different, this is not checked. -+ Multiple documents can be polled from a single key and the rate will be calculated. -+ Multiple sheets can be included under each 'googleDocId' and will not add to each APIKey rate limit. +## Rate limiting + +The app enforces a maximum of `rateLimitPerMinute` requests per minute to avoid exceeding API limits. + +## Output + +CSV files are written to `output` by default. Individual documents may override this with their own `outputDir` in `config.json`.