diff --git a/index.js b/index.js index cc32205..c057112 100644 --- a/index.js +++ b/index.js @@ -5,7 +5,7 @@ // === Imports & constants === const fs = require('fs'); const path = require('path'); -const {google} = require('googleapis'); +const readline = require('readline'); const stringifyModule = require('./node_modules/csv-stringify/dist/cjs/sync.cjs'); const stringify = typeof stringifyModule === 'function' ? stringifyModule @@ -14,6 +14,7 @@ const {loadConfig, resolveOutputPath} = require('./lib/config'); const {buildThrottle, applyRateLimitBuffer, getTotalRateLimitPerMinute} = require('./lib/rate'); const {runOnboarding} = require('./lib/onboarding'); const authHelpers = require('./lib/auth'); +const {fetchSpreadsheetTitle, fetchSpreadsheetValuesBatch} = require('./lib/sheets'); const {createApiKeyBatchFetcher, createServiceAccountBatchFetcher, getFirstApiKey, getFirstServiceAccountAuthClient, loadAuth} = authHelpers; // === Status rendering / console display === @@ -26,11 +27,36 @@ function mapDocumentToFilename(sheetName) { const statusManager = { keys: [], + lineIndexes: new Map(), statuses: new Map(), - lastRenderLines: 0, + anchorSaved: false, enabled: process.stdout && process.stdout.isTTY, }; +function clearConsoleForRun() { + if (process.stdout && process.stdout.isTTY && typeof console.clear === 'function') { + console.clear(); + return; + } + + if (process.stdout && typeof process.stdout.write === 'function') { + process.stdout.write('\u001b[2J\u001b[0f'); + } +} + +function limitToTerminalWidth(text) { + const width = Math.max(20, (process.stdout && process.stdout.columns) || 80); + if (text.length <= width) { + return text; + } + + if (width <= 1) { + return text.slice(0, width); + } + + return text.slice(0, width - 1) + '…'; +} + // === Document grouping & mapping === // Map `config.documents` entries to internal groups with resolved output paths. @@ -39,31 +65,54 @@ function renderStatusBlock() { return; } - if (statusManager.lastRenderLines > 0) { - process.stdout.write(`\x1B[${statusManager.lastRenderLines}F`); - } + process.stdout.write('\u001b[s'); - for (const key of statusManager.keys) { + statusManager.keys.forEach((key, index) => { const statusText = statusManager.statuses.get(key) || `${key} - pending`; - process.stdout.clearLine(0); - process.stdout.cursorTo(0); - process.stdout.write(statusText + '\n'); + readline.cursorTo(process.stdout, 0); + readline.clearLine(process.stdout, 0); + process.stdout.write(limitToTerminalWidth(statusText)); + if (index < statusManager.keys.length - 1) { + process.stdout.write('\n'); + } + }); + + statusManager.anchorSaved = true; +} + +function updateStatusLine(taskId, statusText) { + if (!statusManager.enabled || !statusManager.anchorSaved) { + return; } - statusManager.lastRenderLines = statusManager.keys.length; + const lineIndex = statusManager.lineIndexes.get(taskId); + if (lineIndex === undefined) { + return; + } + + process.stdout.write('\u001b[u'); + if (lineIndex > 0) { + readline.moveCursor(process.stdout, 0, lineIndex); + } + readline.cursorTo(process.stdout, 0); + readline.clearLine(process.stdout, 0); + process.stdout.write(limitToTerminalWidth(statusText)); + process.stdout.write('\u001b[u'); } function setTaskStatus(taskId, statusText) { statusManager.statuses.set(taskId, statusText); - renderStatusBlock(); + updateStatusLine(taskId, statusText); } function initializeTaskStatuses(tasks) { statusManager.keys = tasks.map(task => `${task.documentId}:${task.sheetName}`); + statusManager.lineIndexes.clear(); statusManager.statuses.clear(); - statusManager.lastRenderLines = 0; + statusManager.anchorSaved = false; - for (const key of statusManager.keys) { + for (const [index, key] of statusManager.keys.entries()) { + statusManager.lineIndexes.set(key, index); statusManager.statuses.set(key, `${key} - pending`); } @@ -73,26 +122,6 @@ function initializeTaskStatuses(tasks) { // Authentication helpers: see `./lib/auth` for credential loaders and // per-credential, throttled batch fetchers (API key and service account modes). -async function fetchSpreadsheetTitle(context, documentId) { - const sheetOptions = {version: 'v4'}; - if (context.authClient) { - sheetOptions.auth = context.authClient; - } - - const sheets = google.sheets(sheetOptions); - const request = { - spreadsheetId: documentId, - fields: 'properties/title', - }; - - if (context.apiKey) { - request.key = context.apiKey; - } - - const response = await sheets.spreadsheets.get(request); - return response.data.properties?.title || documentId; -} - function buildDocumentGroups(config) { const groups = []; @@ -132,31 +161,7 @@ function buildDocumentGroups(config) { // `index.js` provides a simple `batchFetchDocumentSheets` fallback used when // no per-credential batch fetcher is configured by `./lib/auth`. async function batchFetchDocumentSheets(context, documentId, sheetNames) { - const sheetOptions = {version: 'v4'}; - if (context.authClient) { - sheetOptions.auth = context.authClient; - } - - const sheets = google.sheets(sheetOptions); - const request = { - spreadsheetId: documentId, - ranges: sheetNames, - }; - - if (context.apiKey) { - request.key = context.apiKey; - } - - const response = await sheets.spreadsheets.values.batchGet(request); - const valueRanges = response.data.valueRanges || []; - const rowsBySheet = {}; - - for (let i = 0; i < sheetNames.length; i += 1) { - const sheetName = sheetNames[i]; - rowsBySheet[sheetName] = valueRanges[i]?.values || []; - } - - return rowsBySheet; + return fetchSpreadsheetValuesBatch(context, documentId, sheetNames); } function formatStatusTimestamp(date = new Date()) { @@ -331,6 +336,8 @@ async function scheduleRuns(config) { throw new Error('No sheets configured to pull in config.json'); } + console.log('Google Sheets Rate Assistant'); + const firstApiKey = getFirstApiKey(config); const titleMap = {}; @@ -393,7 +400,8 @@ async function scheduleRuns(config) { const totalRateLimit = getTotalRateLimitPerMinute(config); const intervalMs = Math.max(1, Math.ceil(60000 / totalRateLimit)); - console.log(`Configured for ${totalRateLimit} requests per minute across ${groups.length} documents (${intervalMs}ms interval)`); + console.log(limitToTerminalWidth(`Configured for ${totalRateLimit} requests per minute across ${groups.length} documents (${intervalMs}ms interval)`)); + console.log(''); initializeTaskStatuses(tasks); let currentIndex = 0; @@ -429,6 +437,8 @@ if (require.main === module) { (async () => { try { + clearConsoleForRun(); + let config; try { diff --git a/lib/auth.js b/lib/auth.js index 1c502b4..d7922ac 100644 --- a/lib/auth.js +++ b/lib/auth.js @@ -1,7 +1,28 @@ const fs = require('fs'); const path = require('path'); -const {google} = require('googleapis'); const {applyRateLimitBuffer} = require('./rate'); +const {fetchSpreadsheetValuesBatch} = require('./sheets'); + +function getConfigBaseDir(config) { + const configPath = config && (config.__configPath || config.configPath); + if (configPath) { + return path.dirname(configPath); + } + + return process.cwd(); +} + +function resolveConfigPath(config, candidatePath) { + if (!candidatePath) { + return ''; + } + + if (path.isAbsolute(candidatePath)) { + return candidatePath; + } + + return path.resolve(getConfigBaseDir(config), candidatePath); +} function loadApiKeys(config) { const apiKeys = []; @@ -60,7 +81,7 @@ function createServiceAccountBatchFetcher(config) { // selecting a credential whose slot is available now. If none are // immediately available, wait the minimum required time. const fetchers = accounts.map(({path: credentialsPath, rateLimitPerMinute}) => { - const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath); + const resolvedPath = resolveConfigPath(config, credentialsPath); if (!fs.existsSync(resolvedPath)) { throw new Error(`Service account credentials not found: ${resolvedPath}`); } @@ -84,16 +105,7 @@ function createServiceAccountBatchFetcher(config) { const interval = Math.ceil(60000 / bufferedRateLimit); const fetchRaw = async (documentId, sheetNames) => { - const sheets = google.sheets({version: 'v4', auth: authClient}); - const request = {spreadsheetId: documentId, ranges: sheetNames}; - const response = await sheets.spreadsheets.values.batchGet(request); - const valueRanges = response.data.valueRanges || []; - const rowsBySheet = {}; - for (let i = 0; i < sheetNames.length; i += 1) { - const sheetName = sheetNames[i]; - rowsBySheet[sheetName] = valueRanges[i]?.values || []; - } - return rowsBySheet; + return fetchSpreadsheetValuesBatch({authClient}, documentId, sheetNames); }; return {authClient, fetchRaw, interval, nextAvailable: 0}; @@ -148,16 +160,7 @@ function createApiKeyBatchFetcher(config) { const interval = Math.ceil(60000 / bufferedRateLimit); const fetchRaw = async (documentId, sheetNames) => { - const sheets = google.sheets({version: 'v4'}); - const request = {spreadsheetId: documentId, ranges: sheetNames, key}; - const response = await sheets.spreadsheets.values.batchGet(request); - const valueRanges = response.data.valueRanges || []; - const rowsBySheet = {}; - for (let i = 0; i < sheetNames.length; i += 1) { - const sheetName = sheetNames[i]; - rowsBySheet[sheetName] = valueRanges[i]?.values || []; - } - return rowsBySheet; + return fetchSpreadsheetValuesBatch({apiKey: key}, documentId, sheetNames); }; return {key, fetchRaw, interval, nextAvailable: 0}; @@ -215,7 +218,7 @@ function getFirstServiceAccountAuthClient(config) { const credentialsPath = entryObj.path || entryObj.credentialsPath; if (!credentialsPath) return null; - const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath); + const resolvedPath = resolveConfigPath(config, credentialsPath); if (!fs.existsSync(resolvedPath)) return null; try { @@ -235,7 +238,7 @@ function loadAuth(config) { if (config.credentialsPath) { const credentialsPath = config.credentialsPath; - const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath); + const resolvedPath = resolveConfigPath(config, credentialsPath); if (!fs.existsSync(resolvedPath)) { throw new Error(`Credentials file not found: ${resolvedPath}`); } diff --git a/lib/config.js b/lib/config.js index 3454996..c5b69d8 100644 --- a/lib/config.js +++ b/lib/config.js @@ -118,6 +118,12 @@ function loadConfig() { const json = fs.readFileSync(configPath, 'utf8'); const config = JSON.parse(json); + Object.defineProperty(config, '__configPath', { + value: configPath, + enumerable: false, + configurable: true, + writable: true, + }); validateConfig(config); return config; } diff --git a/lib/onboarding.js b/lib/onboarding.js index 41b48e4..86b6e93 100644 --- a/lib/onboarding.js +++ b/lib/onboarding.js @@ -1,7 +1,7 @@ const readline = require('readline'); -const {google} = require('googleapis'); const {createStarterConfig, getDefaultConfigPath, writeConfigFile} = require('./config'); const {getFirstServiceAccountAuthClient} = require('./auth'); +const {fetchSpreadsheetTitle} = require('./sheets'); function createPrompt() { const rl = readline.createInterface({input: process.stdin, output: process.stdout}); @@ -82,26 +82,6 @@ async function pauseForEnter(ask) { await ask('Press Enter to close this window...'); } -async function fetchSpreadsheetTitle(context, documentId) { - const sheetOptions = {version: 'v4'}; - if (context.authClient) { - sheetOptions.auth = context.authClient; - } - - const sheets = google.sheets(sheetOptions); - const request = { - spreadsheetId: documentId, - fields: 'properties/title', - }; - - if (context.apiKey) { - request.key = context.apiKey; - } - - const response = await sheets.spreadsheets.get(request); - return response.data.properties?.title || documentId; -} - function buildTitleContext(authMode, config) { if (authMode === 'apiKeys' && Array.isArray(config.apiKeys) && config.apiKeys.length > 0) { return {apiKey: config.apiKeys[0].key}; @@ -162,6 +142,12 @@ async function runOnboarding() { serviceAccounts: [], documents: [], }; + Object.defineProperty(config, '__configPath', { + value: configPath, + enumerable: false, + configurable: true, + writable: true, + }); if (authMode === 'apiKeys') { for (let index = 0; index < credentialCount; index += 1) { diff --git a/lib/sheets.js b/lib/sheets.js new file mode 100644 index 0000000..c9ec31e --- /dev/null +++ b/lib/sheets.js @@ -0,0 +1,149 @@ +const https = require('https'); +const {URL} = require('url'); + +function parseResponseBody(body) { + if (!body) { + return {}; + } + + try { + return JSON.parse(body); + } catch (_) { + return body; + } +} + +function extractErrorMessage(data) { + if (!data) { + return ''; + } + + if (typeof data === 'string') { + return data; + } + + if (data.error && typeof data.error === 'object') { + if (typeof data.error.message === 'string' && data.error.message) { + return data.error.message; + } + + if (Array.isArray(data.error.errors) && data.error.errors.length > 0) { + const firstError = data.error.errors[0]; + if (firstError && typeof firstError.message === 'string' && firstError.message) { + return firstError.message; + } + } + } + + if (typeof data.message === 'string' && data.message) { + return data.message; + } + + return ''; +} + +function requestJson(urlString, headers = {}) { + return new Promise((resolve, reject) => { + const url = new URL(urlString); + const request = https.request( + url, + { + method: 'GET', + headers: Object.assign({Accept: 'application/json'}, headers), + }, + response => { + let body = ''; + response.setEncoding('utf8'); + + response.on('data', chunk => { + body += chunk; + }); + + response.on('end', () => { + const data = parseResponseBody(body); + + if (response.statusCode >= 200 && response.statusCode < 300) { + resolve({status: response.statusCode, headers: response.headers, data}); + return; + } + + const message = extractErrorMessage(data) || `Request failed with status ${response.statusCode}`; + const error = new Error(message); + error.response = {status: response.statusCode, headers: response.headers, data}; + reject(error); + }); + } + ); + + request.on('error', reject); + request.end(); + }); +} + +async function requestSheetsJson(pathname, context, queryParams = {}) { + const url = new URL(`https://sheets.googleapis.com${pathname}`); + + for (const [key, value] of Object.entries(queryParams)) { + if (Array.isArray(value)) { + for (const item of value) { + url.searchParams.append(key, item); + } + } else if (value !== undefined && value !== null) { + url.searchParams.set(key, String(value)); + } + } + + const headers = {}; + + if (context && context.authClient) { + if (typeof context.authClient.getRequestHeaders === 'function') { + const authHeaders = await context.authClient.getRequestHeaders(url.toString()); + Object.assign(headers, authHeaders || {}); + } else if (typeof context.authClient.getAccessToken === 'function') { + const token = await context.authClient.getAccessToken(); + const accessToken = token && typeof token === 'object' ? token.token || token.access_token : token; + if (accessToken) { + headers.Authorization = `Bearer ${accessToken}`; + } + } + } + + if (context && context.apiKey) { + url.searchParams.set('key', context.apiKey); + } + + return requestJson(url.toString(), headers); +} + +async function fetchSpreadsheetTitle(context, documentId) { + const response = await requestSheetsJson( + `/v4/spreadsheets/${encodeURIComponent(documentId)}`, + context, + {fields: 'properties/title'} + ); + + return response.data && response.data.properties && response.data.properties.title || documentId; +} + +async function fetchSpreadsheetValuesBatch(context, documentId, sheetNames) { + const response = await requestSheetsJson( + `/v4/spreadsheets/${encodeURIComponent(documentId)}/values:batchGet`, + context, + {ranges: sheetNames} + ); + + const valueRanges = response.data.valueRanges || []; + const rowsBySheet = {}; + + for (let i = 0; i < sheetNames.length; i += 1) { + const sheetName = sheetNames[i]; + rowsBySheet[sheetName] = valueRanges[i] && valueRanges[i].values ? valueRanges[i].values : []; + } + + return rowsBySheet; +} + +module.exports = { + fetchSpreadsheetTitle, + fetchSpreadsheetValuesBatch, +}; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 05c737e..c76fcdd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "google-sheets-rate-assistant", - "version": "2.0.0", + "version": "2.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "google-sheets-rate-assistant", - "version": "2.0.0", + "version": "2.1.0", "dependencies": { "csv-stringify": "^6.0.0", "googleapis": "^173.0.0", diff --git a/package.json b/package.json index f04157a..7587678 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "google-sheets-rate-assistant", - "version": "2.1.0", + "version": "2.1.3", "description": "Pull Google Sheets data and export to CSV with configurable schedules and rate limiting.", "main": "index.js", "bin": "index.js",