This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
name: build-windows-exe
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*.*.*'
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
BUILD_VERSION: ${{ github.ref_name }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '24'
|
||||
cache: npm
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build Windows executable
|
||||
run: npm run build:win
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: gSheets-Rate-Assistant-${{ github.ref_name }}
|
||||
path: dist/gSheets-Rate-Assistant-${{ github.ref_name }}.exe
|
||||
@@ -6,21 +6,22 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const {google} = require('googleapis');
|
||||
const stringifyModule = require('csv-stringify/sync');
|
||||
const stringifyModule = require('./node_modules/csv-stringify/dist/cjs/sync.cjs');
|
||||
const stringify = typeof stringifyModule === 'function'
|
||||
? stringifyModule
|
||||
: stringifyModule.default || stringifyModule.stringify || stringifyModule;
|
||||
const {loadConfig, resolveOutputPath, validateConfig} = require('./lib/config');
|
||||
const {loadConfig, resolveOutputPath} = require('./lib/config');
|
||||
const {buildThrottle, applyRateLimitBuffer, getTotalRateLimitPerMinute} = require('./lib/rate');
|
||||
const {runOnboarding} = require('./lib/onboarding');
|
||||
const authHelpers = require('./lib/auth');
|
||||
const {createApiKeyBatchFetcher, createServiceAccountBatchFetcher, getFirstApiKey, getFirstServiceAccountAuthClient, loadAuth} = authHelpers;
|
||||
|
||||
// === Status rendering / console display ===
|
||||
// Renders per-sheet status lines when running in a TTY, updating in-place.
|
||||
|
||||
function mapDocumentToFilename(documentConfig, sheetName) {
|
||||
function mapDocumentToFilename(sheetName) {
|
||||
const safeName = sheetName.replace(/[\\/:*?"<>|]/g, '_');
|
||||
return `${documentConfig.documentId}-${safeName}.csv`;
|
||||
return `${safeName}.csv`;
|
||||
}
|
||||
|
||||
const statusManager = {
|
||||
@@ -109,7 +110,7 @@ function buildDocumentGroups(config) {
|
||||
|
||||
for (const sheetConfig of documentConfig.sheets) {
|
||||
const name = typeof sheetConfig === 'string' ? sheetConfig : sheetConfig.name;
|
||||
const outputFilename = sheetConfig.outputFilename || mapDocumentToFilename(documentConfig, name);
|
||||
const outputFilename = sheetConfig.outputFilename || mapDocumentToFilename(name);
|
||||
const outputPath = resolveOutputPath(documentOutputDir, outputFilename);
|
||||
|
||||
sheets.push({
|
||||
@@ -250,17 +251,65 @@ function sleep(milliseconds) {
|
||||
return new Promise(resolve => setTimeout(resolve, milliseconds));
|
||||
}
|
||||
|
||||
function waitForEnter() {
|
||||
if (!process.stdin || (!process.stdin.isTTY && !(process.stdout && process.stdout.isTTY))) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
return new Promise(resolve => {
|
||||
const readline = require('readline');
|
||||
const rl = readline.createInterface({input: process.stdin, output: process.stdout});
|
||||
rl.question('Press Enter to close this window...', () => {
|
||||
rl.close();
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function formatError(err) {
|
||||
if (!err) {
|
||||
return 'Unknown error';
|
||||
}
|
||||
|
||||
if (err.message) {
|
||||
return err.message;
|
||||
}
|
||||
|
||||
return String(err);
|
||||
}
|
||||
|
||||
function isUserFacingError(err) {
|
||||
if (!err) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const message = formatError(err);
|
||||
return /config\.json|api key|service account|spreadsheet|sheet|credential|document/i.test(message);
|
||||
}
|
||||
|
||||
async function handleFatalError(err, prefix = 'Application error:') {
|
||||
const message = formatError(err);
|
||||
const friendlyPrefix = prefix || 'Application error:';
|
||||
|
||||
console.error(friendlyPrefix);
|
||||
|
||||
if (isUserFacingError(err)) {
|
||||
console.error(message);
|
||||
} else {
|
||||
console.error('Something went wrong.');
|
||||
if (message && message !== 'Unknown error') {
|
||||
console.error(message);
|
||||
}
|
||||
}
|
||||
|
||||
await waitForEnter();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// === Main scheduler ===
|
||||
// Orchestrates startup, selects the appropriate fetcher (API keys,
|
||||
// service accounts), and runs the periodic loop.
|
||||
async function scheduleRuns() {
|
||||
const config = loadConfig();
|
||||
try {
|
||||
validateConfig(config);
|
||||
} catch (e) {
|
||||
console.error('Configuration error:', e.message);
|
||||
process.exit(1);
|
||||
}
|
||||
async function scheduleRuns(config) {
|
||||
const apiBatchFetcher = createApiKeyBatchFetcher(config);
|
||||
const svcBatchFetcher = createServiceAccountBatchFetcher(config);
|
||||
let fetcher;
|
||||
@@ -370,8 +419,35 @@ async function scheduleRuns() {
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
scheduleRuns().catch(err => {
|
||||
console.error('Application error:', err.message || err);
|
||||
process.exit(1);
|
||||
process.on('uncaughtException', err => {
|
||||
handleFatalError(err, 'Unexpected error:').catch(() => process.exit(1));
|
||||
});
|
||||
|
||||
process.on('unhandledRejection', err => {
|
||||
handleFatalError(err, 'Unexpected error:').catch(() => process.exit(1));
|
||||
});
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
let config;
|
||||
|
||||
try {
|
||||
config = loadConfig();
|
||||
} catch (err) {
|
||||
if (err && err.code === 'ERR_CONFIG_MISSING') {
|
||||
const onboardingResult = await runOnboarding();
|
||||
if (onboardingResult && onboardingResult.mode === 'config') {
|
||||
return;
|
||||
}
|
||||
config = onboardingResult.config;
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
await scheduleRuns(config);
|
||||
} catch (err) {
|
||||
handleFatalError(err, 'Configuration error:').catch(() => process.exit(1));
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
+208
-13
@@ -1,14 +1,125 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const CONFIG_PATH = path.resolve(__dirname, '..', 'config.json');
|
||||
const CONFIG_FILE_NAME = 'config.json';
|
||||
const DEFAULT_CONFIG_TEMPLATE = JSON.stringify({
|
||||
apiKeys: [
|
||||
{
|
||||
key: 'YOUR_API_KEY_1',
|
||||
rateLimitPerMinute: 50,
|
||||
},
|
||||
{
|
||||
key: 'YOUR_API_KEY_2',
|
||||
rateLimitPerMinute: 50,
|
||||
},
|
||||
],
|
||||
serviceAccounts: [
|
||||
{
|
||||
path: 'credentials.json',
|
||||
rateLimitPerMinute: 50,
|
||||
},
|
||||
{
|
||||
path: 'credentials_2.json',
|
||||
rateLimitPerMinute: 50,
|
||||
},
|
||||
],
|
||||
documents: [
|
||||
{
|
||||
documentId: 'YOUR_SPREADSHEET_ID',
|
||||
outputDir: 'output/your-document',
|
||||
sheets: [
|
||||
{
|
||||
name: 'Sheet1',
|
||||
outputFilename: 'sheet1.csv',
|
||||
},
|
||||
{
|
||||
name: 'Sheet2',
|
||||
outputFilename: 'sheet2.csv',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
documentId: 'YOUR_SPREADSHEET_ID_2',
|
||||
outputDir: 'output/your-document-2',
|
||||
sheets: [
|
||||
{
|
||||
name: 'Sheet1',
|
||||
outputFilename: 'sheet1.csv',
|
||||
},
|
||||
{
|
||||
name: 'Sheet2',
|
||||
outputFilename: 'sheet2.csv',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}, null, 2) + '\n';
|
||||
|
||||
function getDefaultConfigPath() {
|
||||
const packagedApp = Boolean(process.pkg);
|
||||
return path.resolve(packagedApp ? path.dirname(process.execPath) : process.cwd(), CONFIG_FILE_NAME);
|
||||
}
|
||||
|
||||
function getConfigPathCandidates() {
|
||||
const packagedApp = Boolean(process.pkg);
|
||||
const externalDefault = getDefaultConfigPath();
|
||||
|
||||
return [
|
||||
process.env.GSA_CONFIG,
|
||||
externalDefault,
|
||||
!packagedApp ? path.resolve(__dirname, '..', CONFIG_FILE_NAME) : null,
|
||||
].filter(Boolean);
|
||||
}
|
||||
|
||||
function createStarterConfig(configPath) {
|
||||
const directory = path.dirname(configPath);
|
||||
if (!fs.existsSync(directory)) {
|
||||
fs.mkdirSync(directory, {recursive: true});
|
||||
}
|
||||
|
||||
fs.writeFileSync(configPath, DEFAULT_CONFIG_TEMPLATE, 'utf8');
|
||||
}
|
||||
|
||||
function writeConfigFile(configPath, config) {
|
||||
const directory = path.dirname(configPath);
|
||||
if (!fs.existsSync(directory)) {
|
||||
fs.mkdirSync(directory, {recursive: true});
|
||||
}
|
||||
|
||||
fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n', 'utf8');
|
||||
}
|
||||
|
||||
function getApiKeyValue(item) {
|
||||
if (typeof item === 'string') {
|
||||
return item.trim();
|
||||
}
|
||||
|
||||
if (item && typeof item === 'object') {
|
||||
return (item.key || item.apiKey || '').toString().trim();
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
function isPlaceholderApiKey(key) {
|
||||
return key === 'YOUR_API_KEY_HERE';
|
||||
}
|
||||
|
||||
function loadConfig() {
|
||||
if (!fs.existsSync(CONFIG_PATH)) {
|
||||
throw new Error(`Missing config file at ${CONFIG_PATH}`);
|
||||
const configPath = getConfigPathCandidates().find(candidate => fs.existsSync(candidate));
|
||||
|
||||
if (!configPath) {
|
||||
const starterPath = process.env.GSA_CONFIG || getDefaultConfigPath();
|
||||
const error = new Error(`Missing config file at ${starterPath}`);
|
||||
error.code = 'ERR_CONFIG_MISSING';
|
||||
error.configPath = starterPath;
|
||||
throw error;
|
||||
}
|
||||
const json = fs.readFileSync(CONFIG_PATH, 'utf8');
|
||||
return JSON.parse(json);
|
||||
|
||||
const json = fs.readFileSync(configPath, 'utf8');
|
||||
const config = JSON.parse(json);
|
||||
validateConfig(config);
|
||||
return config;
|
||||
}
|
||||
|
||||
function resolveOutputPath(outputDir, filename) {
|
||||
@@ -26,6 +137,18 @@ function validateConfig(config) {
|
||||
throw new Error('Missing or invalid configuration object');
|
||||
}
|
||||
|
||||
if (config.apiKeys !== undefined && !Array.isArray(config.apiKeys)) {
|
||||
throw new Error('`apiKeys` must be an array when present in config.json');
|
||||
}
|
||||
|
||||
if (config.serviceAccounts !== undefined && !Array.isArray(config.serviceAccounts)) {
|
||||
throw new Error('`serviceAccounts` must be an array when present in config.json');
|
||||
}
|
||||
|
||||
if (config.documents !== undefined && !Array.isArray(config.documents)) {
|
||||
throw new Error('`documents` must be an array when present in config.json');
|
||||
}
|
||||
|
||||
if (!Array.isArray(config.documents) || config.documents.length === 0) {
|
||||
throw new Error('`documents` must be a non-empty array in config.json');
|
||||
}
|
||||
@@ -44,20 +167,58 @@ function validateConfig(config) {
|
||||
|
||||
// Ensure API key values are unique when using apiKeys
|
||||
if (hasApiKeys) {
|
||||
const extracted = config.apiKeys.map(item => {
|
||||
if (typeof item === 'string') return item.trim();
|
||||
if (item && typeof item === 'object') return (item.key || item.apiKey || '').toString().trim();
|
||||
return '';
|
||||
});
|
||||
const extracted = config.apiKeys.map(getApiKeyValue);
|
||||
const nonEmpty = extracted.filter(Boolean);
|
||||
const unique = new Set(nonEmpty);
|
||||
if (unique.size !== nonEmpty.length) {
|
||||
throw new Error('`apiKeys` contains duplicate `key` values; ensure all API keys are unique');
|
||||
}
|
||||
|
||||
for (const item of config.apiKeys) {
|
||||
const key = getApiKeyValue(item);
|
||||
|
||||
if (!key) {
|
||||
throw new Error('Each entry in `apiKeys` must include a non-empty `key`');
|
||||
}
|
||||
|
||||
if (isPlaceholderApiKey(key)) {
|
||||
throw new Error('Each entry in `apiKeys` must include a real API key; replace `YOUR_API_KEY_HERE`');
|
||||
}
|
||||
|
||||
if (typeof item !== 'string' && (!item || typeof item !== 'object')) {
|
||||
throw new Error('`apiKeys` entries must be strings or objects');
|
||||
}
|
||||
|
||||
if (item.rateLimitPerMinute !== undefined && (!Number.isFinite(item.rateLimitPerMinute) || item.rateLimitPerMinute <= 0)) {
|
||||
throw new Error('`rateLimitPerMinute` in `apiKeys` must be a positive number when provided');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure service account credential paths are unique when using serviceAccounts
|
||||
if (hasSvcAccounts) {
|
||||
for (const item of config.serviceAccounts) {
|
||||
if (typeof item === 'string') {
|
||||
if (!item.trim()) {
|
||||
throw new Error('Each string entry in `serviceAccounts` must be non-empty');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!item || typeof item !== 'object') {
|
||||
throw new Error('`serviceAccounts` entries must be strings or objects');
|
||||
}
|
||||
|
||||
const p = (item.path || item.credentialsPath || item.file || '').toString().trim();
|
||||
if (!p) {
|
||||
throw new Error('Each entry in `serviceAccounts` must include `path` or `credentialsPath`');
|
||||
}
|
||||
|
||||
if (item.rateLimitPerMinute !== undefined && (!Number.isFinite(item.rateLimitPerMinute) || item.rateLimitPerMinute <= 0)) {
|
||||
throw new Error('`rateLimitPerMinute` in `serviceAccounts` must be a positive number when provided');
|
||||
}
|
||||
}
|
||||
|
||||
// Read and canonicalize the contents of each service account credential
|
||||
const svcContents = config.serviceAccounts.map(item => {
|
||||
let p = '';
|
||||
@@ -92,9 +253,43 @@ function validateConfig(config) {
|
||||
|
||||
// Basic validation of documents/sheets structure
|
||||
for (const doc of config.documents) {
|
||||
if (!doc.documentId) throw new Error('Each document must include a `documentId`');
|
||||
if (!Array.isArray(doc.sheets) || doc.sheets.length === 0) throw new Error(`Document ${doc.documentId} must include a non-empty 'sheets' array`);
|
||||
if (!doc || typeof doc !== 'object') {
|
||||
throw new Error('Each document entry must be an object');
|
||||
}
|
||||
|
||||
if (typeof doc.documentId !== 'string' || !doc.documentId.trim()) {
|
||||
throw new Error('Each document must include a non-empty `documentId`');
|
||||
}
|
||||
|
||||
if (doc.outputDir !== undefined && typeof doc.outputDir !== 'string') {
|
||||
throw new Error(`Document ${doc.documentId} has an invalid outputDir; it must be a string`);
|
||||
}
|
||||
|
||||
if (!Array.isArray(doc.sheets) || doc.sheets.length === 0) {
|
||||
throw new Error(`Document ${doc.documentId} must include a non-empty 'sheets' array`);
|
||||
}
|
||||
|
||||
for (const sheet of doc.sheets) {
|
||||
if (typeof sheet === 'string') {
|
||||
if (!sheet.trim()) {
|
||||
throw new Error(`Document ${doc.documentId} contains an empty sheet name`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!sheet || typeof sheet !== 'object') {
|
||||
throw new Error(`Document ${doc.documentId} contains an invalid sheet entry`);
|
||||
}
|
||||
|
||||
if (typeof sheet.name !== 'string' || !sheet.name.trim()) {
|
||||
throw new Error(`Document ${doc.documentId} contains a sheet entry missing name`);
|
||||
}
|
||||
|
||||
if (sheet.outputFilename !== undefined && typeof sheet.outputFilename !== 'string') {
|
||||
throw new Error(`Document ${doc.documentId} sheet ${sheet.name} has an invalid outputFilename; it must be a string`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {loadConfig, resolveOutputPath, validateConfig};
|
||||
module.exports = {loadConfig, resolveOutputPath, validateConfig, createStarterConfig, getDefaultConfigPath, writeConfigFile};
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
const readline = require('readline');
|
||||
const {google} = require('googleapis');
|
||||
const {createStarterConfig, getDefaultConfigPath, writeConfigFile} = require('./config');
|
||||
const {getFirstServiceAccountAuthClient} = require('./auth');
|
||||
|
||||
function createPrompt() {
|
||||
const rl = readline.createInterface({input: process.stdin, output: process.stdout});
|
||||
|
||||
const ask = question => new Promise(resolve => {
|
||||
rl.question(question, answer => resolve((answer || '').trim()));
|
||||
});
|
||||
|
||||
const close = () => new Promise(resolve => {
|
||||
rl.close();
|
||||
resolve();
|
||||
});
|
||||
|
||||
return {ask, close};
|
||||
}
|
||||
|
||||
async function askNonEmpty(ask, question) {
|
||||
while (true) {
|
||||
const answer = await ask(question);
|
||||
if (answer) {
|
||||
return answer;
|
||||
}
|
||||
console.log('Please enter a value.');
|
||||
}
|
||||
}
|
||||
|
||||
async function askPositiveInteger(ask, question) {
|
||||
while (true) {
|
||||
const answer = await ask(question);
|
||||
const value = Number.parseInt(answer, 10);
|
||||
if (Number.isInteger(value) && value > 0) {
|
||||
return value;
|
||||
}
|
||||
console.log('Please enter a whole number greater than zero.');
|
||||
}
|
||||
}
|
||||
|
||||
async function askChoice(ask, question, options) {
|
||||
const optionText = options
|
||||
.map((option, index) => `${index + 1}. ${option.label}`)
|
||||
.join('\n');
|
||||
|
||||
while (true) {
|
||||
const answer = await ask(`${question}\n${optionText}\n> `);
|
||||
const lower = answer.toLowerCase();
|
||||
|
||||
const byNumber = Number.parseInt(answer, 10);
|
||||
if (Number.isInteger(byNumber) && byNumber >= 1 && byNumber <= options.length) {
|
||||
return options[byNumber - 1].value;
|
||||
}
|
||||
|
||||
const match = options.find(option => option.value.toLowerCase() === lower || option.label.toLowerCase() === lower);
|
||||
if (match) {
|
||||
return match.value;
|
||||
}
|
||||
|
||||
console.log('Please choose one of the listed options.');
|
||||
}
|
||||
}
|
||||
|
||||
async function askOptionalPositiveInteger(ask, question, defaultValue) {
|
||||
while (true) {
|
||||
const answer = await ask(question);
|
||||
if (!answer) {
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
const value = Number.parseInt(answer, 10);
|
||||
if (Number.isInteger(value) && value > 0) {
|
||||
return value;
|
||||
}
|
||||
|
||||
console.log('Please enter a whole number greater than zero, or press Enter to use the default.');
|
||||
}
|
||||
}
|
||||
|
||||
async function pauseForEnter(ask) {
|
||||
await ask('Press Enter to close this window...');
|
||||
}
|
||||
|
||||
async function fetchSpreadsheetTitle(context, documentId) {
|
||||
const sheetOptions = {version: 'v4'};
|
||||
if (context.authClient) {
|
||||
sheetOptions.auth = context.authClient;
|
||||
}
|
||||
|
||||
const sheets = google.sheets(sheetOptions);
|
||||
const request = {
|
||||
spreadsheetId: documentId,
|
||||
fields: 'properties/title',
|
||||
};
|
||||
|
||||
if (context.apiKey) {
|
||||
request.key = context.apiKey;
|
||||
}
|
||||
|
||||
const response = await sheets.spreadsheets.get(request);
|
||||
return response.data.properties?.title || documentId;
|
||||
}
|
||||
|
||||
function buildTitleContext(authMode, config) {
|
||||
if (authMode === 'apiKeys' && Array.isArray(config.apiKeys) && config.apiKeys.length > 0) {
|
||||
return {apiKey: config.apiKeys[0].key};
|
||||
}
|
||||
|
||||
if (authMode === 'serviceAccounts' && Array.isArray(config.serviceAccounts) && config.serviceAccounts.length > 0) {
|
||||
const authClient = getFirstServiceAccountAuthClient(config);
|
||||
if (authClient) {
|
||||
return {authClient};
|
||||
}
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
async function runOnboarding() {
|
||||
const {ask, close} = createPrompt();
|
||||
const configPath = getDefaultConfigPath();
|
||||
|
||||
try {
|
||||
console.log('Google Sheets Rate Assistant setup');
|
||||
console.log('');
|
||||
|
||||
const setupMode = await askChoice(
|
||||
ask,
|
||||
'Would you like to use the wizard or just create a config file?',
|
||||
[
|
||||
{label: 'Wizard', value: 'wizard'},
|
||||
{label: 'Config file', value: 'config'},
|
||||
]
|
||||
);
|
||||
|
||||
if (setupMode === 'config') {
|
||||
createStarterConfig(configPath);
|
||||
console.log('');
|
||||
console.log(`Starter config created at ${configPath}`);
|
||||
console.log('Edit that file, then run the app again.');
|
||||
await pauseForEnter(ask);
|
||||
return {mode: 'config', configPath};
|
||||
}
|
||||
|
||||
const authMode = await askChoice(
|
||||
ask,
|
||||
'Which authentication method do you want to use?',
|
||||
[
|
||||
{label: 'API keys', value: 'apiKeys'},
|
||||
{label: 'Service accounts', value: 'serviceAccounts'},
|
||||
]
|
||||
);
|
||||
|
||||
const credentialCount = await askPositiveInteger(
|
||||
ask,
|
||||
`How many ${authMode === 'apiKeys' ? 'API keys' : 'service accounts'} do you want to add? `
|
||||
);
|
||||
|
||||
const config = {
|
||||
apiKeys: [],
|
||||
serviceAccounts: [],
|
||||
documents: [],
|
||||
};
|
||||
|
||||
if (authMode === 'apiKeys') {
|
||||
for (let index = 0; index < credentialCount; index += 1) {
|
||||
const key = await askNonEmpty(ask, `Enter API key ${index + 1}: `);
|
||||
const rateLimitPerMinute = await askOptionalPositiveInteger(
|
||||
ask,
|
||||
`Rate limit per minute for API key ${index + 1} (press Enter for 50): `,
|
||||
50
|
||||
);
|
||||
|
||||
config.apiKeys.push({key, rateLimitPerMinute});
|
||||
}
|
||||
} else {
|
||||
for (let index = 0; index < credentialCount; index += 1) {
|
||||
const pathValue = await askNonEmpty(ask, `Enter service account path ${index + 1}: `);
|
||||
const rateLimitPerMinute = await askOptionalPositiveInteger(
|
||||
ask,
|
||||
`Rate limit per minute for service account ${index + 1} (press Enter for 50): `,
|
||||
50
|
||||
);
|
||||
|
||||
config.serviceAccounts.push({path: pathValue, rateLimitPerMinute});
|
||||
}
|
||||
}
|
||||
|
||||
const documentCount = await askPositiveInteger(ask, 'How many documents do you want to pull from? ');
|
||||
|
||||
for (let documentIndex = 0; documentIndex < documentCount; documentIndex += 1) {
|
||||
const documentId = await askNonEmpty(ask, `Enter document ID ${documentIndex + 1}: `);
|
||||
let documentTitle = documentId;
|
||||
|
||||
try {
|
||||
const titleContext = buildTitleContext(authMode, config);
|
||||
documentTitle = await fetchSpreadsheetTitle(titleContext, documentId);
|
||||
} catch (err) {
|
||||
console.log(`Could not load spreadsheet title for ${documentId}; using the document ID instead.`);
|
||||
}
|
||||
|
||||
const sheetCount = await askPositiveInteger(ask, `How many sheets for document ${documentIndex + 1}? `);
|
||||
const sheets = [];
|
||||
|
||||
for (let sheetIndex = 0; sheetIndex < sheetCount; sheetIndex += 1) {
|
||||
const sheetName = await askNonEmpty(ask, `Enter sheet name ${sheetIndex + 1} for document ${documentIndex + 1}: `);
|
||||
sheets.push({
|
||||
name: sheetName,
|
||||
outputFilename: `${sheetName}.csv`,
|
||||
});
|
||||
}
|
||||
|
||||
config.documents.push({
|
||||
documentId,
|
||||
outputDir: `output/${documentTitle}`,
|
||||
sheets,
|
||||
});
|
||||
}
|
||||
|
||||
writeConfigFile(configPath, config);
|
||||
console.log('');
|
||||
console.log(`Config saved to ${configPath}`);
|
||||
return {mode: 'wizard', configPath, config};
|
||||
} finally {
|
||||
await close();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {runOnboarding};
|
||||
+18
-3
@@ -1,10 +1,25 @@
|
||||
const pThrottleModule = require('p-throttle');
|
||||
const pThrottle = pThrottleModule.default || pThrottleModule;
|
||||
function sleep(milliseconds) {
|
||||
return new Promise(resolve => setTimeout(resolve, milliseconds));
|
||||
}
|
||||
|
||||
function buildThrottle(maxRequestsPerMinute) {
|
||||
const limit = Math.max(1, Math.floor(maxRequestsPerMinute));
|
||||
const interval = Math.ceil(60000 / limit);
|
||||
return pThrottle({limit: 1, interval});
|
||||
let nextAvailable = 0;
|
||||
|
||||
return function throttle(fn) {
|
||||
return async function throttledFunction(...args) {
|
||||
const now = Date.now();
|
||||
const waitMs = Math.max(0, nextAvailable - now);
|
||||
nextAvailable = Math.max(nextAvailable, now) + interval;
|
||||
|
||||
if (waitMs > 0) {
|
||||
await sleep(waitMs);
|
||||
}
|
||||
|
||||
return fn(...args);
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
function applyRateLimitBuffer(rateLimitPerMinute) {
|
||||
|
||||
Generated
+1555
-2
File diff suppressed because it is too large
Load Diff
+12
-2
@@ -1,14 +1,24 @@
|
||||
{
|
||||
"name": "google-sheets-rate-assistant",
|
||||
"version": "2.0.0",
|
||||
"version": "2.1.0",
|
||||
"description": "Pull Google Sheets data and export to CSV with configurable schedules and rate limiting.",
|
||||
"main": "index.js",
|
||||
"bin": "index.js",
|
||||
"scripts": {
|
||||
"start": "node index.js"
|
||||
"start": "node index.js",
|
||||
"build:win": "node scripts/build-win.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"csv-stringify": "^6.0.0",
|
||||
"googleapis": "^173.0.0",
|
||||
"p-throttle": "^8.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"pkg": "^5.8.1"
|
||||
},
|
||||
"pkg": {
|
||||
"assets": [
|
||||
"node_modules/csv-stringify/dist/cjs/sync.cjs"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,20 +4,13 @@ This Node.js application pulls configured Google Sheets documents and exports th
|
||||
|
||||
## Setup
|
||||
|
||||
1. Copy `config.example.json` to `config.json`.
|
||||
2. Configure authentication in `config.json` — choose one of:
|
||||
1. Download and run the latest version.
|
||||
2. If `config.json` does not exist, the app will ask whether you want a wizard or a starter config file.
|
||||
3. If you choose the config-file path instead of the wizard, configure authentication in `config.json` — choose one of:
|
||||
|
||||
- Service account: place your service account JSON in `credentials.json` and set `credentialsPath`, or add one or more entries to the `serviceAccounts` array (each entry may be a path string or an object with `path` and optional `rateLimitPerMinute`).
|
||||
- API keys: add one or more API keys to the `apiKeys` array in `config.json` (each entry may be a string or an object with `key` and optional `rateLimitPerMinute`).
|
||||
|
||||
3. Install dependencies:
|
||||
|
||||
npm install
|
||||
|
||||
4. Run the application:
|
||||
|
||||
node index.js
|
||||
|
||||
## Setup Guides
|
||||
|
||||
For detailed instructions on authentication methods, please refer to the following guides:
|
||||
@@ -63,7 +56,23 @@ There is a 5% buffer applied to any limitation to cater for any network fluctuat
|
||||
|
||||
## Output
|
||||
|
||||
CSV files are written to `output` by default. Individual documents may override this with their own `outputDir` in `config.json`.
|
||||
CSV files are written to `output` by default. The onboarding wizard uses that output directory automatically and names each file after the sheet.
|
||||
|
||||
If `config.json` is missing, the app creates a starter file next to the executable when packaged, or in the current working directory when running from source, and exits so you can fill it in before the next run.
|
||||
|
||||
If you choose the wizard, it will ask for the auth method, credential count, document count, and sheet names. It will not ask for output directories or file names.
|
||||
|
||||
## Windows Packaging
|
||||
|
||||
To build a Windows executable from source, run:
|
||||
|
||||
```bash
|
||||
npm run build:win
|
||||
```
|
||||
|
||||
That creates `dist/gSheets-Rate-Assistant-v2.0.0.exe` using the current version in the filename. For distribution, ship the exe alongside a `config.json` and any service account credential files, or set `GSA_CONFIG` to point at a different config file.
|
||||
|
||||
Gitea now builds only on version tags like `v1.1.1` through the workflow in `.gitea/workflows/build.yml`.
|
||||
|
||||
## AI Disclosure
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const {exec} = require('pkg');
|
||||
const {version: packageVersion} = require('../package.json');
|
||||
|
||||
function normalizeBuildVersion(value) {
|
||||
const version = (value || packageVersion || '0.0.0').toString().trim();
|
||||
return version.startsWith('v') ? version : `v${version}`;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const distDir = path.resolve(__dirname, '..', 'dist');
|
||||
if (!fs.existsSync(distDir)) {
|
||||
fs.mkdirSync(distDir, {recursive: true});
|
||||
}
|
||||
|
||||
const buildVersion = normalizeBuildVersion(process.env.BUILD_VERSION);
|
||||
const outputFile = path.join(distDir, `gSheets-Rate-Assistant-${buildVersion}.exe`);
|
||||
await exec(['.', '--targets', 'latest-win-x64', '--output', outputFile]);
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
console.error(error && error.stack ? error.stack : error);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user