Added onboarding and exe output.
This commit is contained in:
@@ -0,0 +1,33 @@
|
|||||||
|
name: build-windows-exe
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- 'v*.*.*'
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
BUILD_VERSION: ${{ github.ref_name }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Setup Node.js
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: '24'
|
||||||
|
cache: npm
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: npm ci
|
||||||
|
|
||||||
|
- name: Build Windows executable
|
||||||
|
run: npm run build:win
|
||||||
|
|
||||||
|
- name: Upload artifact
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: gSheets-Rate-Assistant-${{ github.ref_name }}
|
||||||
|
path: dist/gSheets-Rate-Assistant-${{ github.ref_name }}.exe
|
||||||
@@ -6,21 +6,22 @@
|
|||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
const {google} = require('googleapis');
|
const {google} = require('googleapis');
|
||||||
const stringifyModule = require('csv-stringify/sync');
|
const stringifyModule = require('./node_modules/csv-stringify/dist/cjs/sync.cjs');
|
||||||
const stringify = typeof stringifyModule === 'function'
|
const stringify = typeof stringifyModule === 'function'
|
||||||
? stringifyModule
|
? stringifyModule
|
||||||
: stringifyModule.default || stringifyModule.stringify || stringifyModule;
|
: stringifyModule.default || stringifyModule.stringify || stringifyModule;
|
||||||
const {loadConfig, resolveOutputPath, validateConfig} = require('./lib/config');
|
const {loadConfig, resolveOutputPath} = require('./lib/config');
|
||||||
const {buildThrottle, applyRateLimitBuffer, getTotalRateLimitPerMinute} = require('./lib/rate');
|
const {buildThrottle, applyRateLimitBuffer, getTotalRateLimitPerMinute} = require('./lib/rate');
|
||||||
|
const {runOnboarding} = require('./lib/onboarding');
|
||||||
const authHelpers = require('./lib/auth');
|
const authHelpers = require('./lib/auth');
|
||||||
const {createApiKeyBatchFetcher, createServiceAccountBatchFetcher, getFirstApiKey, getFirstServiceAccountAuthClient, loadAuth} = authHelpers;
|
const {createApiKeyBatchFetcher, createServiceAccountBatchFetcher, getFirstApiKey, getFirstServiceAccountAuthClient, loadAuth} = authHelpers;
|
||||||
|
|
||||||
// === Status rendering / console display ===
|
// === Status rendering / console display ===
|
||||||
// Renders per-sheet status lines when running in a TTY, updating in-place.
|
// Renders per-sheet status lines when running in a TTY, updating in-place.
|
||||||
|
|
||||||
function mapDocumentToFilename(documentConfig, sheetName) {
|
function mapDocumentToFilename(sheetName) {
|
||||||
const safeName = sheetName.replace(/[\\/:*?"<>|]/g, '_');
|
const safeName = sheetName.replace(/[\\/:*?"<>|]/g, '_');
|
||||||
return `${documentConfig.documentId}-${safeName}.csv`;
|
return `${safeName}.csv`;
|
||||||
}
|
}
|
||||||
|
|
||||||
const statusManager = {
|
const statusManager = {
|
||||||
@@ -109,7 +110,7 @@ function buildDocumentGroups(config) {
|
|||||||
|
|
||||||
for (const sheetConfig of documentConfig.sheets) {
|
for (const sheetConfig of documentConfig.sheets) {
|
||||||
const name = typeof sheetConfig === 'string' ? sheetConfig : sheetConfig.name;
|
const name = typeof sheetConfig === 'string' ? sheetConfig : sheetConfig.name;
|
||||||
const outputFilename = sheetConfig.outputFilename || mapDocumentToFilename(documentConfig, name);
|
const outputFilename = sheetConfig.outputFilename || mapDocumentToFilename(name);
|
||||||
const outputPath = resolveOutputPath(documentOutputDir, outputFilename);
|
const outputPath = resolveOutputPath(documentOutputDir, outputFilename);
|
||||||
|
|
||||||
sheets.push({
|
sheets.push({
|
||||||
@@ -250,17 +251,65 @@ function sleep(milliseconds) {
|
|||||||
return new Promise(resolve => setTimeout(resolve, milliseconds));
|
return new Promise(resolve => setTimeout(resolve, milliseconds));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function waitForEnter() {
|
||||||
|
if (!process.stdin || (!process.stdin.isTTY && !(process.stdout && process.stdout.isTTY))) {
|
||||||
|
return Promise.resolve();
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Promise(resolve => {
|
||||||
|
const readline = require('readline');
|
||||||
|
const rl = readline.createInterface({input: process.stdin, output: process.stdout});
|
||||||
|
rl.question('Press Enter to close this window...', () => {
|
||||||
|
rl.close();
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatError(err) {
|
||||||
|
if (!err) {
|
||||||
|
return 'Unknown error';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (err.message) {
|
||||||
|
return err.message;
|
||||||
|
}
|
||||||
|
|
||||||
|
return String(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isUserFacingError(err) {
|
||||||
|
if (!err) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const message = formatError(err);
|
||||||
|
return /config\.json|api key|service account|spreadsheet|sheet|credential|document/i.test(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleFatalError(err, prefix = 'Application error:') {
|
||||||
|
const message = formatError(err);
|
||||||
|
const friendlyPrefix = prefix || 'Application error:';
|
||||||
|
|
||||||
|
console.error(friendlyPrefix);
|
||||||
|
|
||||||
|
if (isUserFacingError(err)) {
|
||||||
|
console.error(message);
|
||||||
|
} else {
|
||||||
|
console.error('Something went wrong.');
|
||||||
|
if (message && message !== 'Unknown error') {
|
||||||
|
console.error(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await waitForEnter();
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
// === Main scheduler ===
|
// === Main scheduler ===
|
||||||
// Orchestrates startup, selects the appropriate fetcher (API keys,
|
// Orchestrates startup, selects the appropriate fetcher (API keys,
|
||||||
// service accounts), and runs the periodic loop.
|
// service accounts), and runs the periodic loop.
|
||||||
async function scheduleRuns() {
|
async function scheduleRuns(config) {
|
||||||
const config = loadConfig();
|
|
||||||
try {
|
|
||||||
validateConfig(config);
|
|
||||||
} catch (e) {
|
|
||||||
console.error('Configuration error:', e.message);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
const apiBatchFetcher = createApiKeyBatchFetcher(config);
|
const apiBatchFetcher = createApiKeyBatchFetcher(config);
|
||||||
const svcBatchFetcher = createServiceAccountBatchFetcher(config);
|
const svcBatchFetcher = createServiceAccountBatchFetcher(config);
|
||||||
let fetcher;
|
let fetcher;
|
||||||
@@ -370,8 +419,35 @@ async function scheduleRuns() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (require.main === module) {
|
if (require.main === module) {
|
||||||
scheduleRuns().catch(err => {
|
process.on('uncaughtException', err => {
|
||||||
console.error('Application error:', err.message || err);
|
handleFatalError(err, 'Unexpected error:').catch(() => process.exit(1));
|
||||||
process.exit(1);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
process.on('unhandledRejection', err => {
|
||||||
|
handleFatalError(err, 'Unexpected error:').catch(() => process.exit(1));
|
||||||
|
});
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
try {
|
||||||
|
let config;
|
||||||
|
|
||||||
|
try {
|
||||||
|
config = loadConfig();
|
||||||
|
} catch (err) {
|
||||||
|
if (err && err.code === 'ERR_CONFIG_MISSING') {
|
||||||
|
const onboardingResult = await runOnboarding();
|
||||||
|
if (onboardingResult && onboardingResult.mode === 'config') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
config = onboardingResult.config;
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await scheduleRuns(config);
|
||||||
|
} catch (err) {
|
||||||
|
handleFatalError(err, 'Configuration error:').catch(() => process.exit(1));
|
||||||
|
}
|
||||||
|
})();
|
||||||
}
|
}
|
||||||
|
|||||||
+208
-13
@@ -1,14 +1,125 @@
|
|||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
|
|
||||||
const CONFIG_PATH = path.resolve(__dirname, '..', 'config.json');
|
const CONFIG_FILE_NAME = 'config.json';
|
||||||
|
const DEFAULT_CONFIG_TEMPLATE = JSON.stringify({
|
||||||
|
apiKeys: [
|
||||||
|
{
|
||||||
|
key: 'YOUR_API_KEY_1',
|
||||||
|
rateLimitPerMinute: 50,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'YOUR_API_KEY_2',
|
||||||
|
rateLimitPerMinute: 50,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
serviceAccounts: [
|
||||||
|
{
|
||||||
|
path: 'credentials.json',
|
||||||
|
rateLimitPerMinute: 50,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
path: 'credentials_2.json',
|
||||||
|
rateLimitPerMinute: 50,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
documentId: 'YOUR_SPREADSHEET_ID',
|
||||||
|
outputDir: 'output/your-document',
|
||||||
|
sheets: [
|
||||||
|
{
|
||||||
|
name: 'Sheet1',
|
||||||
|
outputFilename: 'sheet1.csv',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'Sheet2',
|
||||||
|
outputFilename: 'sheet2.csv',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
documentId: 'YOUR_SPREADSHEET_ID_2',
|
||||||
|
outputDir: 'output/your-document-2',
|
||||||
|
sheets: [
|
||||||
|
{
|
||||||
|
name: 'Sheet1',
|
||||||
|
outputFilename: 'sheet1.csv',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'Sheet2',
|
||||||
|
outputFilename: 'sheet2.csv',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}, null, 2) + '\n';
|
||||||
|
|
||||||
|
function getDefaultConfigPath() {
|
||||||
|
const packagedApp = Boolean(process.pkg);
|
||||||
|
return path.resolve(packagedApp ? path.dirname(process.execPath) : process.cwd(), CONFIG_FILE_NAME);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getConfigPathCandidates() {
|
||||||
|
const packagedApp = Boolean(process.pkg);
|
||||||
|
const externalDefault = getDefaultConfigPath();
|
||||||
|
|
||||||
|
return [
|
||||||
|
process.env.GSA_CONFIG,
|
||||||
|
externalDefault,
|
||||||
|
!packagedApp ? path.resolve(__dirname, '..', CONFIG_FILE_NAME) : null,
|
||||||
|
].filter(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
|
function createStarterConfig(configPath) {
|
||||||
|
const directory = path.dirname(configPath);
|
||||||
|
if (!fs.existsSync(directory)) {
|
||||||
|
fs.mkdirSync(directory, {recursive: true});
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.writeFileSync(configPath, DEFAULT_CONFIG_TEMPLATE, 'utf8');
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeConfigFile(configPath, config) {
|
||||||
|
const directory = path.dirname(configPath);
|
||||||
|
if (!fs.existsSync(directory)) {
|
||||||
|
fs.mkdirSync(directory, {recursive: true});
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n', 'utf8');
|
||||||
|
}
|
||||||
|
|
||||||
|
function getApiKeyValue(item) {
|
||||||
|
if (typeof item === 'string') {
|
||||||
|
return item.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (item && typeof item === 'object') {
|
||||||
|
return (item.key || item.apiKey || '').toString().trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPlaceholderApiKey(key) {
|
||||||
|
return key === 'YOUR_API_KEY_HERE';
|
||||||
|
}
|
||||||
|
|
||||||
function loadConfig() {
|
function loadConfig() {
|
||||||
if (!fs.existsSync(CONFIG_PATH)) {
|
const configPath = getConfigPathCandidates().find(candidate => fs.existsSync(candidate));
|
||||||
throw new Error(`Missing config file at ${CONFIG_PATH}`);
|
|
||||||
|
if (!configPath) {
|
||||||
|
const starterPath = process.env.GSA_CONFIG || getDefaultConfigPath();
|
||||||
|
const error = new Error(`Missing config file at ${starterPath}`);
|
||||||
|
error.code = 'ERR_CONFIG_MISSING';
|
||||||
|
error.configPath = starterPath;
|
||||||
|
throw error;
|
||||||
}
|
}
|
||||||
const json = fs.readFileSync(CONFIG_PATH, 'utf8');
|
|
||||||
return JSON.parse(json);
|
const json = fs.readFileSync(configPath, 'utf8');
|
||||||
|
const config = JSON.parse(json);
|
||||||
|
validateConfig(config);
|
||||||
|
return config;
|
||||||
}
|
}
|
||||||
|
|
||||||
function resolveOutputPath(outputDir, filename) {
|
function resolveOutputPath(outputDir, filename) {
|
||||||
@@ -26,6 +137,18 @@ function validateConfig(config) {
|
|||||||
throw new Error('Missing or invalid configuration object');
|
throw new Error('Missing or invalid configuration object');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (config.apiKeys !== undefined && !Array.isArray(config.apiKeys)) {
|
||||||
|
throw new Error('`apiKeys` must be an array when present in config.json');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config.serviceAccounts !== undefined && !Array.isArray(config.serviceAccounts)) {
|
||||||
|
throw new Error('`serviceAccounts` must be an array when present in config.json');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config.documents !== undefined && !Array.isArray(config.documents)) {
|
||||||
|
throw new Error('`documents` must be an array when present in config.json');
|
||||||
|
}
|
||||||
|
|
||||||
if (!Array.isArray(config.documents) || config.documents.length === 0) {
|
if (!Array.isArray(config.documents) || config.documents.length === 0) {
|
||||||
throw new Error('`documents` must be a non-empty array in config.json');
|
throw new Error('`documents` must be a non-empty array in config.json');
|
||||||
}
|
}
|
||||||
@@ -44,20 +167,58 @@ function validateConfig(config) {
|
|||||||
|
|
||||||
// Ensure API key values are unique when using apiKeys
|
// Ensure API key values are unique when using apiKeys
|
||||||
if (hasApiKeys) {
|
if (hasApiKeys) {
|
||||||
const extracted = config.apiKeys.map(item => {
|
const extracted = config.apiKeys.map(getApiKeyValue);
|
||||||
if (typeof item === 'string') return item.trim();
|
|
||||||
if (item && typeof item === 'object') return (item.key || item.apiKey || '').toString().trim();
|
|
||||||
return '';
|
|
||||||
});
|
|
||||||
const nonEmpty = extracted.filter(Boolean);
|
const nonEmpty = extracted.filter(Boolean);
|
||||||
const unique = new Set(nonEmpty);
|
const unique = new Set(nonEmpty);
|
||||||
if (unique.size !== nonEmpty.length) {
|
if (unique.size !== nonEmpty.length) {
|
||||||
throw new Error('`apiKeys` contains duplicate `key` values; ensure all API keys are unique');
|
throw new Error('`apiKeys` contains duplicate `key` values; ensure all API keys are unique');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for (const item of config.apiKeys) {
|
||||||
|
const key = getApiKeyValue(item);
|
||||||
|
|
||||||
|
if (!key) {
|
||||||
|
throw new Error('Each entry in `apiKeys` must include a non-empty `key`');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isPlaceholderApiKey(key)) {
|
||||||
|
throw new Error('Each entry in `apiKeys` must include a real API key; replace `YOUR_API_KEY_HERE`');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof item !== 'string' && (!item || typeof item !== 'object')) {
|
||||||
|
throw new Error('`apiKeys` entries must be strings or objects');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (item.rateLimitPerMinute !== undefined && (!Number.isFinite(item.rateLimitPerMinute) || item.rateLimitPerMinute <= 0)) {
|
||||||
|
throw new Error('`rateLimitPerMinute` in `apiKeys` must be a positive number when provided');
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure service account credential paths are unique when using serviceAccounts
|
// Ensure service account credential paths are unique when using serviceAccounts
|
||||||
if (hasSvcAccounts) {
|
if (hasSvcAccounts) {
|
||||||
|
for (const item of config.serviceAccounts) {
|
||||||
|
if (typeof item === 'string') {
|
||||||
|
if (!item.trim()) {
|
||||||
|
throw new Error('Each string entry in `serviceAccounts` must be non-empty');
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!item || typeof item !== 'object') {
|
||||||
|
throw new Error('`serviceAccounts` entries must be strings or objects');
|
||||||
|
}
|
||||||
|
|
||||||
|
const p = (item.path || item.credentialsPath || item.file || '').toString().trim();
|
||||||
|
if (!p) {
|
||||||
|
throw new Error('Each entry in `serviceAccounts` must include `path` or `credentialsPath`');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (item.rateLimitPerMinute !== undefined && (!Number.isFinite(item.rateLimitPerMinute) || item.rateLimitPerMinute <= 0)) {
|
||||||
|
throw new Error('`rateLimitPerMinute` in `serviceAccounts` must be a positive number when provided');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Read and canonicalize the contents of each service account credential
|
// Read and canonicalize the contents of each service account credential
|
||||||
const svcContents = config.serviceAccounts.map(item => {
|
const svcContents = config.serviceAccounts.map(item => {
|
||||||
let p = '';
|
let p = '';
|
||||||
@@ -92,9 +253,43 @@ function validateConfig(config) {
|
|||||||
|
|
||||||
// Basic validation of documents/sheets structure
|
// Basic validation of documents/sheets structure
|
||||||
for (const doc of config.documents) {
|
for (const doc of config.documents) {
|
||||||
if (!doc.documentId) throw new Error('Each document must include a `documentId`');
|
if (!doc || typeof doc !== 'object') {
|
||||||
if (!Array.isArray(doc.sheets) || doc.sheets.length === 0) throw new Error(`Document ${doc.documentId} must include a non-empty 'sheets' array`);
|
throw new Error('Each document entry must be an object');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof doc.documentId !== 'string' || !doc.documentId.trim()) {
|
||||||
|
throw new Error('Each document must include a non-empty `documentId`');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (doc.outputDir !== undefined && typeof doc.outputDir !== 'string') {
|
||||||
|
throw new Error(`Document ${doc.documentId} has an invalid outputDir; it must be a string`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!Array.isArray(doc.sheets) || doc.sheets.length === 0) {
|
||||||
|
throw new Error(`Document ${doc.documentId} must include a non-empty 'sheets' array`);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const sheet of doc.sheets) {
|
||||||
|
if (typeof sheet === 'string') {
|
||||||
|
if (!sheet.trim()) {
|
||||||
|
throw new Error(`Document ${doc.documentId} contains an empty sheet name`);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!sheet || typeof sheet !== 'object') {
|
||||||
|
throw new Error(`Document ${doc.documentId} contains an invalid sheet entry`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof sheet.name !== 'string' || !sheet.name.trim()) {
|
||||||
|
throw new Error(`Document ${doc.documentId} contains a sheet entry missing name`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sheet.outputFilename !== undefined && typeof sheet.outputFilename !== 'string') {
|
||||||
|
throw new Error(`Document ${doc.documentId} sheet ${sheet.name} has an invalid outputFilename; it must be a string`);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {loadConfig, resolveOutputPath, validateConfig};
|
module.exports = {loadConfig, resolveOutputPath, validateConfig, createStarterConfig, getDefaultConfigPath, writeConfigFile};
|
||||||
|
|||||||
@@ -0,0 +1,230 @@
|
|||||||
|
const readline = require('readline');
|
||||||
|
const {google} = require('googleapis');
|
||||||
|
const {createStarterConfig, getDefaultConfigPath, writeConfigFile} = require('./config');
|
||||||
|
const {getFirstServiceAccountAuthClient} = require('./auth');
|
||||||
|
|
||||||
|
function createPrompt() {
|
||||||
|
const rl = readline.createInterface({input: process.stdin, output: process.stdout});
|
||||||
|
|
||||||
|
const ask = question => new Promise(resolve => {
|
||||||
|
rl.question(question, answer => resolve((answer || '').trim()));
|
||||||
|
});
|
||||||
|
|
||||||
|
const close = () => new Promise(resolve => {
|
||||||
|
rl.close();
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
|
||||||
|
return {ask, close};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function askNonEmpty(ask, question) {
|
||||||
|
while (true) {
|
||||||
|
const answer = await ask(question);
|
||||||
|
if (answer) {
|
||||||
|
return answer;
|
||||||
|
}
|
||||||
|
console.log('Please enter a value.');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function askPositiveInteger(ask, question) {
|
||||||
|
while (true) {
|
||||||
|
const answer = await ask(question);
|
||||||
|
const value = Number.parseInt(answer, 10);
|
||||||
|
if (Number.isInteger(value) && value > 0) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
console.log('Please enter a whole number greater than zero.');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function askChoice(ask, question, options) {
|
||||||
|
const optionText = options
|
||||||
|
.map((option, index) => `${index + 1}. ${option.label}`)
|
||||||
|
.join('\n');
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const answer = await ask(`${question}\n${optionText}\n> `);
|
||||||
|
const lower = answer.toLowerCase();
|
||||||
|
|
||||||
|
const byNumber = Number.parseInt(answer, 10);
|
||||||
|
if (Number.isInteger(byNumber) && byNumber >= 1 && byNumber <= options.length) {
|
||||||
|
return options[byNumber - 1].value;
|
||||||
|
}
|
||||||
|
|
||||||
|
const match = options.find(option => option.value.toLowerCase() === lower || option.label.toLowerCase() === lower);
|
||||||
|
if (match) {
|
||||||
|
return match.value;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Please choose one of the listed options.');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function askOptionalPositiveInteger(ask, question, defaultValue) {
|
||||||
|
while (true) {
|
||||||
|
const answer = await ask(question);
|
||||||
|
if (!answer) {
|
||||||
|
return defaultValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const value = Number.parseInt(answer, 10);
|
||||||
|
if (Number.isInteger(value) && value > 0) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Please enter a whole number greater than zero, or press Enter to use the default.');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function pauseForEnter(ask) {
|
||||||
|
await ask('Press Enter to close this window...');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchSpreadsheetTitle(context, documentId) {
|
||||||
|
const sheetOptions = {version: 'v4'};
|
||||||
|
if (context.authClient) {
|
||||||
|
sheetOptions.auth = context.authClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
const sheets = google.sheets(sheetOptions);
|
||||||
|
const request = {
|
||||||
|
spreadsheetId: documentId,
|
||||||
|
fields: 'properties/title',
|
||||||
|
};
|
||||||
|
|
||||||
|
if (context.apiKey) {
|
||||||
|
request.key = context.apiKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = await sheets.spreadsheets.get(request);
|
||||||
|
return response.data.properties?.title || documentId;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildTitleContext(authMode, config) {
|
||||||
|
if (authMode === 'apiKeys' && Array.isArray(config.apiKeys) && config.apiKeys.length > 0) {
|
||||||
|
return {apiKey: config.apiKeys[0].key};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (authMode === 'serviceAccounts' && Array.isArray(config.serviceAccounts) && config.serviceAccounts.length > 0) {
|
||||||
|
const authClient = getFirstServiceAccountAuthClient(config);
|
||||||
|
if (authClient) {
|
||||||
|
return {authClient};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runOnboarding() {
|
||||||
|
const {ask, close} = createPrompt();
|
||||||
|
const configPath = getDefaultConfigPath();
|
||||||
|
|
||||||
|
try {
|
||||||
|
console.log('Google Sheets Rate Assistant setup');
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
const setupMode = await askChoice(
|
||||||
|
ask,
|
||||||
|
'Would you like to use the wizard or just create a config file?',
|
||||||
|
[
|
||||||
|
{label: 'Wizard', value: 'wizard'},
|
||||||
|
{label: 'Config file', value: 'config'},
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
if (setupMode === 'config') {
|
||||||
|
createStarterConfig(configPath);
|
||||||
|
console.log('');
|
||||||
|
console.log(`Starter config created at ${configPath}`);
|
||||||
|
console.log('Edit that file, then run the app again.');
|
||||||
|
await pauseForEnter(ask);
|
||||||
|
return {mode: 'config', configPath};
|
||||||
|
}
|
||||||
|
|
||||||
|
const authMode = await askChoice(
|
||||||
|
ask,
|
||||||
|
'Which authentication method do you want to use?',
|
||||||
|
[
|
||||||
|
{label: 'API keys', value: 'apiKeys'},
|
||||||
|
{label: 'Service accounts', value: 'serviceAccounts'},
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
const credentialCount = await askPositiveInteger(
|
||||||
|
ask,
|
||||||
|
`How many ${authMode === 'apiKeys' ? 'API keys' : 'service accounts'} do you want to add? `
|
||||||
|
);
|
||||||
|
|
||||||
|
const config = {
|
||||||
|
apiKeys: [],
|
||||||
|
serviceAccounts: [],
|
||||||
|
documents: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
if (authMode === 'apiKeys') {
|
||||||
|
for (let index = 0; index < credentialCount; index += 1) {
|
||||||
|
const key = await askNonEmpty(ask, `Enter API key ${index + 1}: `);
|
||||||
|
const rateLimitPerMinute = await askOptionalPositiveInteger(
|
||||||
|
ask,
|
||||||
|
`Rate limit per minute for API key ${index + 1} (press Enter for 50): `,
|
||||||
|
50
|
||||||
|
);
|
||||||
|
|
||||||
|
config.apiKeys.push({key, rateLimitPerMinute});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for (let index = 0; index < credentialCount; index += 1) {
|
||||||
|
const pathValue = await askNonEmpty(ask, `Enter service account path ${index + 1}: `);
|
||||||
|
const rateLimitPerMinute = await askOptionalPositiveInteger(
|
||||||
|
ask,
|
||||||
|
`Rate limit per minute for service account ${index + 1} (press Enter for 50): `,
|
||||||
|
50
|
||||||
|
);
|
||||||
|
|
||||||
|
config.serviceAccounts.push({path: pathValue, rateLimitPerMinute});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const documentCount = await askPositiveInteger(ask, 'How many documents do you want to pull from? ');
|
||||||
|
|
||||||
|
for (let documentIndex = 0; documentIndex < documentCount; documentIndex += 1) {
|
||||||
|
const documentId = await askNonEmpty(ask, `Enter document ID ${documentIndex + 1}: `);
|
||||||
|
let documentTitle = documentId;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const titleContext = buildTitleContext(authMode, config);
|
||||||
|
documentTitle = await fetchSpreadsheetTitle(titleContext, documentId);
|
||||||
|
} catch (err) {
|
||||||
|
console.log(`Could not load spreadsheet title for ${documentId}; using the document ID instead.`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const sheetCount = await askPositiveInteger(ask, `How many sheets for document ${documentIndex + 1}? `);
|
||||||
|
const sheets = [];
|
||||||
|
|
||||||
|
for (let sheetIndex = 0; sheetIndex < sheetCount; sheetIndex += 1) {
|
||||||
|
const sheetName = await askNonEmpty(ask, `Enter sheet name ${sheetIndex + 1} for document ${documentIndex + 1}: `);
|
||||||
|
sheets.push({
|
||||||
|
name: sheetName,
|
||||||
|
outputFilename: `${sheetName}.csv`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
config.documents.push({
|
||||||
|
documentId,
|
||||||
|
outputDir: `output/${documentTitle}`,
|
||||||
|
sheets,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
writeConfigFile(configPath, config);
|
||||||
|
console.log('');
|
||||||
|
console.log(`Config saved to ${configPath}`);
|
||||||
|
return {mode: 'wizard', configPath, config};
|
||||||
|
} finally {
|
||||||
|
await close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {runOnboarding};
|
||||||
+18
-3
@@ -1,10 +1,25 @@
|
|||||||
const pThrottleModule = require('p-throttle');
|
function sleep(milliseconds) {
|
||||||
const pThrottle = pThrottleModule.default || pThrottleModule;
|
return new Promise(resolve => setTimeout(resolve, milliseconds));
|
||||||
|
}
|
||||||
|
|
||||||
function buildThrottle(maxRequestsPerMinute) {
|
function buildThrottle(maxRequestsPerMinute) {
|
||||||
const limit = Math.max(1, Math.floor(maxRequestsPerMinute));
|
const limit = Math.max(1, Math.floor(maxRequestsPerMinute));
|
||||||
const interval = Math.ceil(60000 / limit);
|
const interval = Math.ceil(60000 / limit);
|
||||||
return pThrottle({limit: 1, interval});
|
let nextAvailable = 0;
|
||||||
|
|
||||||
|
return function throttle(fn) {
|
||||||
|
return async function throttledFunction(...args) {
|
||||||
|
const now = Date.now();
|
||||||
|
const waitMs = Math.max(0, nextAvailable - now);
|
||||||
|
nextAvailable = Math.max(nextAvailable, now) + interval;
|
||||||
|
|
||||||
|
if (waitMs > 0) {
|
||||||
|
await sleep(waitMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
return fn(...args);
|
||||||
|
};
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function applyRateLimitBuffer(rateLimitPerMinute) {
|
function applyRateLimitBuffer(rateLimitPerMinute) {
|
||||||
|
|||||||
Generated
+1555
-2
File diff suppressed because it is too large
Load Diff
+12
-2
@@ -1,14 +1,24 @@
|
|||||||
{
|
{
|
||||||
"name": "google-sheets-rate-assistant",
|
"name": "google-sheets-rate-assistant",
|
||||||
"version": "2.0.0",
|
"version": "2.1.0",
|
||||||
"description": "Pull Google Sheets data and export to CSV with configurable schedules and rate limiting.",
|
"description": "Pull Google Sheets data and export to CSV with configurable schedules and rate limiting.",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
|
"bin": "index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "node index.js"
|
"start": "node index.js",
|
||||||
|
"build:win": "node scripts/build-win.js"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"csv-stringify": "^6.0.0",
|
"csv-stringify": "^6.0.0",
|
||||||
"googleapis": "^173.0.0",
|
"googleapis": "^173.0.0",
|
||||||
"p-throttle": "^8.1.0"
|
"p-throttle": "^8.1.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"pkg": "^5.8.1"
|
||||||
|
},
|
||||||
|
"pkg": {
|
||||||
|
"assets": [
|
||||||
|
"node_modules/csv-stringify/dist/cjs/sync.cjs"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,20 +4,13 @@ This Node.js application pulls configured Google Sheets documents and exports th
|
|||||||
|
|
||||||
## Setup
|
## Setup
|
||||||
|
|
||||||
1. Copy `config.example.json` to `config.json`.
|
1. Download and run the latest version.
|
||||||
2. Configure authentication in `config.json` — choose one of:
|
2. If `config.json` does not exist, the app will ask whether you want a wizard or a starter config file.
|
||||||
|
3. If you choose the config-file path instead of the wizard, configure authentication in `config.json` — choose one of:
|
||||||
|
|
||||||
- Service account: place your service account JSON in `credentials.json` and set `credentialsPath`, or add one or more entries to the `serviceAccounts` array (each entry may be a path string or an object with `path` and optional `rateLimitPerMinute`).
|
- Service account: place your service account JSON in `credentials.json` and set `credentialsPath`, or add one or more entries to the `serviceAccounts` array (each entry may be a path string or an object with `path` and optional `rateLimitPerMinute`).
|
||||||
- API keys: add one or more API keys to the `apiKeys` array in `config.json` (each entry may be a string or an object with `key` and optional `rateLimitPerMinute`).
|
- API keys: add one or more API keys to the `apiKeys` array in `config.json` (each entry may be a string or an object with `key` and optional `rateLimitPerMinute`).
|
||||||
|
|
||||||
3. Install dependencies:
|
|
||||||
|
|
||||||
npm install
|
|
||||||
|
|
||||||
4. Run the application:
|
|
||||||
|
|
||||||
node index.js
|
|
||||||
|
|
||||||
## Setup Guides
|
## Setup Guides
|
||||||
|
|
||||||
For detailed instructions on authentication methods, please refer to the following guides:
|
For detailed instructions on authentication methods, please refer to the following guides:
|
||||||
@@ -63,7 +56,23 @@ There is a 5% buffer applied to any limitation to cater for any network fluctuat
|
|||||||
|
|
||||||
## Output
|
## Output
|
||||||
|
|
||||||
CSV files are written to `output` by default. Individual documents may override this with their own `outputDir` in `config.json`.
|
CSV files are written to `output` by default. The onboarding wizard uses that output directory automatically and names each file after the sheet.
|
||||||
|
|
||||||
|
If `config.json` is missing, the app creates a starter file next to the executable when packaged, or in the current working directory when running from source, and exits so you can fill it in before the next run.
|
||||||
|
|
||||||
|
If you choose the wizard, it will ask for the auth method, credential count, document count, and sheet names. It will not ask for output directories or file names.
|
||||||
|
|
||||||
|
## Windows Packaging
|
||||||
|
|
||||||
|
To build a Windows executable from source, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run build:win
|
||||||
|
```
|
||||||
|
|
||||||
|
That creates `dist/gSheets-Rate-Assistant-v2.0.0.exe` using the current version in the filename. For distribution, ship the exe alongside a `config.json` and any service account credential files, or set `GSA_CONFIG` to point at a different config file.
|
||||||
|
|
||||||
|
Gitea now builds only on version tags like `v1.1.1` through the workflow in `.gitea/workflows/build.yml`.
|
||||||
|
|
||||||
## AI Disclosure
|
## AI Disclosure
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,25 @@
|
|||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const {exec} = require('pkg');
|
||||||
|
const {version: packageVersion} = require('../package.json');
|
||||||
|
|
||||||
|
function normalizeBuildVersion(value) {
|
||||||
|
const version = (value || packageVersion || '0.0.0').toString().trim();
|
||||||
|
return version.startsWith('v') ? version : `v${version}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const distDir = path.resolve(__dirname, '..', 'dist');
|
||||||
|
if (!fs.existsSync(distDir)) {
|
||||||
|
fs.mkdirSync(distDir, {recursive: true});
|
||||||
|
}
|
||||||
|
|
||||||
|
const buildVersion = normalizeBuildVersion(process.env.BUILD_VERSION);
|
||||||
|
const outputFile = path.join(distDir, `gSheets-Rate-Assistant-${buildVersion}.exe`);
|
||||||
|
await exec(['.', '--targets', 'latest-win-x64', '--output', outputFile]);
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch(error => {
|
||||||
|
console.error(error && error.stack ? error.stack : error);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user