Added onboarding and exe output.
build-windows-exe / build (push) Failing after 1m41s

This commit is contained in:
2026-07-17 09:57:49 +01:00
parent 12edaa054d
commit a598abe1e6
9 changed files with 2193 additions and 47 deletions
+33
View File
@@ -0,0 +1,33 @@
name: build-windows-exe
on:
push:
tags:
- 'v*.*.*'
jobs:
build:
runs-on: ubuntu-latest
env:
BUILD_VERSION: ${{ github.ref_name }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '24'
cache: npm
- name: Install dependencies
run: npm ci
- name: Build Windows executable
run: npm run build:win
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: gSheets-Rate-Assistant-${{ github.ref_name }}
path: dist/gSheets-Rate-Assistant-${{ github.ref_name }}.exe
+92 -16
View File
@@ -6,21 +6,22 @@
const fs = require('fs');
const path = require('path');
const {google} = require('googleapis');
const stringifyModule = require('csv-stringify/sync');
const stringifyModule = require('./node_modules/csv-stringify/dist/cjs/sync.cjs');
const stringify = typeof stringifyModule === 'function'
? stringifyModule
: stringifyModule.default || stringifyModule.stringify || stringifyModule;
const {loadConfig, resolveOutputPath, validateConfig} = require('./lib/config');
const {loadConfig, resolveOutputPath} = require('./lib/config');
const {buildThrottle, applyRateLimitBuffer, getTotalRateLimitPerMinute} = require('./lib/rate');
const {runOnboarding} = require('./lib/onboarding');
const authHelpers = require('./lib/auth');
const {createApiKeyBatchFetcher, createServiceAccountBatchFetcher, getFirstApiKey, getFirstServiceAccountAuthClient, loadAuth} = authHelpers;
// === Status rendering / console display ===
// Renders per-sheet status lines when running in a TTY, updating in-place.
function mapDocumentToFilename(documentConfig, sheetName) {
function mapDocumentToFilename(sheetName) {
const safeName = sheetName.replace(/[\\/:*?"<>|]/g, '_');
return `${documentConfig.documentId}-${safeName}.csv`;
return `${safeName}.csv`;
}
const statusManager = {
@@ -109,7 +110,7 @@ function buildDocumentGroups(config) {
for (const sheetConfig of documentConfig.sheets) {
const name = typeof sheetConfig === 'string' ? sheetConfig : sheetConfig.name;
const outputFilename = sheetConfig.outputFilename || mapDocumentToFilename(documentConfig, name);
const outputFilename = sheetConfig.outputFilename || mapDocumentToFilename(name);
const outputPath = resolveOutputPath(documentOutputDir, outputFilename);
sheets.push({
@@ -250,17 +251,65 @@ function sleep(milliseconds) {
return new Promise(resolve => setTimeout(resolve, milliseconds));
}
function waitForEnter() {
if (!process.stdin || (!process.stdin.isTTY && !(process.stdout && process.stdout.isTTY))) {
return Promise.resolve();
}
return new Promise(resolve => {
const readline = require('readline');
const rl = readline.createInterface({input: process.stdin, output: process.stdout});
rl.question('Press Enter to close this window...', () => {
rl.close();
resolve();
});
});
}
function formatError(err) {
if (!err) {
return 'Unknown error';
}
if (err.message) {
return err.message;
}
return String(err);
}
function isUserFacingError(err) {
if (!err) {
return false;
}
const message = formatError(err);
return /config\.json|api key|service account|spreadsheet|sheet|credential|document/i.test(message);
}
async function handleFatalError(err, prefix = 'Application error:') {
const message = formatError(err);
const friendlyPrefix = prefix || 'Application error:';
console.error(friendlyPrefix);
if (isUserFacingError(err)) {
console.error(message);
} else {
console.error('Something went wrong.');
if (message && message !== 'Unknown error') {
console.error(message);
}
}
await waitForEnter();
process.exit(1);
}
// === Main scheduler ===
// Orchestrates startup, selects the appropriate fetcher (API keys,
// service accounts), and runs the periodic loop.
async function scheduleRuns() {
const config = loadConfig();
try {
validateConfig(config);
} catch (e) {
console.error('Configuration error:', e.message);
process.exit(1);
}
async function scheduleRuns(config) {
const apiBatchFetcher = createApiKeyBatchFetcher(config);
const svcBatchFetcher = createServiceAccountBatchFetcher(config);
let fetcher;
@@ -370,8 +419,35 @@ async function scheduleRuns() {
}
if (require.main === module) {
scheduleRuns().catch(err => {
console.error('Application error:', err.message || err);
process.exit(1);
process.on('uncaughtException', err => {
handleFatalError(err, 'Unexpected error:').catch(() => process.exit(1));
});
process.on('unhandledRejection', err => {
handleFatalError(err, 'Unexpected error:').catch(() => process.exit(1));
});
(async () => {
try {
let config;
try {
config = loadConfig();
} catch (err) {
if (err && err.code === 'ERR_CONFIG_MISSING') {
const onboardingResult = await runOnboarding();
if (onboardingResult && onboardingResult.mode === 'config') {
return;
}
config = onboardingResult.config;
} else {
throw err;
}
}
await scheduleRuns(config);
} catch (err) {
handleFatalError(err, 'Configuration error:').catch(() => process.exit(1));
}
})();
}
+208 -13
View File
@@ -1,14 +1,125 @@
const fs = require('fs');
const path = require('path');
const CONFIG_PATH = path.resolve(__dirname, '..', 'config.json');
const CONFIG_FILE_NAME = 'config.json';
const DEFAULT_CONFIG_TEMPLATE = JSON.stringify({
apiKeys: [
{
key: 'YOUR_API_KEY_1',
rateLimitPerMinute: 50,
},
{
key: 'YOUR_API_KEY_2',
rateLimitPerMinute: 50,
},
],
serviceAccounts: [
{
path: 'credentials.json',
rateLimitPerMinute: 50,
},
{
path: 'credentials_2.json',
rateLimitPerMinute: 50,
},
],
documents: [
{
documentId: 'YOUR_SPREADSHEET_ID',
outputDir: 'output/your-document',
sheets: [
{
name: 'Sheet1',
outputFilename: 'sheet1.csv',
},
{
name: 'Sheet2',
outputFilename: 'sheet2.csv',
},
],
},
{
documentId: 'YOUR_SPREADSHEET_ID_2',
outputDir: 'output/your-document-2',
sheets: [
{
name: 'Sheet1',
outputFilename: 'sheet1.csv',
},
{
name: 'Sheet2',
outputFilename: 'sheet2.csv',
},
],
},
],
}, null, 2) + '\n';
function getDefaultConfigPath() {
const packagedApp = Boolean(process.pkg);
return path.resolve(packagedApp ? path.dirname(process.execPath) : process.cwd(), CONFIG_FILE_NAME);
}
function getConfigPathCandidates() {
const packagedApp = Boolean(process.pkg);
const externalDefault = getDefaultConfigPath();
return [
process.env.GSA_CONFIG,
externalDefault,
!packagedApp ? path.resolve(__dirname, '..', CONFIG_FILE_NAME) : null,
].filter(Boolean);
}
function createStarterConfig(configPath) {
const directory = path.dirname(configPath);
if (!fs.existsSync(directory)) {
fs.mkdirSync(directory, {recursive: true});
}
fs.writeFileSync(configPath, DEFAULT_CONFIG_TEMPLATE, 'utf8');
}
function writeConfigFile(configPath, config) {
const directory = path.dirname(configPath);
if (!fs.existsSync(directory)) {
fs.mkdirSync(directory, {recursive: true});
}
fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n', 'utf8');
}
function getApiKeyValue(item) {
if (typeof item === 'string') {
return item.trim();
}
if (item && typeof item === 'object') {
return (item.key || item.apiKey || '').toString().trim();
}
return '';
}
function isPlaceholderApiKey(key) {
return key === 'YOUR_API_KEY_HERE';
}
function loadConfig() {
if (!fs.existsSync(CONFIG_PATH)) {
throw new Error(`Missing config file at ${CONFIG_PATH}`);
const configPath = getConfigPathCandidates().find(candidate => fs.existsSync(candidate));
if (!configPath) {
const starterPath = process.env.GSA_CONFIG || getDefaultConfigPath();
const error = new Error(`Missing config file at ${starterPath}`);
error.code = 'ERR_CONFIG_MISSING';
error.configPath = starterPath;
throw error;
}
const json = fs.readFileSync(CONFIG_PATH, 'utf8');
return JSON.parse(json);
const json = fs.readFileSync(configPath, 'utf8');
const config = JSON.parse(json);
validateConfig(config);
return config;
}
function resolveOutputPath(outputDir, filename) {
@@ -26,6 +137,18 @@ function validateConfig(config) {
throw new Error('Missing or invalid configuration object');
}
if (config.apiKeys !== undefined && !Array.isArray(config.apiKeys)) {
throw new Error('`apiKeys` must be an array when present in config.json');
}
if (config.serviceAccounts !== undefined && !Array.isArray(config.serviceAccounts)) {
throw new Error('`serviceAccounts` must be an array when present in config.json');
}
if (config.documents !== undefined && !Array.isArray(config.documents)) {
throw new Error('`documents` must be an array when present in config.json');
}
if (!Array.isArray(config.documents) || config.documents.length === 0) {
throw new Error('`documents` must be a non-empty array in config.json');
}
@@ -44,20 +167,58 @@ function validateConfig(config) {
// Ensure API key values are unique when using apiKeys
if (hasApiKeys) {
const extracted = config.apiKeys.map(item => {
if (typeof item === 'string') return item.trim();
if (item && typeof item === 'object') return (item.key || item.apiKey || '').toString().trim();
return '';
});
const extracted = config.apiKeys.map(getApiKeyValue);
const nonEmpty = extracted.filter(Boolean);
const unique = new Set(nonEmpty);
if (unique.size !== nonEmpty.length) {
throw new Error('`apiKeys` contains duplicate `key` values; ensure all API keys are unique');
}
for (const item of config.apiKeys) {
const key = getApiKeyValue(item);
if (!key) {
throw new Error('Each entry in `apiKeys` must include a non-empty `key`');
}
if (isPlaceholderApiKey(key)) {
throw new Error('Each entry in `apiKeys` must include a real API key; replace `YOUR_API_KEY_HERE`');
}
if (typeof item !== 'string' && (!item || typeof item !== 'object')) {
throw new Error('`apiKeys` entries must be strings or objects');
}
if (item.rateLimitPerMinute !== undefined && (!Number.isFinite(item.rateLimitPerMinute) || item.rateLimitPerMinute <= 0)) {
throw new Error('`rateLimitPerMinute` in `apiKeys` must be a positive number when provided');
}
}
}
// Ensure service account credential paths are unique when using serviceAccounts
if (hasSvcAccounts) {
for (const item of config.serviceAccounts) {
if (typeof item === 'string') {
if (!item.trim()) {
throw new Error('Each string entry in `serviceAccounts` must be non-empty');
}
continue;
}
if (!item || typeof item !== 'object') {
throw new Error('`serviceAccounts` entries must be strings or objects');
}
const p = (item.path || item.credentialsPath || item.file || '').toString().trim();
if (!p) {
throw new Error('Each entry in `serviceAccounts` must include `path` or `credentialsPath`');
}
if (item.rateLimitPerMinute !== undefined && (!Number.isFinite(item.rateLimitPerMinute) || item.rateLimitPerMinute <= 0)) {
throw new Error('`rateLimitPerMinute` in `serviceAccounts` must be a positive number when provided');
}
}
// Read and canonicalize the contents of each service account credential
const svcContents = config.serviceAccounts.map(item => {
let p = '';
@@ -92,9 +253,43 @@ function validateConfig(config) {
// Basic validation of documents/sheets structure
for (const doc of config.documents) {
if (!doc.documentId) throw new Error('Each document must include a `documentId`');
if (!Array.isArray(doc.sheets) || doc.sheets.length === 0) throw new Error(`Document ${doc.documentId} must include a non-empty 'sheets' array`);
if (!doc || typeof doc !== 'object') {
throw new Error('Each document entry must be an object');
}
if (typeof doc.documentId !== 'string' || !doc.documentId.trim()) {
throw new Error('Each document must include a non-empty `documentId`');
}
if (doc.outputDir !== undefined && typeof doc.outputDir !== 'string') {
throw new Error(`Document ${doc.documentId} has an invalid outputDir; it must be a string`);
}
if (!Array.isArray(doc.sheets) || doc.sheets.length === 0) {
throw new Error(`Document ${doc.documentId} must include a non-empty 'sheets' array`);
}
for (const sheet of doc.sheets) {
if (typeof sheet === 'string') {
if (!sheet.trim()) {
throw new Error(`Document ${doc.documentId} contains an empty sheet name`);
}
continue;
}
if (!sheet || typeof sheet !== 'object') {
throw new Error(`Document ${doc.documentId} contains an invalid sheet entry`);
}
if (typeof sheet.name !== 'string' || !sheet.name.trim()) {
throw new Error(`Document ${doc.documentId} contains a sheet entry missing name`);
}
if (sheet.outputFilename !== undefined && typeof sheet.outputFilename !== 'string') {
throw new Error(`Document ${doc.documentId} sheet ${sheet.name} has an invalid outputFilename; it must be a string`);
}
}
}
}
module.exports = {loadConfig, resolveOutputPath, validateConfig};
module.exports = {loadConfig, resolveOutputPath, validateConfig, createStarterConfig, getDefaultConfigPath, writeConfigFile};
+230
View File
@@ -0,0 +1,230 @@
const readline = require('readline');
const {google} = require('googleapis');
const {createStarterConfig, getDefaultConfigPath, writeConfigFile} = require('./config');
const {getFirstServiceAccountAuthClient} = require('./auth');
function createPrompt() {
const rl = readline.createInterface({input: process.stdin, output: process.stdout});
const ask = question => new Promise(resolve => {
rl.question(question, answer => resolve((answer || '').trim()));
});
const close = () => new Promise(resolve => {
rl.close();
resolve();
});
return {ask, close};
}
async function askNonEmpty(ask, question) {
while (true) {
const answer = await ask(question);
if (answer) {
return answer;
}
console.log('Please enter a value.');
}
}
async function askPositiveInteger(ask, question) {
while (true) {
const answer = await ask(question);
const value = Number.parseInt(answer, 10);
if (Number.isInteger(value) && value > 0) {
return value;
}
console.log('Please enter a whole number greater than zero.');
}
}
async function askChoice(ask, question, options) {
const optionText = options
.map((option, index) => `${index + 1}. ${option.label}`)
.join('\n');
while (true) {
const answer = await ask(`${question}\n${optionText}\n> `);
const lower = answer.toLowerCase();
const byNumber = Number.parseInt(answer, 10);
if (Number.isInteger(byNumber) && byNumber >= 1 && byNumber <= options.length) {
return options[byNumber - 1].value;
}
const match = options.find(option => option.value.toLowerCase() === lower || option.label.toLowerCase() === lower);
if (match) {
return match.value;
}
console.log('Please choose one of the listed options.');
}
}
async function askOptionalPositiveInteger(ask, question, defaultValue) {
while (true) {
const answer = await ask(question);
if (!answer) {
return defaultValue;
}
const value = Number.parseInt(answer, 10);
if (Number.isInteger(value) && value > 0) {
return value;
}
console.log('Please enter a whole number greater than zero, or press Enter to use the default.');
}
}
async function pauseForEnter(ask) {
await ask('Press Enter to close this window...');
}
async function fetchSpreadsheetTitle(context, documentId) {
const sheetOptions = {version: 'v4'};
if (context.authClient) {
sheetOptions.auth = context.authClient;
}
const sheets = google.sheets(sheetOptions);
const request = {
spreadsheetId: documentId,
fields: 'properties/title',
};
if (context.apiKey) {
request.key = context.apiKey;
}
const response = await sheets.spreadsheets.get(request);
return response.data.properties?.title || documentId;
}
function buildTitleContext(authMode, config) {
if (authMode === 'apiKeys' && Array.isArray(config.apiKeys) && config.apiKeys.length > 0) {
return {apiKey: config.apiKeys[0].key};
}
if (authMode === 'serviceAccounts' && Array.isArray(config.serviceAccounts) && config.serviceAccounts.length > 0) {
const authClient = getFirstServiceAccountAuthClient(config);
if (authClient) {
return {authClient};
}
}
return {};
}
async function runOnboarding() {
const {ask, close} = createPrompt();
const configPath = getDefaultConfigPath();
try {
console.log('Google Sheets Rate Assistant setup');
console.log('');
const setupMode = await askChoice(
ask,
'Would you like to use the wizard or just create a config file?',
[
{label: 'Wizard', value: 'wizard'},
{label: 'Config file', value: 'config'},
]
);
if (setupMode === 'config') {
createStarterConfig(configPath);
console.log('');
console.log(`Starter config created at ${configPath}`);
console.log('Edit that file, then run the app again.');
await pauseForEnter(ask);
return {mode: 'config', configPath};
}
const authMode = await askChoice(
ask,
'Which authentication method do you want to use?',
[
{label: 'API keys', value: 'apiKeys'},
{label: 'Service accounts', value: 'serviceAccounts'},
]
);
const credentialCount = await askPositiveInteger(
ask,
`How many ${authMode === 'apiKeys' ? 'API keys' : 'service accounts'} do you want to add? `
);
const config = {
apiKeys: [],
serviceAccounts: [],
documents: [],
};
if (authMode === 'apiKeys') {
for (let index = 0; index < credentialCount; index += 1) {
const key = await askNonEmpty(ask, `Enter API key ${index + 1}: `);
const rateLimitPerMinute = await askOptionalPositiveInteger(
ask,
`Rate limit per minute for API key ${index + 1} (press Enter for 50): `,
50
);
config.apiKeys.push({key, rateLimitPerMinute});
}
} else {
for (let index = 0; index < credentialCount; index += 1) {
const pathValue = await askNonEmpty(ask, `Enter service account path ${index + 1}: `);
const rateLimitPerMinute = await askOptionalPositiveInteger(
ask,
`Rate limit per minute for service account ${index + 1} (press Enter for 50): `,
50
);
config.serviceAccounts.push({path: pathValue, rateLimitPerMinute});
}
}
const documentCount = await askPositiveInteger(ask, 'How many documents do you want to pull from? ');
for (let documentIndex = 0; documentIndex < documentCount; documentIndex += 1) {
const documentId = await askNonEmpty(ask, `Enter document ID ${documentIndex + 1}: `);
let documentTitle = documentId;
try {
const titleContext = buildTitleContext(authMode, config);
documentTitle = await fetchSpreadsheetTitle(titleContext, documentId);
} catch (err) {
console.log(`Could not load spreadsheet title for ${documentId}; using the document ID instead.`);
}
const sheetCount = await askPositiveInteger(ask, `How many sheets for document ${documentIndex + 1}? `);
const sheets = [];
for (let sheetIndex = 0; sheetIndex < sheetCount; sheetIndex += 1) {
const sheetName = await askNonEmpty(ask, `Enter sheet name ${sheetIndex + 1} for document ${documentIndex + 1}: `);
sheets.push({
name: sheetName,
outputFilename: `${sheetName}.csv`,
});
}
config.documents.push({
documentId,
outputDir: `output/${documentTitle}`,
sheets,
});
}
writeConfigFile(configPath, config);
console.log('');
console.log(`Config saved to ${configPath}`);
return {mode: 'wizard', configPath, config};
} finally {
await close();
}
}
module.exports = {runOnboarding};
+18 -3
View File
@@ -1,10 +1,25 @@
const pThrottleModule = require('p-throttle');
const pThrottle = pThrottleModule.default || pThrottleModule;
function sleep(milliseconds) {
return new Promise(resolve => setTimeout(resolve, milliseconds));
}
function buildThrottle(maxRequestsPerMinute) {
const limit = Math.max(1, Math.floor(maxRequestsPerMinute));
const interval = Math.ceil(60000 / limit);
return pThrottle({limit: 1, interval});
let nextAvailable = 0;
return function throttle(fn) {
return async function throttledFunction(...args) {
const now = Date.now();
const waitMs = Math.max(0, nextAvailable - now);
nextAvailable = Math.max(nextAvailable, now) + interval;
if (waitMs > 0) {
await sleep(waitMs);
}
return fn(...args);
};
};
}
function applyRateLimitBuffer(rateLimitPerMinute) {
+1555 -2
View File
File diff suppressed because it is too large Load Diff
+12 -2
View File
@@ -1,14 +1,24 @@
{
"name": "google-sheets-rate-assistant",
"version": "2.0.0",
"version": "2.1.0",
"description": "Pull Google Sheets data and export to CSV with configurable schedules and rate limiting.",
"main": "index.js",
"bin": "index.js",
"scripts": {
"start": "node index.js"
"start": "node index.js",
"build:win": "node scripts/build-win.js"
},
"dependencies": {
"csv-stringify": "^6.0.0",
"googleapis": "^173.0.0",
"p-throttle": "^8.1.0"
},
"devDependencies": {
"pkg": "^5.8.1"
},
"pkg": {
"assets": [
"node_modules/csv-stringify/dist/cjs/sync.cjs"
]
}
}
+20 -11
View File
@@ -4,20 +4,13 @@ This Node.js application pulls configured Google Sheets documents and exports th
## Setup
1. Copy `config.example.json` to `config.json`.
2. Configure authentication in `config.json` — choose one of:
1. Download and run the latest version.
2. If `config.json` does not exist, the app will ask whether you want a wizard or a starter config file.
3. If you choose the config-file path instead of the wizard, configure authentication in `config.json` — choose one of:
- Service account: place your service account JSON in `credentials.json` and set `credentialsPath`, or add one or more entries to the `serviceAccounts` array (each entry may be a path string or an object with `path` and optional `rateLimitPerMinute`).
- API keys: add one or more API keys to the `apiKeys` array in `config.json` (each entry may be a string or an object with `key` and optional `rateLimitPerMinute`).
3. Install dependencies:
npm install
4. Run the application:
node index.js
## Setup Guides
For detailed instructions on authentication methods, please refer to the following guides:
@@ -63,7 +56,23 @@ There is a 5% buffer applied to any limitation to cater for any network fluctuat
## Output
CSV files are written to `output` by default. Individual documents may override this with their own `outputDir` in `config.json`.
CSV files are written to `output` by default. The onboarding wizard uses that output directory automatically and names each file after the sheet.
If `config.json` is missing, the app creates a starter file next to the executable when packaged, or in the current working directory when running from source, and exits so you can fill it in before the next run.
If you choose the wizard, it will ask for the auth method, credential count, document count, and sheet names. It will not ask for output directories or file names.
## Windows Packaging
To build a Windows executable from source, run:
```bash
npm run build:win
```
That creates `dist/gSheets-Rate-Assistant-v2.0.0.exe` using the current version in the filename. For distribution, ship the exe alongside a `config.json` and any service account credential files, or set `GSA_CONFIG` to point at a different config file.
Gitea now builds only on version tags like `v1.1.1` through the workflow in `.gitea/workflows/build.yml`.
## AI Disclosure
+25
View File
@@ -0,0 +1,25 @@
const fs = require('fs');
const path = require('path');
const {exec} = require('pkg');
const {version: packageVersion} = require('../package.json');
function normalizeBuildVersion(value) {
const version = (value || packageVersion || '0.0.0').toString().trim();
return version.startsWith('v') ? version : `v${version}`;
}
async function main() {
const distDir = path.resolve(__dirname, '..', 'dist');
if (!fs.existsSync(distDir)) {
fs.mkdirSync(distDir, {recursive: true});
}
const buildVersion = normalizeBuildVersion(process.env.BUILD_VERSION);
const outputFile = path.join(distDir, `gSheets-Rate-Assistant-${buildVersion}.exe`);
await exec(['.', '--targets', 'latest-win-x64', '--output', outputFile]);
}
main().catch(error => {
console.error(error && error.stack ? error.stack : error);
process.exit(1);
});