12 Commits
10 changed files with 506 additions and 128 deletions
+102
View File
@@ -37,3 +37,105 @@ jobs:
tag_name: ${{ github.ref_name }} tag_name: ${{ github.ref_name }}
name: gSheets-Rate-Assistant ${{ github.ref_name }} name: gSheets-Rate-Assistant ${{ github.ref_name }}
files: dist/*.exe files: dist/*.exe
- name: Mirror release to GitHub
env:
GITHUB_TOKEN: ${{ secrets.RELEASE_GITHUB_TOKEN }}
RELEASE_GITHUB_REPO: ${{ secrets.RELEASE_GITHUB_REPO }}
run: |
python - <<'PY'
import json
import os
import sys
import urllib.error
import urllib.parse
import urllib.request
token = os.environ.get("GITHUB_TOKEN", "")
repository = os.environ.get("RELEASE_GITHUB_REPO", "")
if not token or not repository:
print("Skipping GitHub mirror because RELEASE_GITHUB_REPO or RELEASE_GITHUB_TOKEN is missing.")
raise SystemExit(0)
tag = os.environ["BUILD_VERSION"]
release_name = f"gSheets-Rate-Assistant {tag}"
asset_path = next((os.path.join("dist", name) for name in os.listdir("dist") if name.endswith(".exe")), None)
if asset_path is None:
raise SystemExit("No Windows executable found in dist/")
def request_json(url, method="GET", payload=None, headers=None):
request_headers = {
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
}
if headers:
request_headers.update(headers)
data = None if payload is None else json.dumps(payload).encode("utf-8")
if data is not None:
request_headers["Content-Type"] = "application/json"
request = urllib.request.Request(url, data=data, method=method, headers=request_headers)
try:
with urllib.request.urlopen(request) as response:
body = response.read().decode("utf-8")
return json.loads(body) if body else {}
except urllib.error.HTTPError as error:
message = error.read().decode("utf-8", errors="replace")
print(message, file=sys.stderr)
raise
release_url = f"https://api.github.com/repos/{repository}/releases/tags/{urllib.parse.quote(tag)}"
release = None
try:
release = request_json(release_url)
except urllib.error.HTTPError as error:
if error.code != 404:
raise
payload = {
"tag_name": tag,
"name": release_name,
"draft": False,
"prerelease": False,
"generate_release_notes": False,
}
if release is None:
release = request_json(f"https://api.github.com/repos/{repository}/releases", method="POST", payload=payload)
else:
release = request_json(release["url"], method="PATCH", payload=payload)
asset_name = os.path.basename(asset_path)
for asset in release.get("assets", []):
if asset.get("name") == asset_name:
request_json(asset["url"], method="DELETE")
upload_url = release["upload_url"].split("{", 1)[0]
with open(asset_path, "rb") as asset_file:
asset_data = asset_file.read()
upload_request = urllib.request.Request(
f"{upload_url}?name={urllib.parse.quote(asset_name)}",
data=asset_data,
method="POST",
headers={
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"Content-Type": "application/octet-stream",
"X-GitHub-Api-Version": "2022-11-28",
},
)
try:
with urllib.request.urlopen(upload_request) as response:
response.read()
except urllib.error.HTTPError as error:
message = error.read().decode("utf-8", errors="replace")
print(message, file=sys.stderr)
raise
PY
+1
View File
@@ -9,6 +9,7 @@ dist/
# Credentials and config (do NOT commit secrets) # Credentials and config (do NOT commit secrets)
credentials.json credentials.json
credentials-*.json credentials-*.json
serviceAccounts/
config.json config.json
config-*.json config-*.json
.env .env
+69 -59
View File
@@ -5,7 +5,7 @@
// === Imports & constants === // === Imports & constants ===
const fs = require('fs'); const fs = require('fs');
const path = require('path'); const path = require('path');
const {google} = require('googleapis'); const readline = require('readline');
const stringifyModule = require('./node_modules/csv-stringify/dist/cjs/sync.cjs'); const stringifyModule = require('./node_modules/csv-stringify/dist/cjs/sync.cjs');
const stringify = typeof stringifyModule === 'function' const stringify = typeof stringifyModule === 'function'
? stringifyModule ? stringifyModule
@@ -14,6 +14,7 @@ const {loadConfig, resolveOutputPath} = require('./lib/config');
const {buildThrottle, applyRateLimitBuffer, getTotalRateLimitPerMinute} = require('./lib/rate'); const {buildThrottle, applyRateLimitBuffer, getTotalRateLimitPerMinute} = require('./lib/rate');
const {runOnboarding} = require('./lib/onboarding'); const {runOnboarding} = require('./lib/onboarding');
const authHelpers = require('./lib/auth'); const authHelpers = require('./lib/auth');
const {fetchSpreadsheetTitle, fetchSpreadsheetValuesBatch} = require('./lib/sheets');
const {createApiKeyBatchFetcher, createServiceAccountBatchFetcher, getFirstApiKey, getFirstServiceAccountAuthClient, loadAuth} = authHelpers; const {createApiKeyBatchFetcher, createServiceAccountBatchFetcher, getFirstApiKey, getFirstServiceAccountAuthClient, loadAuth} = authHelpers;
// === Status rendering / console display === // === Status rendering / console display ===
@@ -26,11 +27,36 @@ function mapDocumentToFilename(sheetName) {
const statusManager = { const statusManager = {
keys: [], keys: [],
lineIndexes: new Map(),
statuses: new Map(), statuses: new Map(),
lastRenderLines: 0, anchorSaved: false,
enabled: process.stdout && process.stdout.isTTY, enabled: process.stdout && process.stdout.isTTY,
}; };
function clearConsoleForRun() {
if (process.stdout && process.stdout.isTTY && typeof console.clear === 'function') {
console.clear();
return;
}
if (process.stdout && typeof process.stdout.write === 'function') {
process.stdout.write('\u001b[2J\u001b[0f');
}
}
function limitToTerminalWidth(text) {
const width = Math.max(20, (process.stdout && process.stdout.columns) || 80);
if (text.length <= width) {
return text;
}
if (width <= 1) {
return text.slice(0, width);
}
return text.slice(0, width - 1) + '…';
}
// === Document grouping & mapping === // === Document grouping & mapping ===
// Map `config.documents` entries to internal groups with resolved output paths. // Map `config.documents` entries to internal groups with resolved output paths.
@@ -39,31 +65,54 @@ function renderStatusBlock() {
return; return;
} }
if (statusManager.lastRenderLines > 0) { process.stdout.write('\u001b[s');
process.stdout.write(`\x1B[${statusManager.lastRenderLines}F`);
}
for (const key of statusManager.keys) { statusManager.keys.forEach((key, index) => {
const statusText = statusManager.statuses.get(key) || `${key} - pending`; const statusText = statusManager.statuses.get(key) || `${key} - pending`;
process.stdout.clearLine(0); readline.cursorTo(process.stdout, 0);
process.stdout.cursorTo(0); readline.clearLine(process.stdout, 0);
process.stdout.write(statusText + '\n'); process.stdout.write(limitToTerminalWidth(statusText));
if (index < statusManager.keys.length - 1) {
process.stdout.write('\n');
}
});
statusManager.anchorSaved = true;
}
function updateStatusLine(taskId, statusText) {
if (!statusManager.enabled || !statusManager.anchorSaved) {
return;
} }
statusManager.lastRenderLines = statusManager.keys.length; const lineIndex = statusManager.lineIndexes.get(taskId);
if (lineIndex === undefined) {
return;
}
process.stdout.write('\u001b[u');
if (lineIndex > 0) {
readline.moveCursor(process.stdout, 0, lineIndex);
}
readline.cursorTo(process.stdout, 0);
readline.clearLine(process.stdout, 0);
process.stdout.write(limitToTerminalWidth(statusText));
process.stdout.write('\u001b[u');
} }
function setTaskStatus(taskId, statusText) { function setTaskStatus(taskId, statusText) {
statusManager.statuses.set(taskId, statusText); statusManager.statuses.set(taskId, statusText);
renderStatusBlock(); updateStatusLine(taskId, statusText);
} }
function initializeTaskStatuses(tasks) { function initializeTaskStatuses(tasks) {
statusManager.keys = tasks.map(task => `${task.documentId}:${task.sheetName}`); statusManager.keys = tasks.map(task => `${task.documentId}:${task.sheetName}`);
statusManager.lineIndexes.clear();
statusManager.statuses.clear(); statusManager.statuses.clear();
statusManager.lastRenderLines = 0; statusManager.anchorSaved = false;
for (const key of statusManager.keys) { for (const [index, key] of statusManager.keys.entries()) {
statusManager.lineIndexes.set(key, index);
statusManager.statuses.set(key, `${key} - pending`); statusManager.statuses.set(key, `${key} - pending`);
} }
@@ -73,26 +122,6 @@ function initializeTaskStatuses(tasks) {
// Authentication helpers: see `./lib/auth` for credential loaders and // Authentication helpers: see `./lib/auth` for credential loaders and
// per-credential, throttled batch fetchers (API key and service account modes). // per-credential, throttled batch fetchers (API key and service account modes).
async function fetchSpreadsheetTitle(context, documentId) {
const sheetOptions = {version: 'v4'};
if (context.authClient) {
sheetOptions.auth = context.authClient;
}
const sheets = google.sheets(sheetOptions);
const request = {
spreadsheetId: documentId,
fields: 'properties/title',
};
if (context.apiKey) {
request.key = context.apiKey;
}
const response = await sheets.spreadsheets.get(request);
return response.data.properties?.title || documentId;
}
function buildDocumentGroups(config) { function buildDocumentGroups(config) {
const groups = []; const groups = [];
@@ -132,31 +161,7 @@ function buildDocumentGroups(config) {
// `index.js` provides a simple `batchFetchDocumentSheets` fallback used when // `index.js` provides a simple `batchFetchDocumentSheets` fallback used when
// no per-credential batch fetcher is configured by `./lib/auth`. // no per-credential batch fetcher is configured by `./lib/auth`.
async function batchFetchDocumentSheets(context, documentId, sheetNames) { async function batchFetchDocumentSheets(context, documentId, sheetNames) {
const sheetOptions = {version: 'v4'}; return fetchSpreadsheetValuesBatch(context, documentId, sheetNames);
if (context.authClient) {
sheetOptions.auth = context.authClient;
}
const sheets = google.sheets(sheetOptions);
const request = {
spreadsheetId: documentId,
ranges: sheetNames,
};
if (context.apiKey) {
request.key = context.apiKey;
}
const response = await sheets.spreadsheets.values.batchGet(request);
const valueRanges = response.data.valueRanges || [];
const rowsBySheet = {};
for (let i = 0; i < sheetNames.length; i += 1) {
const sheetName = sheetNames[i];
rowsBySheet[sheetName] = valueRanges[i]?.values || [];
}
return rowsBySheet;
} }
function formatStatusTimestamp(date = new Date()) { function formatStatusTimestamp(date = new Date()) {
@@ -331,6 +336,8 @@ async function scheduleRuns(config) {
throw new Error('No sheets configured to pull in config.json'); throw new Error('No sheets configured to pull in config.json');
} }
console.log('Google Sheets Rate Assistant');
const firstApiKey = getFirstApiKey(config); const firstApiKey = getFirstApiKey(config);
const titleMap = {}; const titleMap = {};
@@ -393,7 +400,8 @@ async function scheduleRuns(config) {
const totalRateLimit = getTotalRateLimitPerMinute(config); const totalRateLimit = getTotalRateLimitPerMinute(config);
const intervalMs = Math.max(1, Math.ceil(60000 / totalRateLimit)); const intervalMs = Math.max(1, Math.ceil(60000 / totalRateLimit));
console.log(`Configured for ${totalRateLimit} requests per minute across ${groups.length} documents (${intervalMs}ms interval)`); console.log(limitToTerminalWidth(`Configured for ${totalRateLimit} requests per minute across ${groups.length} documents (${intervalMs}ms interval)`));
console.log('');
initializeTaskStatuses(tasks); initializeTaskStatuses(tasks);
let currentIndex = 0; let currentIndex = 0;
@@ -429,6 +437,8 @@ if (require.main === module) {
(async () => { (async () => {
try { try {
clearConsoleForRun();
let config; let config;
try { try {
+27 -24
View File
@@ -1,7 +1,28 @@
const fs = require('fs'); const fs = require('fs');
const path = require('path'); const path = require('path');
const {google} = require('googleapis');
const {applyRateLimitBuffer} = require('./rate'); const {applyRateLimitBuffer} = require('./rate');
const {fetchSpreadsheetValuesBatch} = require('./sheets');
function getConfigBaseDir(config) {
const configPath = config && (config.__configPath || config.configPath);
if (configPath) {
return path.dirname(configPath);
}
return process.cwd();
}
function resolveConfigPath(config, candidatePath) {
if (!candidatePath) {
return '';
}
if (path.isAbsolute(candidatePath)) {
return candidatePath;
}
return path.resolve(getConfigBaseDir(config), candidatePath);
}
function loadApiKeys(config) { function loadApiKeys(config) {
const apiKeys = []; const apiKeys = [];
@@ -60,7 +81,7 @@ function createServiceAccountBatchFetcher(config) {
// selecting a credential whose slot is available now. If none are // selecting a credential whose slot is available now. If none are
// immediately available, wait the minimum required time. // immediately available, wait the minimum required time.
const fetchers = accounts.map(({path: credentialsPath, rateLimitPerMinute}) => { const fetchers = accounts.map(({path: credentialsPath, rateLimitPerMinute}) => {
const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath); const resolvedPath = resolveConfigPath(config, credentialsPath);
if (!fs.existsSync(resolvedPath)) { if (!fs.existsSync(resolvedPath)) {
throw new Error(`Service account credentials not found: ${resolvedPath}`); throw new Error(`Service account credentials not found: ${resolvedPath}`);
} }
@@ -84,16 +105,7 @@ function createServiceAccountBatchFetcher(config) {
const interval = Math.ceil(60000 / bufferedRateLimit); const interval = Math.ceil(60000 / bufferedRateLimit);
const fetchRaw = async (documentId, sheetNames) => { const fetchRaw = async (documentId, sheetNames) => {
const sheets = google.sheets({version: 'v4', auth: authClient}); return fetchSpreadsheetValuesBatch({authClient}, documentId, sheetNames);
const request = {spreadsheetId: documentId, ranges: sheetNames};
const response = await sheets.spreadsheets.values.batchGet(request);
const valueRanges = response.data.valueRanges || [];
const rowsBySheet = {};
for (let i = 0; i < sheetNames.length; i += 1) {
const sheetName = sheetNames[i];
rowsBySheet[sheetName] = valueRanges[i]?.values || [];
}
return rowsBySheet;
}; };
return {authClient, fetchRaw, interval, nextAvailable: 0}; return {authClient, fetchRaw, interval, nextAvailable: 0};
@@ -148,16 +160,7 @@ function createApiKeyBatchFetcher(config) {
const interval = Math.ceil(60000 / bufferedRateLimit); const interval = Math.ceil(60000 / bufferedRateLimit);
const fetchRaw = async (documentId, sheetNames) => { const fetchRaw = async (documentId, sheetNames) => {
const sheets = google.sheets({version: 'v4'}); return fetchSpreadsheetValuesBatch({apiKey: key}, documentId, sheetNames);
const request = {spreadsheetId: documentId, ranges: sheetNames, key};
const response = await sheets.spreadsheets.values.batchGet(request);
const valueRanges = response.data.valueRanges || [];
const rowsBySheet = {};
for (let i = 0; i < sheetNames.length; i += 1) {
const sheetName = sheetNames[i];
rowsBySheet[sheetName] = valueRanges[i]?.values || [];
}
return rowsBySheet;
}; };
return {key, fetchRaw, interval, nextAvailable: 0}; return {key, fetchRaw, interval, nextAvailable: 0};
@@ -215,7 +218,7 @@ function getFirstServiceAccountAuthClient(config) {
const credentialsPath = entryObj.path || entryObj.credentialsPath; const credentialsPath = entryObj.path || entryObj.credentialsPath;
if (!credentialsPath) return null; if (!credentialsPath) return null;
const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath); const resolvedPath = resolveConfigPath(config, credentialsPath);
if (!fs.existsSync(resolvedPath)) return null; if (!fs.existsSync(resolvedPath)) return null;
try { try {
@@ -235,7 +238,7 @@ function loadAuth(config) {
if (config.credentialsPath) { if (config.credentialsPath) {
const credentialsPath = config.credentialsPath; const credentialsPath = config.credentialsPath;
const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath); const resolvedPath = resolveConfigPath(config, credentialsPath);
if (!fs.existsSync(resolvedPath)) { if (!fs.existsSync(resolvedPath)) {
throw new Error(`Credentials file not found: ${resolvedPath}`); throw new Error(`Credentials file not found: ${resolvedPath}`);
} }
+18 -1
View File
@@ -118,10 +118,25 @@ function loadConfig() {
const json = fs.readFileSync(configPath, 'utf8'); const json = fs.readFileSync(configPath, 'utf8');
const config = JSON.parse(json); const config = JSON.parse(json);
Object.defineProperty(config, '__configPath', {
value: configPath,
enumerable: false,
configurable: true,
writable: true,
});
validateConfig(config); validateConfig(config);
return config; return config;
} }
function getConfigBaseDir(config) {
const configPath = config && (config.__configPath || config.configPath);
if (configPath) {
return path.dirname(configPath);
}
return process.cwd();
}
function resolveOutputPath(outputDir, filename) { function resolveOutputPath(outputDir, filename) {
if (!path.isAbsolute(outputDir)) { if (!path.isAbsolute(outputDir)) {
outputDir = path.resolve(process.cwd(), outputDir); outputDir = path.resolve(process.cwd(), outputDir);
@@ -197,6 +212,8 @@ function validateConfig(config) {
// Ensure service account credential paths are unique when using serviceAccounts // Ensure service account credential paths are unique when using serviceAccounts
if (hasSvcAccounts) { if (hasSvcAccounts) {
const baseDir = getConfigBaseDir(config);
for (const item of config.serviceAccounts) { for (const item of config.serviceAccounts) {
if (typeof item === 'string') { if (typeof item === 'string') {
if (!item.trim()) { if (!item.trim()) {
@@ -225,7 +242,7 @@ function validateConfig(config) {
if (typeof item === 'string') p = item.trim(); if (typeof item === 'string') p = item.trim();
else if (item && typeof item === 'object') p = (item.path || item.file || item.credentialsPath || '').toString().trim(); else if (item && typeof item === 'object') p = (item.path || item.file || item.credentialsPath || '').toString().trim();
if (!p) return ''; if (!p) return '';
const full = path.isAbsolute(p) ? p : path.resolve(process.cwd(), p); const full = path.isAbsolute(p) ? p : path.resolve(baseDir, p);
if (!fs.existsSync(full)) { if (!fs.existsSync(full)) {
throw new Error(`Service account file not found: ${p}`); throw new Error(`Service account file not found: ${p}`);
} }
+133 -26
View File
@@ -1,7 +1,9 @@
const fs = require('fs');
const path = require('path');
const readline = require('readline'); const readline = require('readline');
const {google} = require('googleapis');
const {createStarterConfig, getDefaultConfigPath, writeConfigFile} = require('./config'); const {createStarterConfig, getDefaultConfigPath, writeConfigFile} = require('./config');
const {getFirstServiceAccountAuthClient} = require('./auth'); const {getFirstServiceAccountAuthClient} = require('./auth');
const {fetchSpreadsheetTitle} = require('./sheets');
function createPrompt() { function createPrompt() {
const rl = readline.createInterface({input: process.stdin, output: process.stdout}); const rl = readline.createInterface({input: process.stdin, output: process.stdout});
@@ -78,28 +80,81 @@ async function askOptionalPositiveInteger(ask, question, defaultValue) {
} }
} }
async function pauseForEnter(ask) { function getServiceAccountIdentity(inputPath, baseDir) {
await ask('Press Enter to close this window...'); const value = (inputPath || '').trim();
if (!value) {
return '';
}
const fullPath = path.isAbsolute(value) ? value : path.resolve(baseDir || process.cwd(), value);
if (!fs.existsSync(fullPath)) {
return value;
}
const raw = fs.readFileSync(fullPath, 'utf8').trim();
try {
const parsed = JSON.parse(raw);
const canonicalize = obj => {
if (obj === null || typeof obj !== 'object') return JSON.stringify(obj);
if (Array.isArray(obj)) return '[' + obj.map(canonicalize).join(',') + ']';
const keys = Object.keys(obj).sort();
return '{' + keys.map(key => JSON.stringify(key) + ':' + canonicalize(obj[key])).join(',') + '}';
};
return canonicalize(parsed);
} catch (_) {
return raw;
}
} }
async function fetchSpreadsheetTitle(context, documentId) { function getServiceAccountsFolderPath(configPath) {
const sheetOptions = {version: 'v4'}; return path.join(path.dirname(configPath), 'serviceAccounts');
if (context.authClient) { }
sheetOptions.auth = context.authClient;
function listServiceAccountFiles(folderPath) {
if (!fs.existsSync(folderPath)) {
return [];
} }
const sheets = google.sheets(sheetOptions); return fs.readdirSync(folderPath)
const request = { .map(fileName => path.join(folderPath, fileName))
spreadsheetId: documentId, .filter(filePath => {
fields: 'properties/title', try {
}; return fs.statSync(filePath).isFile();
} catch (_) {
return false;
}
});
}
if (context.apiKey) { function extractSpreadsheetId(input) {
request.key = context.apiKey; const value = (input || '').trim();
if (!value) {
return '';
} }
const response = await sheets.spreadsheets.get(request); const urlMatch = value.match(/\/spreadsheets\/d\/([a-zA-Z0-9-_]+)/i);
return response.data.properties?.title || documentId; if (urlMatch && urlMatch[1]) {
return urlMatch[1];
}
return value;
}
async function askSpreadsheetIdOrUrl(ask, question) {
while (true) {
const answer = await ask(question);
const documentId = extractSpreadsheetId(answer);
if (documentId) {
return documentId;
}
console.log('Please enter a Google Sheets URL or spreadsheet ID.');
}
}
async function pauseForEnter(ask) {
await ask('Press Enter to close this window...');
} }
function buildTitleContext(authMode, config) { function buildTitleContext(authMode, config) {
@@ -152,45 +207,97 @@ async function runOnboarding() {
] ]
); );
const credentialCount = await askPositiveInteger( if (authMode === 'apiKeys') {
ask, console.log('Warning: when using API keys, the spreadsheet must be publicly accessible.');
`How many ${authMode === 'apiKeys' ? 'API keys' : 'service accounts'} do you want to add? ` console.log('');
); }
const config = { const config = {
apiKeys: [], apiKeys: [],
serviceAccounts: [], serviceAccounts: [],
documents: [], documents: [],
}; };
Object.defineProperty(config, '__configPath', {
value: configPath,
enumerable: false,
configurable: true,
writable: true,
});
if (authMode === 'apiKeys') { if (authMode === 'apiKeys') {
const credentialCount = await askPositiveInteger(
ask,
'How many API keys do you want to add? '
);
const seenKeys = new Set();
for (let index = 0; index < credentialCount; index += 1) { for (let index = 0; index < credentialCount; index += 1) {
const key = await askNonEmpty(ask, `Enter API key ${index + 1}: `); let key;
while (true) {
key = await askNonEmpty(ask, `Enter API key ${index + 1}: `);
if (seenKeys.has(key)) {
console.log('That API key was already entered. Please enter a unique API key.');
continue;
}
break;
}
const rateLimitPerMinute = await askOptionalPositiveInteger( const rateLimitPerMinute = await askOptionalPositiveInteger(
ask, ask,
`Rate limit per minute for API key ${index + 1} (press Enter for 50): `, `Rate limit per minute for API key ${index + 1} (press Enter for 50): `,
50 50
); );
seenKeys.add(key);
config.apiKeys.push({key, rateLimitPerMinute}); config.apiKeys.push({key, rateLimitPerMinute});
} }
} else { } else {
for (let index = 0; index < credentialCount; index += 1) { const serviceAccountsFolder = getServiceAccountsFolderPath(configPath);
const pathValue = await askNonEmpty(ask, `Enter service account path ${index + 1}: `); if (!fs.existsSync(serviceAccountsFolder)) {
fs.mkdirSync(serviceAccountsFolder, {recursive: true});
}
console.log(`A serviceAccounts folder has been created at ${serviceAccountsFolder}.`);
console.log('Paste your service account files into that folder, then press Enter to continue.');
await pauseForEnter(ask);
const serviceAccountFiles = listServiceAccountFiles(serviceAccountsFolder);
console.log(`Found ${serviceAccountFiles.length} service account${serviceAccountFiles.length === 1 ? '' : 's'}.`);
if (serviceAccountFiles.length === 0) {
throw new Error(`No service account files were found in ${serviceAccountsFolder}`);
}
const seenServiceAccounts = new Set();
for (const filePath of serviceAccountFiles) {
const relativePath = path.relative(path.dirname(configPath), filePath).split(path.sep).join('/');
const serviceAccountIdentity = getServiceAccountIdentity(relativePath, path.dirname(configPath));
if (seenServiceAccounts.has(serviceAccountIdentity)) {
console.log(`Skipping duplicate service account file: ${path.basename(filePath)}`);
continue;
}
seenServiceAccounts.add(serviceAccountIdentity);
const rateLimitPerMinute = await askOptionalPositiveInteger( const rateLimitPerMinute = await askOptionalPositiveInteger(
ask, ask,
`Rate limit per minute for service account ${index + 1} (press Enter for 50): `, `Rate limit per minute for ${path.basename(filePath)} (press Enter for 50): `,
50 50
); );
config.serviceAccounts.push({path: pathValue, rateLimitPerMinute}); config.serviceAccounts.push({path: relativePath, rateLimitPerMinute});
} }
} }
const documentCount = await askPositiveInteger(ask, 'How many documents do you want to pull from? '); const documentCount = await askPositiveInteger(ask, 'How many documents do you want to pull from? ');
for (let documentIndex = 0; documentIndex < documentCount; documentIndex += 1) { for (let documentIndex = 0; documentIndex < documentCount; documentIndex += 1) {
const documentId = await askNonEmpty(ask, `Enter document ID ${documentIndex + 1}: `); const documentId = await askSpreadsheetIdOrUrl(
ask,
`Enter Google Sheets URL or document ID ${documentIndex + 1}: `
);
let documentTitle = documentId; let documentTitle = documentId;
try { try {
+149
View File
@@ -0,0 +1,149 @@
const https = require('https');
const {URL} = require('url');
function parseResponseBody(body) {
if (!body) {
return {};
}
try {
return JSON.parse(body);
} catch (_) {
return body;
}
}
function extractErrorMessage(data) {
if (!data) {
return '';
}
if (typeof data === 'string') {
return data;
}
if (data.error && typeof data.error === 'object') {
if (typeof data.error.message === 'string' && data.error.message) {
return data.error.message;
}
if (Array.isArray(data.error.errors) && data.error.errors.length > 0) {
const firstError = data.error.errors[0];
if (firstError && typeof firstError.message === 'string' && firstError.message) {
return firstError.message;
}
}
}
if (typeof data.message === 'string' && data.message) {
return data.message;
}
return '';
}
function requestJson(urlString, headers = {}) {
return new Promise((resolve, reject) => {
const url = new URL(urlString);
const request = https.request(
url,
{
method: 'GET',
headers: Object.assign({Accept: 'application/json'}, headers),
},
response => {
let body = '';
response.setEncoding('utf8');
response.on('data', chunk => {
body += chunk;
});
response.on('end', () => {
const data = parseResponseBody(body);
if (response.statusCode >= 200 && response.statusCode < 300) {
resolve({status: response.statusCode, headers: response.headers, data});
return;
}
const message = extractErrorMessage(data) || `Request failed with status ${response.statusCode}`;
const error = new Error(message);
error.response = {status: response.statusCode, headers: response.headers, data};
reject(error);
});
}
);
request.on('error', reject);
request.end();
});
}
async function requestSheetsJson(pathname, context, queryParams = {}) {
const url = new URL(`https://sheets.googleapis.com${pathname}`);
for (const [key, value] of Object.entries(queryParams)) {
if (Array.isArray(value)) {
for (const item of value) {
url.searchParams.append(key, item);
}
} else if (value !== undefined && value !== null) {
url.searchParams.set(key, String(value));
}
}
const headers = {};
if (context && context.authClient) {
if (typeof context.authClient.getRequestHeaders === 'function') {
const authHeaders = await context.authClient.getRequestHeaders(url.toString());
Object.assign(headers, authHeaders || {});
} else if (typeof context.authClient.getAccessToken === 'function') {
const token = await context.authClient.getAccessToken();
const accessToken = token && typeof token === 'object' ? token.token || token.access_token : token;
if (accessToken) {
headers.Authorization = `Bearer ${accessToken}`;
}
}
}
if (context && context.apiKey) {
url.searchParams.set('key', context.apiKey);
}
return requestJson(url.toString(), headers);
}
async function fetchSpreadsheetTitle(context, documentId) {
const response = await requestSheetsJson(
`/v4/spreadsheets/${encodeURIComponent(documentId)}`,
context,
{fields: 'properties/title'}
);
return response.data && response.data.properties && response.data.properties.title || documentId;
}
async function fetchSpreadsheetValuesBatch(context, documentId, sheetNames) {
const response = await requestSheetsJson(
`/v4/spreadsheets/${encodeURIComponent(documentId)}/values:batchGet`,
context,
{ranges: sheetNames}
);
const valueRanges = response.data.valueRanges || [];
const rowsBySheet = {};
for (let i = 0; i < sheetNames.length; i += 1) {
const sheetName = sheetNames[i];
rowsBySheet[sheetName] = valueRanges[i] && valueRanges[i].values ? valueRanges[i].values : [];
}
return rowsBySheet;
}
module.exports = {
fetchSpreadsheetTitle,
fetchSpreadsheetValuesBatch,
};
+2 -2
View File
@@ -1,12 +1,12 @@
{ {
"name": "google-sheets-rate-assistant", "name": "google-sheets-rate-assistant",
"version": "2.0.0", "version": "2.1.9",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "google-sheets-rate-assistant", "name": "google-sheets-rate-assistant",
"version": "2.0.0", "version": "2.1.9",
"dependencies": { "dependencies": {
"csv-stringify": "^6.0.0", "csv-stringify": "^6.0.0",
"googleapis": "^173.0.0", "googleapis": "^173.0.0",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "google-sheets-rate-assistant", "name": "google-sheets-rate-assistant",
"version": "2.1.0", "version": "2.1.9",
"description": "Pull Google Sheets data and export to CSV with configurable schedules and rate limiting.", "description": "Pull Google Sheets data and export to CSV with configurable schedules and rate limiting.",
"main": "index.js", "main": "index.js",
"bin": "index.js", "bin": "index.js",
+4 -15
View File
@@ -4,9 +4,10 @@ This Node.js application pulls configured Google Sheets documents and exports th
## Setup ## Setup
1. Download and run the latest version. Download the latest version from the [Releases tab](./releases) and run it.
2. If `config.json` does not exist, the app will ask whether you want a wizard or a starter config file.
3. If you choose the config-file path instead of the wizard, configure authentication in `config.json` — choose one of: 1. If `config.json` does not exist, the app will ask whether you want a wizard or a starter config file.
2. If you choose the config-file path instead of the wizard, configure authentication in `config.json` — choose one of:
- Service account: place your service account JSON in `credentials.json` and set `credentialsPath`, or add one or more entries to the `serviceAccounts` array (each entry may be a path string or an object with `path` and optional `rateLimitPerMinute`). - Service account: place your service account JSON in `credentials.json` and set `credentialsPath`, or add one or more entries to the `serviceAccounts` array (each entry may be a path string or an object with `path` and optional `rateLimitPerMinute`).
- API keys: add one or more API keys to the `apiKeys` array in `config.json` (each entry may be a string or an object with `key` and optional `rateLimitPerMinute`). - API keys: add one or more API keys to the `apiKeys` array in `config.json` (each entry may be a string or an object with `key` and optional `rateLimitPerMinute`).
@@ -62,18 +63,6 @@ If `config.json` is missing, the app creates a starter file next to the executab
If you choose the wizard, it will ask for the auth method, credential count, document count, and sheet names. It will not ask for output directories or file names. If you choose the wizard, it will ask for the auth method, credential count, document count, and sheet names. It will not ask for output directories or file names.
## Windows Packaging
To build a Windows executable from source, run:
```bash
npm run build:win
```
That creates `dist/gSheets-Rate-Assistant-v2.0.0.exe` using the current version in the filename. For distribution, ship the exe alongside a `config.json` and any service account credential files, or set `GSA_CONFIG` to point at a different config file.
Gitea now builds only on version tags like `v1.1.1` through the workflow in `.gitea/workflows/build.yml`.
## AI Disclosure ## AI Disclosure
A large chunk of the project was coded using AI, this is the first pass at integrating some AI assistance into my projects. However all code written has been verified and checked before submission. AI use has only been included since the v2 rewrite. A large chunk of the project was coded using AI, this is the first pass at integrating some AI assistance into my projects. However all code written has been verified and checked before submission. AI use has only been included since the v2 rewrite.