9 Commits
10 changed files with 508 additions and 117 deletions
+102
View File
@@ -37,3 +37,105 @@ jobs:
tag_name: ${{ github.ref_name }} tag_name: ${{ github.ref_name }}
name: gSheets-Rate-Assistant ${{ github.ref_name }} name: gSheets-Rate-Assistant ${{ github.ref_name }}
files: dist/*.exe files: dist/*.exe
- name: Mirror release to GitHub
env:
GITHUB_TOKEN: ${{ secrets.RELEASE_GITHUB_TOKEN }}
RELEASE_GITHUB_REPO: ${{ secrets.RELEASE_GITHUB_REPO }}
run: |
python - <<'PY'
import json
import os
import sys
import urllib.error
import urllib.parse
import urllib.request
token = os.environ.get("GITHUB_TOKEN", "")
repository = os.environ.get("RELEASE_GITHUB_REPO", "")
if not token or not repository:
print("Skipping GitHub mirror because RELEASE_GITHUB_REPO or RELEASE_GITHUB_TOKEN is missing.")
raise SystemExit(0)
tag = os.environ["BUILD_VERSION"]
release_name = f"gSheets-Rate-Assistant {tag}"
asset_path = next((os.path.join("dist", name) for name in os.listdir("dist") if name.endswith(".exe")), None)
if asset_path is None:
raise SystemExit("No Windows executable found in dist/")
def request_json(url, method="GET", payload=None, headers=None):
request_headers = {
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
}
if headers:
request_headers.update(headers)
data = None if payload is None else json.dumps(payload).encode("utf-8")
if data is not None:
request_headers["Content-Type"] = "application/json"
request = urllib.request.Request(url, data=data, method=method, headers=request_headers)
try:
with urllib.request.urlopen(request) as response:
body = response.read().decode("utf-8")
return json.loads(body) if body else {}
except urllib.error.HTTPError as error:
message = error.read().decode("utf-8", errors="replace")
print(message, file=sys.stderr)
raise
release_url = f"https://api.github.com/repos/{repository}/releases/tags/{urllib.parse.quote(tag)}"
release = None
try:
release = request_json(release_url)
except urllib.error.HTTPError as error:
if error.code != 404:
raise
payload = {
"tag_name": tag,
"name": release_name,
"draft": False,
"prerelease": False,
"generate_release_notes": False,
}
if release is None:
release = request_json(f"https://api.github.com/repos/{repository}/releases", method="POST", payload=payload)
else:
release = request_json(release["url"], method="PATCH", payload=payload)
asset_name = os.path.basename(asset_path)
for asset in release.get("assets", []):
if asset.get("name") == asset_name:
request_json(asset["url"], method="DELETE")
upload_url = release["upload_url"].split("{", 1)[0]
with open(asset_path, "rb") as asset_file:
asset_data = asset_file.read()
upload_request = urllib.request.Request(
f"{upload_url}?name={urllib.parse.quote(asset_name)}",
data=asset_data,
method="POST",
headers={
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"Content-Type": "application/octet-stream",
"X-GitHub-Api-Version": "2022-11-28",
},
)
try:
with urllib.request.urlopen(upload_request) as response:
response.read()
except urllib.error.HTTPError as error:
message = error.read().decode("utf-8", errors="replace")
print(message, file=sys.stderr)
raise
PY
+1
View File
@@ -9,6 +9,7 @@ dist/
# Credentials and config (do NOT commit secrets) # Credentials and config (do NOT commit secrets)
credentials.json credentials.json
credentials-*.json credentials-*.json
serviceAccounts/
config.json config.json
config-*.json config-*.json
.env .env
+69 -59
View File
@@ -5,7 +5,7 @@
// === Imports & constants === // === Imports & constants ===
const fs = require('fs'); const fs = require('fs');
const path = require('path'); const path = require('path');
const {google} = require('googleapis'); const readline = require('readline');
const stringifyModule = require('./node_modules/csv-stringify/dist/cjs/sync.cjs'); const stringifyModule = require('./node_modules/csv-stringify/dist/cjs/sync.cjs');
const stringify = typeof stringifyModule === 'function' const stringify = typeof stringifyModule === 'function'
? stringifyModule ? stringifyModule
@@ -14,6 +14,7 @@ const {loadConfig, resolveOutputPath} = require('./lib/config');
const {buildThrottle, applyRateLimitBuffer, getTotalRateLimitPerMinute} = require('./lib/rate'); const {buildThrottle, applyRateLimitBuffer, getTotalRateLimitPerMinute} = require('./lib/rate');
const {runOnboarding} = require('./lib/onboarding'); const {runOnboarding} = require('./lib/onboarding');
const authHelpers = require('./lib/auth'); const authHelpers = require('./lib/auth');
const {fetchSpreadsheetTitle, fetchSpreadsheetValuesBatch} = require('./lib/sheets');
const {createApiKeyBatchFetcher, createServiceAccountBatchFetcher, getFirstApiKey, getFirstServiceAccountAuthClient, loadAuth} = authHelpers; const {createApiKeyBatchFetcher, createServiceAccountBatchFetcher, getFirstApiKey, getFirstServiceAccountAuthClient, loadAuth} = authHelpers;
// === Status rendering / console display === // === Status rendering / console display ===
@@ -26,11 +27,36 @@ function mapDocumentToFilename(sheetName) {
const statusManager = { const statusManager = {
keys: [], keys: [],
lineIndexes: new Map(),
statuses: new Map(), statuses: new Map(),
lastRenderLines: 0, anchorSaved: false,
enabled: process.stdout && process.stdout.isTTY, enabled: process.stdout && process.stdout.isTTY,
}; };
function clearConsoleForRun() {
if (process.stdout && process.stdout.isTTY && typeof console.clear === 'function') {
console.clear();
return;
}
if (process.stdout && typeof process.stdout.write === 'function') {
process.stdout.write('\u001b[2J\u001b[0f');
}
}
function limitToTerminalWidth(text) {
const width = Math.max(20, (process.stdout && process.stdout.columns) || 80);
if (text.length <= width) {
return text;
}
if (width <= 1) {
return text.slice(0, width);
}
return text.slice(0, width - 1) + '…';
}
// === Document grouping & mapping === // === Document grouping & mapping ===
// Map `config.documents` entries to internal groups with resolved output paths. // Map `config.documents` entries to internal groups with resolved output paths.
@@ -39,31 +65,54 @@ function renderStatusBlock() {
return; return;
} }
if (statusManager.lastRenderLines > 0) { process.stdout.write('\u001b[s');
process.stdout.write(`\x1B[${statusManager.lastRenderLines}F`);
}
for (const key of statusManager.keys) { statusManager.keys.forEach((key, index) => {
const statusText = statusManager.statuses.get(key) || `${key} - pending`; const statusText = statusManager.statuses.get(key) || `${key} - pending`;
process.stdout.clearLine(0); readline.cursorTo(process.stdout, 0);
process.stdout.cursorTo(0); readline.clearLine(process.stdout, 0);
process.stdout.write(statusText + '\n'); process.stdout.write(limitToTerminalWidth(statusText));
if (index < statusManager.keys.length - 1) {
process.stdout.write('\n');
}
});
statusManager.anchorSaved = true;
}
function updateStatusLine(taskId, statusText) {
if (!statusManager.enabled || !statusManager.anchorSaved) {
return;
} }
statusManager.lastRenderLines = statusManager.keys.length; const lineIndex = statusManager.lineIndexes.get(taskId);
if (lineIndex === undefined) {
return;
}
process.stdout.write('\u001b[u');
if (lineIndex > 0) {
readline.moveCursor(process.stdout, 0, lineIndex);
}
readline.cursorTo(process.stdout, 0);
readline.clearLine(process.stdout, 0);
process.stdout.write(limitToTerminalWidth(statusText));
process.stdout.write('\u001b[u');
} }
function setTaskStatus(taskId, statusText) { function setTaskStatus(taskId, statusText) {
statusManager.statuses.set(taskId, statusText); statusManager.statuses.set(taskId, statusText);
renderStatusBlock(); updateStatusLine(taskId, statusText);
} }
function initializeTaskStatuses(tasks) { function initializeTaskStatuses(tasks) {
statusManager.keys = tasks.map(task => `${task.documentId}:${task.sheetName}`); statusManager.keys = tasks.map(task => `${task.documentId}:${task.sheetName}`);
statusManager.lineIndexes.clear();
statusManager.statuses.clear(); statusManager.statuses.clear();
statusManager.lastRenderLines = 0; statusManager.anchorSaved = false;
for (const key of statusManager.keys) { for (const [index, key] of statusManager.keys.entries()) {
statusManager.lineIndexes.set(key, index);
statusManager.statuses.set(key, `${key} - pending`); statusManager.statuses.set(key, `${key} - pending`);
} }
@@ -73,26 +122,6 @@ function initializeTaskStatuses(tasks) {
// Authentication helpers: see `./lib/auth` for credential loaders and // Authentication helpers: see `./lib/auth` for credential loaders and
// per-credential, throttled batch fetchers (API key and service account modes). // per-credential, throttled batch fetchers (API key and service account modes).
async function fetchSpreadsheetTitle(context, documentId) {
const sheetOptions = {version: 'v4'};
if (context.authClient) {
sheetOptions.auth = context.authClient;
}
const sheets = google.sheets(sheetOptions);
const request = {
spreadsheetId: documentId,
fields: 'properties/title',
};
if (context.apiKey) {
request.key = context.apiKey;
}
const response = await sheets.spreadsheets.get(request);
return response.data.properties?.title || documentId;
}
function buildDocumentGroups(config) { function buildDocumentGroups(config) {
const groups = []; const groups = [];
@@ -132,31 +161,7 @@ function buildDocumentGroups(config) {
// `index.js` provides a simple `batchFetchDocumentSheets` fallback used when // `index.js` provides a simple `batchFetchDocumentSheets` fallback used when
// no per-credential batch fetcher is configured by `./lib/auth`. // no per-credential batch fetcher is configured by `./lib/auth`.
async function batchFetchDocumentSheets(context, documentId, sheetNames) { async function batchFetchDocumentSheets(context, documentId, sheetNames) {
const sheetOptions = {version: 'v4'}; return fetchSpreadsheetValuesBatch(context, documentId, sheetNames);
if (context.authClient) {
sheetOptions.auth = context.authClient;
}
const sheets = google.sheets(sheetOptions);
const request = {
spreadsheetId: documentId,
ranges: sheetNames,
};
if (context.apiKey) {
request.key = context.apiKey;
}
const response = await sheets.spreadsheets.values.batchGet(request);
const valueRanges = response.data.valueRanges || [];
const rowsBySheet = {};
for (let i = 0; i < sheetNames.length; i += 1) {
const sheetName = sheetNames[i];
rowsBySheet[sheetName] = valueRanges[i]?.values || [];
}
return rowsBySheet;
} }
function formatStatusTimestamp(date = new Date()) { function formatStatusTimestamp(date = new Date()) {
@@ -331,6 +336,8 @@ async function scheduleRuns(config) {
throw new Error('No sheets configured to pull in config.json'); throw new Error('No sheets configured to pull in config.json');
} }
console.log('Google Sheets Rate Assistant');
const firstApiKey = getFirstApiKey(config); const firstApiKey = getFirstApiKey(config);
const titleMap = {}; const titleMap = {};
@@ -393,7 +400,8 @@ async function scheduleRuns(config) {
const totalRateLimit = getTotalRateLimitPerMinute(config); const totalRateLimit = getTotalRateLimitPerMinute(config);
const intervalMs = Math.max(1, Math.ceil(60000 / totalRateLimit)); const intervalMs = Math.max(1, Math.ceil(60000 / totalRateLimit));
console.log(`Configured for ${totalRateLimit} requests per minute across ${groups.length} documents (${intervalMs}ms interval)`); console.log(limitToTerminalWidth(`Configured for ${totalRateLimit} requests per minute across ${groups.length} documents (${intervalMs}ms interval)`));
console.log('');
initializeTaskStatuses(tasks); initializeTaskStatuses(tasks);
let currentIndex = 0; let currentIndex = 0;
@@ -429,6 +437,8 @@ if (require.main === module) {
(async () => { (async () => {
try { try {
clearConsoleForRun();
let config; let config;
try { try {
+27 -24
View File
@@ -1,7 +1,28 @@
const fs = require('fs'); const fs = require('fs');
const path = require('path'); const path = require('path');
const {google} = require('googleapis');
const {applyRateLimitBuffer} = require('./rate'); const {applyRateLimitBuffer} = require('./rate');
const {fetchSpreadsheetValuesBatch} = require('./sheets');
function getConfigBaseDir(config) {
const configPath = config && (config.__configPath || config.configPath);
if (configPath) {
return path.dirname(configPath);
}
return process.cwd();
}
function resolveConfigPath(config, candidatePath) {
if (!candidatePath) {
return '';
}
if (path.isAbsolute(candidatePath)) {
return candidatePath;
}
return path.resolve(getConfigBaseDir(config), candidatePath);
}
function loadApiKeys(config) { function loadApiKeys(config) {
const apiKeys = []; const apiKeys = [];
@@ -60,7 +81,7 @@ function createServiceAccountBatchFetcher(config) {
// selecting a credential whose slot is available now. If none are // selecting a credential whose slot is available now. If none are
// immediately available, wait the minimum required time. // immediately available, wait the minimum required time.
const fetchers = accounts.map(({path: credentialsPath, rateLimitPerMinute}) => { const fetchers = accounts.map(({path: credentialsPath, rateLimitPerMinute}) => {
const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath); const resolvedPath = resolveConfigPath(config, credentialsPath);
if (!fs.existsSync(resolvedPath)) { if (!fs.existsSync(resolvedPath)) {
throw new Error(`Service account credentials not found: ${resolvedPath}`); throw new Error(`Service account credentials not found: ${resolvedPath}`);
} }
@@ -84,16 +105,7 @@ function createServiceAccountBatchFetcher(config) {
const interval = Math.ceil(60000 / bufferedRateLimit); const interval = Math.ceil(60000 / bufferedRateLimit);
const fetchRaw = async (documentId, sheetNames) => { const fetchRaw = async (documentId, sheetNames) => {
const sheets = google.sheets({version: 'v4', auth: authClient}); return fetchSpreadsheetValuesBatch({authClient}, documentId, sheetNames);
const request = {spreadsheetId: documentId, ranges: sheetNames};
const response = await sheets.spreadsheets.values.batchGet(request);
const valueRanges = response.data.valueRanges || [];
const rowsBySheet = {};
for (let i = 0; i < sheetNames.length; i += 1) {
const sheetName = sheetNames[i];
rowsBySheet[sheetName] = valueRanges[i]?.values || [];
}
return rowsBySheet;
}; };
return {authClient, fetchRaw, interval, nextAvailable: 0}; return {authClient, fetchRaw, interval, nextAvailable: 0};
@@ -148,16 +160,7 @@ function createApiKeyBatchFetcher(config) {
const interval = Math.ceil(60000 / bufferedRateLimit); const interval = Math.ceil(60000 / bufferedRateLimit);
const fetchRaw = async (documentId, sheetNames) => { const fetchRaw = async (documentId, sheetNames) => {
const sheets = google.sheets({version: 'v4'}); return fetchSpreadsheetValuesBatch({apiKey: key}, documentId, sheetNames);
const request = {spreadsheetId: documentId, ranges: sheetNames, key};
const response = await sheets.spreadsheets.values.batchGet(request);
const valueRanges = response.data.valueRanges || [];
const rowsBySheet = {};
for (let i = 0; i < sheetNames.length; i += 1) {
const sheetName = sheetNames[i];
rowsBySheet[sheetName] = valueRanges[i]?.values || [];
}
return rowsBySheet;
}; };
return {key, fetchRaw, interval, nextAvailable: 0}; return {key, fetchRaw, interval, nextAvailable: 0};
@@ -215,7 +218,7 @@ function getFirstServiceAccountAuthClient(config) {
const credentialsPath = entryObj.path || entryObj.credentialsPath; const credentialsPath = entryObj.path || entryObj.credentialsPath;
if (!credentialsPath) return null; if (!credentialsPath) return null;
const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath); const resolvedPath = resolveConfigPath(config, credentialsPath);
if (!fs.existsSync(resolvedPath)) return null; if (!fs.existsSync(resolvedPath)) return null;
try { try {
@@ -235,7 +238,7 @@ function loadAuth(config) {
if (config.credentialsPath) { if (config.credentialsPath) {
const credentialsPath = config.credentialsPath; const credentialsPath = config.credentialsPath;
const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath); const resolvedPath = resolveConfigPath(config, credentialsPath);
if (!fs.existsSync(resolvedPath)) { if (!fs.existsSync(resolvedPath)) {
throw new Error(`Credentials file not found: ${resolvedPath}`); throw new Error(`Credentials file not found: ${resolvedPath}`);
} }
+18 -1
View File
@@ -118,10 +118,25 @@ function loadConfig() {
const json = fs.readFileSync(configPath, 'utf8'); const json = fs.readFileSync(configPath, 'utf8');
const config = JSON.parse(json); const config = JSON.parse(json);
Object.defineProperty(config, '__configPath', {
value: configPath,
enumerable: false,
configurable: true,
writable: true,
});
validateConfig(config); validateConfig(config);
return config; return config;
} }
function getConfigBaseDir(config) {
const configPath = config && (config.__configPath || config.configPath);
if (configPath) {
return path.dirname(configPath);
}
return process.cwd();
}
function resolveOutputPath(outputDir, filename) { function resolveOutputPath(outputDir, filename) {
if (!path.isAbsolute(outputDir)) { if (!path.isAbsolute(outputDir)) {
outputDir = path.resolve(process.cwd(), outputDir); outputDir = path.resolve(process.cwd(), outputDir);
@@ -197,6 +212,8 @@ function validateConfig(config) {
// Ensure service account credential paths are unique when using serviceAccounts // Ensure service account credential paths are unique when using serviceAccounts
if (hasSvcAccounts) { if (hasSvcAccounts) {
const baseDir = getConfigBaseDir(config);
for (const item of config.serviceAccounts) { for (const item of config.serviceAccounts) {
if (typeof item === 'string') { if (typeof item === 'string') {
if (!item.trim()) { if (!item.trim()) {
@@ -225,7 +242,7 @@ function validateConfig(config) {
if (typeof item === 'string') p = item.trim(); if (typeof item === 'string') p = item.trim();
else if (item && typeof item === 'object') p = (item.path || item.file || item.credentialsPath || '').toString().trim(); else if (item && typeof item === 'object') p = (item.path || item.file || item.credentialsPath || '').toString().trim();
if (!p) return ''; if (!p) return '';
const full = path.isAbsolute(p) ? p : path.resolve(process.cwd(), p); const full = path.isAbsolute(p) ? p : path.resolve(baseDir, p);
if (!fs.existsSync(full)) { if (!fs.existsSync(full)) {
throw new Error(`Service account file not found: ${p}`); throw new Error(`Service account file not found: ${p}`);
} }
+133 -26
View File
@@ -1,7 +1,9 @@
const fs = require('fs');
const path = require('path');
const readline = require('readline'); const readline = require('readline');
const {google} = require('googleapis');
const {createStarterConfig, getDefaultConfigPath, writeConfigFile} = require('./config'); const {createStarterConfig, getDefaultConfigPath, writeConfigFile} = require('./config');
const {getFirstServiceAccountAuthClient} = require('./auth'); const {getFirstServiceAccountAuthClient} = require('./auth');
const {fetchSpreadsheetTitle} = require('./sheets');
function createPrompt() { function createPrompt() {
const rl = readline.createInterface({input: process.stdin, output: process.stdout}); const rl = readline.createInterface({input: process.stdin, output: process.stdout});
@@ -78,28 +80,81 @@ async function askOptionalPositiveInteger(ask, question, defaultValue) {
} }
} }
async function pauseForEnter(ask) { function getServiceAccountIdentity(inputPath, baseDir) {
await ask('Press Enter to close this window...'); const value = (inputPath || '').trim();
if (!value) {
return '';
}
const fullPath = path.isAbsolute(value) ? value : path.resolve(baseDir || process.cwd(), value);
if (!fs.existsSync(fullPath)) {
return value;
}
const raw = fs.readFileSync(fullPath, 'utf8').trim();
try {
const parsed = JSON.parse(raw);
const canonicalize = obj => {
if (obj === null || typeof obj !== 'object') return JSON.stringify(obj);
if (Array.isArray(obj)) return '[' + obj.map(canonicalize).join(',') + ']';
const keys = Object.keys(obj).sort();
return '{' + keys.map(key => JSON.stringify(key) + ':' + canonicalize(obj[key])).join(',') + '}';
};
return canonicalize(parsed);
} catch (_) {
return raw;
}
} }
async function fetchSpreadsheetTitle(context, documentId) { function getServiceAccountsFolderPath(configPath) {
const sheetOptions = {version: 'v4'}; return path.join(path.dirname(configPath), 'serviceAccounts');
if (context.authClient) { }
sheetOptions.auth = context.authClient;
function listServiceAccountFiles(folderPath) {
if (!fs.existsSync(folderPath)) {
return [];
} }
const sheets = google.sheets(sheetOptions); return fs.readdirSync(folderPath)
const request = { .map(fileName => path.join(folderPath, fileName))
spreadsheetId: documentId, .filter(filePath => {
fields: 'properties/title', try {
}; return fs.statSync(filePath).isFile();
} catch (_) {
return false;
}
});
}
if (context.apiKey) { function extractSpreadsheetId(input) {
request.key = context.apiKey; const value = (input || '').trim();
if (!value) {
return '';
} }
const response = await sheets.spreadsheets.get(request); const urlMatch = value.match(/\/spreadsheets\/d\/([a-zA-Z0-9-_]+)/i);
return response.data.properties?.title || documentId; if (urlMatch && urlMatch[1]) {
return urlMatch[1];
}
return value;
}
async function askSpreadsheetIdOrUrl(ask, question) {
while (true) {
const answer = await ask(question);
const documentId = extractSpreadsheetId(answer);
if (documentId) {
return documentId;
}
console.log('Please enter a Google Sheets URL or spreadsheet ID.');
}
}
async function pauseForEnter(ask) {
await ask('Press Enter to close this window...');
} }
function buildTitleContext(authMode, config) { function buildTitleContext(authMode, config) {
@@ -152,45 +207,97 @@ async function runOnboarding() {
] ]
); );
const credentialCount = await askPositiveInteger( if (authMode === 'apiKeys') {
ask, console.log('Warning: when using API keys, the spreadsheet must be publicly accessible.');
`How many ${authMode === 'apiKeys' ? 'API keys' : 'service accounts'} do you want to add? ` console.log('');
); }
const config = { const config = {
apiKeys: [], apiKeys: [],
serviceAccounts: [], serviceAccounts: [],
documents: [], documents: [],
}; };
Object.defineProperty(config, '__configPath', {
value: configPath,
enumerable: false,
configurable: true,
writable: true,
});
if (authMode === 'apiKeys') { if (authMode === 'apiKeys') {
const credentialCount = await askPositiveInteger(
ask,
'How many API keys do you want to add? '
);
const seenKeys = new Set();
for (let index = 0; index < credentialCount; index += 1) { for (let index = 0; index < credentialCount; index += 1) {
const key = await askNonEmpty(ask, `Enter API key ${index + 1}: `); let key;
while (true) {
key = await askNonEmpty(ask, `Enter API key ${index + 1}: `);
if (seenKeys.has(key)) {
console.log('That API key was already entered. Please enter a unique API key.');
continue;
}
break;
}
const rateLimitPerMinute = await askOptionalPositiveInteger( const rateLimitPerMinute = await askOptionalPositiveInteger(
ask, ask,
`Rate limit per minute for API key ${index + 1} (press Enter for 50): `, `Rate limit per minute for API key ${index + 1} (press Enter for 50): `,
50 50
); );
seenKeys.add(key);
config.apiKeys.push({key, rateLimitPerMinute}); config.apiKeys.push({key, rateLimitPerMinute});
} }
} else { } else {
for (let index = 0; index < credentialCount; index += 1) { const serviceAccountsFolder = getServiceAccountsFolderPath(configPath);
const pathValue = await askNonEmpty(ask, `Enter service account path ${index + 1}: `); if (!fs.existsSync(serviceAccountsFolder)) {
fs.mkdirSync(serviceAccountsFolder, {recursive: true});
}
console.log(`A serviceAccounts folder has been created at ${serviceAccountsFolder}.`);
console.log('Paste your service account files into that folder, then press Enter to continue.');
await pauseForEnter(ask);
const serviceAccountFiles = listServiceAccountFiles(serviceAccountsFolder);
console.log(`Found ${serviceAccountFiles.length} service account${serviceAccountFiles.length === 1 ? '' : 's'}.`);
if (serviceAccountFiles.length === 0) {
throw new Error(`No service account files were found in ${serviceAccountsFolder}`);
}
const seenServiceAccounts = new Set();
for (const filePath of serviceAccountFiles) {
const relativePath = path.relative(path.dirname(configPath), filePath).split(path.sep).join('/');
const serviceAccountIdentity = getServiceAccountIdentity(relativePath, path.dirname(configPath));
if (seenServiceAccounts.has(serviceAccountIdentity)) {
console.log(`Skipping duplicate service account file: ${path.basename(filePath)}`);
continue;
}
seenServiceAccounts.add(serviceAccountIdentity);
const rateLimitPerMinute = await askOptionalPositiveInteger( const rateLimitPerMinute = await askOptionalPositiveInteger(
ask, ask,
`Rate limit per minute for service account ${index + 1} (press Enter for 50): `, `Rate limit per minute for ${path.basename(filePath)} (press Enter for 50): `,
50 50
); );
config.serviceAccounts.push({path: pathValue, rateLimitPerMinute}); config.serviceAccounts.push({path: relativePath, rateLimitPerMinute});
} }
} }
const documentCount = await askPositiveInteger(ask, 'How many documents do you want to pull from? '); const documentCount = await askPositiveInteger(ask, 'How many documents do you want to pull from? ');
for (let documentIndex = 0; documentIndex < documentCount; documentIndex += 1) { for (let documentIndex = 0; documentIndex < documentCount; documentIndex += 1) {
const documentId = await askNonEmpty(ask, `Enter document ID ${documentIndex + 1}: `); const documentId = await askSpreadsheetIdOrUrl(
ask,
`Enter Google Sheets URL or document ID ${documentIndex + 1}: `
);
let documentTitle = documentId; let documentTitle = documentId;
try { try {
+149
View File
@@ -0,0 +1,149 @@
const https = require('https');
const {URL} = require('url');
function parseResponseBody(body) {
if (!body) {
return {};
}
try {
return JSON.parse(body);
} catch (_) {
return body;
}
}
function extractErrorMessage(data) {
if (!data) {
return '';
}
if (typeof data === 'string') {
return data;
}
if (data.error && typeof data.error === 'object') {
if (typeof data.error.message === 'string' && data.error.message) {
return data.error.message;
}
if (Array.isArray(data.error.errors) && data.error.errors.length > 0) {
const firstError = data.error.errors[0];
if (firstError && typeof firstError.message === 'string' && firstError.message) {
return firstError.message;
}
}
}
if (typeof data.message === 'string' && data.message) {
return data.message;
}
return '';
}
function requestJson(urlString, headers = {}) {
return new Promise((resolve, reject) => {
const url = new URL(urlString);
const request = https.request(
url,
{
method: 'GET',
headers: Object.assign({Accept: 'application/json'}, headers),
},
response => {
let body = '';
response.setEncoding('utf8');
response.on('data', chunk => {
body += chunk;
});
response.on('end', () => {
const data = parseResponseBody(body);
if (response.statusCode >= 200 && response.statusCode < 300) {
resolve({status: response.statusCode, headers: response.headers, data});
return;
}
const message = extractErrorMessage(data) || `Request failed with status ${response.statusCode}`;
const error = new Error(message);
error.response = {status: response.statusCode, headers: response.headers, data};
reject(error);
});
}
);
request.on('error', reject);
request.end();
});
}
async function requestSheetsJson(pathname, context, queryParams = {}) {
const url = new URL(`https://sheets.googleapis.com${pathname}`);
for (const [key, value] of Object.entries(queryParams)) {
if (Array.isArray(value)) {
for (const item of value) {
url.searchParams.append(key, item);
}
} else if (value !== undefined && value !== null) {
url.searchParams.set(key, String(value));
}
}
const headers = {};
if (context && context.authClient) {
if (typeof context.authClient.getRequestHeaders === 'function') {
const authHeaders = await context.authClient.getRequestHeaders(url.toString());
Object.assign(headers, authHeaders || {});
} else if (typeof context.authClient.getAccessToken === 'function') {
const token = await context.authClient.getAccessToken();
const accessToken = token && typeof token === 'object' ? token.token || token.access_token : token;
if (accessToken) {
headers.Authorization = `Bearer ${accessToken}`;
}
}
}
if (context && context.apiKey) {
url.searchParams.set('key', context.apiKey);
}
return requestJson(url.toString(), headers);
}
async function fetchSpreadsheetTitle(context, documentId) {
const response = await requestSheetsJson(
`/v4/spreadsheets/${encodeURIComponent(documentId)}`,
context,
{fields: 'properties/title'}
);
return response.data && response.data.properties && response.data.properties.title || documentId;
}
async function fetchSpreadsheetValuesBatch(context, documentId, sheetNames) {
const response = await requestSheetsJson(
`/v4/spreadsheets/${encodeURIComponent(documentId)}/values:batchGet`,
context,
{ranges: sheetNames}
);
const valueRanges = response.data.valueRanges || [];
const rowsBySheet = {};
for (let i = 0; i < sheetNames.length; i += 1) {
const sheetName = sheetNames[i];
rowsBySheet[sheetName] = valueRanges[i] && valueRanges[i].values ? valueRanges[i].values : [];
}
return rowsBySheet;
}
module.exports = {
fetchSpreadsheetTitle,
fetchSpreadsheetValuesBatch,
};
+2 -2
View File
@@ -1,12 +1,12 @@
{ {
"name": "google-sheets-rate-assistant", "name": "google-sheets-rate-assistant",
"version": "2.0.0", "version": "2.1.0",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "google-sheets-rate-assistant", "name": "google-sheets-rate-assistant",
"version": "2.0.0", "version": "2.1.0",
"dependencies": { "dependencies": {
"csv-stringify": "^6.0.0", "csv-stringify": "^6.0.0",
"googleapis": "^173.0.0", "googleapis": "^173.0.0",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "google-sheets-rate-assistant", "name": "google-sheets-rate-assistant",
"version": "2.1.0", "version": "2.1.8",
"description": "Pull Google Sheets data and export to CSV with configurable schedules and rate limiting.", "description": "Pull Google Sheets data and export to CSV with configurable schedules and rate limiting.",
"main": "index.js", "main": "index.js",
"bin": "index.js", "bin": "index.js",
+6 -4
View File
@@ -4,9 +4,10 @@ This Node.js application pulls configured Google Sheets documents and exports th
## Setup ## Setup
1. Download and run the latest version. Download the latest version from the [Releases tab](./releases) and run it.
2. If `config.json` does not exist, the app will ask whether you want a wizard or a starter config file.
3. If you choose the config-file path instead of the wizard, configure authentication in `config.json` — choose one of: 1. If `config.json` does not exist, the app will ask whether you want a wizard or a starter config file.
2. If you choose the config-file path instead of the wizard, configure authentication in `config.json` — choose one of:
- Service account: place your service account JSON in `credentials.json` and set `credentialsPath`, or add one or more entries to the `serviceAccounts` array (each entry may be a path string or an object with `path` and optional `rateLimitPerMinute`). - Service account: place your service account JSON in `credentials.json` and set `credentialsPath`, or add one or more entries to the `serviceAccounts` array (each entry may be a path string or an object with `path` and optional `rateLimitPerMinute`).
- API keys: add one or more API keys to the `apiKeys` array in `config.json` (each entry may be a string or an object with `key` and optional `rateLimitPerMinute`). - API keys: add one or more API keys to the `apiKeys` array in `config.json` (each entry may be a string or an object with `key` and optional `rateLimitPerMinute`).
@@ -70,9 +71,10 @@ To build a Windows executable from source, run:
npm run build:win npm run build:win
``` ```
That creates `dist/gSheets-Rate-Assistant-v2.0.0.exe` using the current version in the filename. For distribution, ship the exe alongside a `config.json` and any service account credential files, or set `GSA_CONFIG` to point at a different config file. That creates `dist/gSheets-Rate-Assistant-v2.1.8.exe` using the current version in the filename. For distribution, ship the exe alongside a `config.json` and any service account credential files, or set `GSA_CONFIG` to point at a different config file.
Gitea now builds only on version tags like `v1.1.1` through the workflow in `.gitea/workflows/build.yml`. Gitea now builds only on version tags like `v1.1.1` through the workflow in `.gitea/workflows/build.yml`.
If you also want the same tag to create a GitHub release, add `RELEASE_GITHUB_TOKEN` and `RELEASE_GITHUB_REPO` as secrets in Gitea so the workflow can mirror the release and upload the Windows asset to GitHub.
## AI Disclosure ## AI Disclosure