Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
54a0be4836 | ||
|
|
ea283dd490 | ||
|
|
0d67a4eb25 | ||
|
|
f93065a377 | ||
|
|
91f6cd1aa2 | ||
|
|
e3f585222a | ||
|
|
e8e771aaf2 | ||
|
|
1a89020243 | ||
|
|
e92a3d364f | ||
|
|
1a714f6deb | ||
|
|
c4c470c073 | ||
|
|
694cc55eaa | ||
|
|
3a5cc191fe | ||
|
|
817af0e9ba | ||
|
|
3a4f1eed68 | ||
|
|
c1154a58d5 | ||
|
|
727df78b53 | ||
|
|
ba656eb679 | ||
|
|
00d0db8255 | ||
|
|
2dfe6aac03 | ||
|
|
5e6b5d37d1 | ||
|
|
26702cec4f | ||
|
|
7e45c40c8a | ||
|
|
f6c289af60 |
+151
-1
@@ -15,11 +15,13 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: '24'
|
node-version: '26'
|
||||||
cache: npm
|
cache: npm
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
@@ -28,6 +30,48 @@ jobs:
|
|||||||
- name: Clean dist
|
- name: Clean dist
|
||||||
run: rm -f dist/*.exe
|
run: rm -f dist/*.exe
|
||||||
|
|
||||||
|
- name: Generate release notes
|
||||||
|
run: |
|
||||||
|
python - <<'PY'
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
tag = os.environ["BUILD_VERSION"]
|
||||||
|
|
||||||
|
try:
|
||||||
|
previous_tag = subprocess.check_output(
|
||||||
|
["git", "describe", "--tags", "--abbrev=0", f"{tag}^"],
|
||||||
|
text=True,
|
||||||
|
).strip()
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
previous_tag = ""
|
||||||
|
|
||||||
|
log_range = f"{previous_tag}..{tag}" if previous_tag else tag
|
||||||
|
subjects = subprocess.check_output(
|
||||||
|
["git", "log", "--pretty=format:%s", log_range],
|
||||||
|
text=True,
|
||||||
|
).splitlines()
|
||||||
|
|
||||||
|
notes = []
|
||||||
|
for subject in subjects:
|
||||||
|
subject = subject.strip()
|
||||||
|
if not subject:
|
||||||
|
continue
|
||||||
|
if subject.startswith("chore: bump version"):
|
||||||
|
continue
|
||||||
|
if subject.lower() == "no message":
|
||||||
|
continue
|
||||||
|
notes.append(f"- {subject}")
|
||||||
|
|
||||||
|
if not notes:
|
||||||
|
notes = ["- Release build and version bump."]
|
||||||
|
|
||||||
|
body = "## What\'s new\n\n" + "\n".join(notes) + "\n"
|
||||||
|
pathlib.Path("release-notes.md").write_text(body, encoding="utf-8")
|
||||||
|
print(body)
|
||||||
|
PY
|
||||||
|
|
||||||
- name: Build Windows executable
|
- name: Build Windows executable
|
||||||
run: npm run build:win
|
run: npm run build:win
|
||||||
|
|
||||||
@@ -36,4 +80,110 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
tag_name: ${{ github.ref_name }}
|
tag_name: ${{ github.ref_name }}
|
||||||
name: gSheets-Rate-Assistant ${{ github.ref_name }}
|
name: gSheets-Rate-Assistant ${{ github.ref_name }}
|
||||||
|
body_path: release-notes.md
|
||||||
files: dist/*.exe
|
files: dist/*.exe
|
||||||
|
|
||||||
|
- name: Mirror release to GitHub
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.RELEASE_GITHUB_TOKEN }}
|
||||||
|
RELEASE_GITHUB_REPO: ${{ secrets.RELEASE_GITHUB_REPO }}
|
||||||
|
run: |
|
||||||
|
python - <<'PY'
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
token = os.environ.get("GITHUB_TOKEN", "")
|
||||||
|
repository = os.environ.get("RELEASE_GITHUB_REPO", "")
|
||||||
|
|
||||||
|
if not token or not repository:
|
||||||
|
print("Skipping GitHub mirror because RELEASE_GITHUB_REPO or RELEASE_GITHUB_TOKEN is missing.")
|
||||||
|
raise SystemExit(0)
|
||||||
|
|
||||||
|
tag = os.environ["BUILD_VERSION"]
|
||||||
|
release_name = f"gSheets-Rate-Assistant {tag}"
|
||||||
|
asset_path = next((os.path.join("dist", name) for name in os.listdir("dist") if name.endswith(".exe")), None)
|
||||||
|
with open("release-notes.md", "r", encoding="utf-8") as notes_file:
|
||||||
|
release_notes = notes_file.read()
|
||||||
|
|
||||||
|
if asset_path is None:
|
||||||
|
raise SystemExit("No Windows executable found in dist/")
|
||||||
|
|
||||||
|
def request_json(url, method="GET", payload=None, headers=None):
|
||||||
|
request_headers = {
|
||||||
|
"Authorization": f"Bearer {token}",
|
||||||
|
"Accept": "application/vnd.github+json",
|
||||||
|
"X-GitHub-Api-Version": "2022-11-28",
|
||||||
|
}
|
||||||
|
if headers:
|
||||||
|
request_headers.update(headers)
|
||||||
|
|
||||||
|
data = None if payload is None else json.dumps(payload).encode("utf-8")
|
||||||
|
if data is not None:
|
||||||
|
request_headers["Content-Type"] = "application/json"
|
||||||
|
|
||||||
|
request = urllib.request.Request(url, data=data, method=method, headers=request_headers)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request) as response:
|
||||||
|
body = response.read().decode("utf-8")
|
||||||
|
return json.loads(body) if body else {}
|
||||||
|
except urllib.error.HTTPError as error:
|
||||||
|
message = error.read().decode("utf-8", errors="replace")
|
||||||
|
print(message, file=sys.stderr)
|
||||||
|
raise
|
||||||
|
|
||||||
|
release_url = f"https://api.github.com/repos/{repository}/releases/tags/{urllib.parse.quote(tag)}"
|
||||||
|
release = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
release = request_json(release_url)
|
||||||
|
except urllib.error.HTTPError as error:
|
||||||
|
if error.code != 404:
|
||||||
|
raise
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"tag_name": tag,
|
||||||
|
"name": release_name,
|
||||||
|
"body": release_notes,
|
||||||
|
"draft": False,
|
||||||
|
"prerelease": False,
|
||||||
|
"generate_release_notes": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
if release is None:
|
||||||
|
release = request_json(f"https://api.github.com/repos/{repository}/releases", method="POST", payload=payload)
|
||||||
|
else:
|
||||||
|
release = request_json(release["url"], method="PATCH", payload=payload)
|
||||||
|
|
||||||
|
asset_name = os.path.basename(asset_path)
|
||||||
|
for asset in release.get("assets", []):
|
||||||
|
if asset.get("name") == asset_name:
|
||||||
|
request_json(asset["url"], method="DELETE")
|
||||||
|
|
||||||
|
upload_url = release["upload_url"].split("{", 1)[0]
|
||||||
|
with open(asset_path, "rb") as asset_file:
|
||||||
|
asset_data = asset_file.read()
|
||||||
|
|
||||||
|
upload_request = urllib.request.Request(
|
||||||
|
f"{upload_url}?name={urllib.parse.quote(asset_name)}",
|
||||||
|
data=asset_data,
|
||||||
|
method="POST",
|
||||||
|
headers={
|
||||||
|
"Authorization": f"Bearer {token}",
|
||||||
|
"Accept": "application/vnd.github+json",
|
||||||
|
"Content-Type": "application/octet-stream",
|
||||||
|
"X-GitHub-Api-Version": "2022-11-28",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(upload_request) as response:
|
||||||
|
response.read()
|
||||||
|
except urllib.error.HTTPError as error:
|
||||||
|
message = error.read().decode("utf-8", errors="replace")
|
||||||
|
print(message, file=sys.stderr)
|
||||||
|
raise
|
||||||
|
PY
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ dist/
|
|||||||
# Credentials and config (do NOT commit secrets)
|
# Credentials and config (do NOT commit secrets)
|
||||||
credentials.json
|
credentials.json
|
||||||
credentials-*.json
|
credentials-*.json
|
||||||
|
serviceAccounts/
|
||||||
config.json
|
config.json
|
||||||
config-*.json
|
config-*.json
|
||||||
.env
|
.env
|
||||||
|
|||||||
@@ -5,15 +5,17 @@
|
|||||||
// === Imports & constants ===
|
// === Imports & constants ===
|
||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
const {google} = require('googleapis');
|
const readline = require('readline');
|
||||||
const stringifyModule = require('./node_modules/csv-stringify/dist/cjs/sync.cjs');
|
const stringifyModule = require('./node_modules/csv-stringify/dist/cjs/sync.cjs');
|
||||||
const stringify = typeof stringifyModule === 'function'
|
const stringify = typeof stringifyModule === 'function'
|
||||||
? stringifyModule
|
? stringifyModule
|
||||||
: stringifyModule.default || stringifyModule.stringify || stringifyModule;
|
: stringifyModule.default || stringifyModule.stringify || stringifyModule;
|
||||||
|
const {version: packageVersion} = require('./package.json');
|
||||||
const {loadConfig, resolveOutputPath} = require('./lib/config');
|
const {loadConfig, resolveOutputPath} = require('./lib/config');
|
||||||
const {buildThrottle, applyRateLimitBuffer, getTotalRateLimitPerMinute} = require('./lib/rate');
|
const {buildThrottle, applyRateLimitBuffer, getTotalRateLimitPerMinute} = require('./lib/rate');
|
||||||
const {runOnboarding} = require('./lib/onboarding');
|
const {runOnboarding} = require('./lib/onboarding');
|
||||||
const authHelpers = require('./lib/auth');
|
const authHelpers = require('./lib/auth');
|
||||||
|
const {fetchSpreadsheetTitle, fetchSpreadsheetValuesBatch} = require('./lib/sheets');
|
||||||
const {createApiKeyBatchFetcher, createServiceAccountBatchFetcher, getFirstApiKey, getFirstServiceAccountAuthClient, loadAuth} = authHelpers;
|
const {createApiKeyBatchFetcher, createServiceAccountBatchFetcher, getFirstApiKey, getFirstServiceAccountAuthClient, loadAuth} = authHelpers;
|
||||||
|
|
||||||
// === Status rendering / console display ===
|
// === Status rendering / console display ===
|
||||||
@@ -26,11 +28,36 @@ function mapDocumentToFilename(sheetName) {
|
|||||||
|
|
||||||
const statusManager = {
|
const statusManager = {
|
||||||
keys: [],
|
keys: [],
|
||||||
|
lineIndexes: new Map(),
|
||||||
statuses: new Map(),
|
statuses: new Map(),
|
||||||
lastRenderLines: 0,
|
anchorSaved: false,
|
||||||
enabled: process.stdout && process.stdout.isTTY,
|
enabled: process.stdout && process.stdout.isTTY,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function clearConsoleForRun() {
|
||||||
|
if (process.stdout && process.stdout.isTTY && typeof console.clear === 'function') {
|
||||||
|
console.clear();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (process.stdout && typeof process.stdout.write === 'function') {
|
||||||
|
process.stdout.write('\u001b[2J\u001b[0f');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function limitToTerminalWidth(text) {
|
||||||
|
const width = Math.max(20, (process.stdout && process.stdout.columns) || 80);
|
||||||
|
if (text.length <= width) {
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (width <= 1) {
|
||||||
|
return text.slice(0, width);
|
||||||
|
}
|
||||||
|
|
||||||
|
return text.slice(0, width - 1) + '…';
|
||||||
|
}
|
||||||
|
|
||||||
// === Document grouping & mapping ===
|
// === Document grouping & mapping ===
|
||||||
// Map `config.documents` entries to internal groups with resolved output paths.
|
// Map `config.documents` entries to internal groups with resolved output paths.
|
||||||
|
|
||||||
@@ -39,31 +66,54 @@ function renderStatusBlock() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (statusManager.lastRenderLines > 0) {
|
process.stdout.write('\u001b[s');
|
||||||
process.stdout.write(`\x1B[${statusManager.lastRenderLines}F`);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const key of statusManager.keys) {
|
statusManager.keys.forEach((key, index) => {
|
||||||
const statusText = statusManager.statuses.get(key) || `${key} - pending`;
|
const statusText = statusManager.statuses.get(key) || `${key} - pending`;
|
||||||
process.stdout.clearLine(0);
|
readline.cursorTo(process.stdout, 0);
|
||||||
process.stdout.cursorTo(0);
|
readline.clearLine(process.stdout, 0);
|
||||||
process.stdout.write(statusText + '\n');
|
process.stdout.write(limitToTerminalWidth(statusText));
|
||||||
|
if (index < statusManager.keys.length - 1) {
|
||||||
|
process.stdout.write('\n');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
statusManager.anchorSaved = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateStatusLine(taskId, statusText) {
|
||||||
|
if (!statusManager.enabled || !statusManager.anchorSaved) {
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
statusManager.lastRenderLines = statusManager.keys.length;
|
const lineIndex = statusManager.lineIndexes.get(taskId);
|
||||||
|
if (lineIndex === undefined) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
process.stdout.write('\u001b[u');
|
||||||
|
if (lineIndex > 0) {
|
||||||
|
readline.moveCursor(process.stdout, 0, lineIndex);
|
||||||
|
}
|
||||||
|
readline.cursorTo(process.stdout, 0);
|
||||||
|
readline.clearLine(process.stdout, 0);
|
||||||
|
process.stdout.write(limitToTerminalWidth(statusText));
|
||||||
|
process.stdout.write('\u001b[u');
|
||||||
}
|
}
|
||||||
|
|
||||||
function setTaskStatus(taskId, statusText) {
|
function setTaskStatus(taskId, statusText) {
|
||||||
statusManager.statuses.set(taskId, statusText);
|
statusManager.statuses.set(taskId, statusText);
|
||||||
renderStatusBlock();
|
updateStatusLine(taskId, statusText);
|
||||||
}
|
}
|
||||||
|
|
||||||
function initializeTaskStatuses(tasks) {
|
function initializeTaskStatuses(tasks) {
|
||||||
statusManager.keys = tasks.map(task => `${task.documentId}:${task.sheetName}`);
|
statusManager.keys = tasks.map(task => `${task.documentId}:${task.sheetName}`);
|
||||||
|
statusManager.lineIndexes.clear();
|
||||||
statusManager.statuses.clear();
|
statusManager.statuses.clear();
|
||||||
statusManager.lastRenderLines = 0;
|
statusManager.anchorSaved = false;
|
||||||
|
|
||||||
for (const key of statusManager.keys) {
|
for (const [index, key] of statusManager.keys.entries()) {
|
||||||
|
statusManager.lineIndexes.set(key, index);
|
||||||
statusManager.statuses.set(key, `${key} - pending`);
|
statusManager.statuses.set(key, `${key} - pending`);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,26 +123,6 @@ function initializeTaskStatuses(tasks) {
|
|||||||
// Authentication helpers: see `./lib/auth` for credential loaders and
|
// Authentication helpers: see `./lib/auth` for credential loaders and
|
||||||
// per-credential, throttled batch fetchers (API key and service account modes).
|
// per-credential, throttled batch fetchers (API key and service account modes).
|
||||||
|
|
||||||
async function fetchSpreadsheetTitle(context, documentId) {
|
|
||||||
const sheetOptions = {version: 'v4'};
|
|
||||||
if (context.authClient) {
|
|
||||||
sheetOptions.auth = context.authClient;
|
|
||||||
}
|
|
||||||
|
|
||||||
const sheets = google.sheets(sheetOptions);
|
|
||||||
const request = {
|
|
||||||
spreadsheetId: documentId,
|
|
||||||
fields: 'properties/title',
|
|
||||||
};
|
|
||||||
|
|
||||||
if (context.apiKey) {
|
|
||||||
request.key = context.apiKey;
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await sheets.spreadsheets.get(request);
|
|
||||||
return response.data.properties?.title || documentId;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildDocumentGroups(config) {
|
function buildDocumentGroups(config) {
|
||||||
const groups = [];
|
const groups = [];
|
||||||
|
|
||||||
@@ -132,31 +162,7 @@ function buildDocumentGroups(config) {
|
|||||||
// `index.js` provides a simple `batchFetchDocumentSheets` fallback used when
|
// `index.js` provides a simple `batchFetchDocumentSheets` fallback used when
|
||||||
// no per-credential batch fetcher is configured by `./lib/auth`.
|
// no per-credential batch fetcher is configured by `./lib/auth`.
|
||||||
async function batchFetchDocumentSheets(context, documentId, sheetNames) {
|
async function batchFetchDocumentSheets(context, documentId, sheetNames) {
|
||||||
const sheetOptions = {version: 'v4'};
|
return fetchSpreadsheetValuesBatch(context, documentId, sheetNames);
|
||||||
if (context.authClient) {
|
|
||||||
sheetOptions.auth = context.authClient;
|
|
||||||
}
|
|
||||||
|
|
||||||
const sheets = google.sheets(sheetOptions);
|
|
||||||
const request = {
|
|
||||||
spreadsheetId: documentId,
|
|
||||||
ranges: sheetNames,
|
|
||||||
};
|
|
||||||
|
|
||||||
if (context.apiKey) {
|
|
||||||
request.key = context.apiKey;
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await sheets.spreadsheets.values.batchGet(request);
|
|
||||||
const valueRanges = response.data.valueRanges || [];
|
|
||||||
const rowsBySheet = {};
|
|
||||||
|
|
||||||
for (let i = 0; i < sheetNames.length; i += 1) {
|
|
||||||
const sheetName = sheetNames[i];
|
|
||||||
rowsBySheet[sheetName] = valueRanges[i]?.values || [];
|
|
||||||
}
|
|
||||||
|
|
||||||
return rowsBySheet;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatStatusTimestamp(date = new Date()) {
|
function formatStatusTimestamp(date = new Date()) {
|
||||||
@@ -331,6 +337,8 @@ async function scheduleRuns(config) {
|
|||||||
throw new Error('No sheets configured to pull in config.json');
|
throw new Error('No sheets configured to pull in config.json');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
console.log(`Google Sheets Rate Assistant v${packageVersion}`);
|
||||||
|
|
||||||
const firstApiKey = getFirstApiKey(config);
|
const firstApiKey = getFirstApiKey(config);
|
||||||
const titleMap = {};
|
const titleMap = {};
|
||||||
|
|
||||||
@@ -393,7 +401,8 @@ async function scheduleRuns(config) {
|
|||||||
|
|
||||||
const totalRateLimit = getTotalRateLimitPerMinute(config);
|
const totalRateLimit = getTotalRateLimitPerMinute(config);
|
||||||
const intervalMs = Math.max(1, Math.ceil(60000 / totalRateLimit));
|
const intervalMs = Math.max(1, Math.ceil(60000 / totalRateLimit));
|
||||||
console.log(`Configured for ${totalRateLimit} requests per minute across ${groups.length} documents (${intervalMs}ms interval)`);
|
console.log(limitToTerminalWidth(`Configured for ${totalRateLimit} requests per minute across ${groups.length} documents (${intervalMs}ms interval)`));
|
||||||
|
console.log('');
|
||||||
initializeTaskStatuses(tasks);
|
initializeTaskStatuses(tasks);
|
||||||
|
|
||||||
let currentIndex = 0;
|
let currentIndex = 0;
|
||||||
@@ -429,6 +438,8 @@ if (require.main === module) {
|
|||||||
|
|
||||||
(async () => {
|
(async () => {
|
||||||
try {
|
try {
|
||||||
|
clearConsoleForRun();
|
||||||
|
|
||||||
let config;
|
let config;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|||||||
+159
-51
@@ -1,7 +1,152 @@
|
|||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
const {google} = require('googleapis');
|
const https = require('https');
|
||||||
|
const crypto = require('crypto');
|
||||||
const {applyRateLimitBuffer} = require('./rate');
|
const {applyRateLimitBuffer} = require('./rate');
|
||||||
|
const {fetchSpreadsheetValuesBatch} = require('./sheets');
|
||||||
|
|
||||||
|
function getConfigBaseDir(config) {
|
||||||
|
const configPath = config && (config.__configPath || config.configPath);
|
||||||
|
if (configPath) {
|
||||||
|
return path.dirname(configPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
return process.cwd();
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveConfigPath(config, candidatePath) {
|
||||||
|
if (!candidatePath) {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (path.isAbsolute(candidatePath)) {
|
||||||
|
return candidatePath;
|
||||||
|
}
|
||||||
|
|
||||||
|
return path.resolve(getConfigBaseDir(config), candidatePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
function base64UrlEncode(value) {
|
||||||
|
return Buffer.from(value)
|
||||||
|
.toString('base64')
|
||||||
|
.replace(/=/g, '')
|
||||||
|
.replace(/\+/g, '-')
|
||||||
|
.replace(/\//g, '_');
|
||||||
|
}
|
||||||
|
|
||||||
|
function createServiceAccountAuthClient(credentials) {
|
||||||
|
const scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly'];
|
||||||
|
const tokenUri = credentials.token_uri || 'https://oauth2.googleapis.com/token';
|
||||||
|
const tokenCache = {accessToken: '', expiryDate: 0};
|
||||||
|
|
||||||
|
async function requestAccessToken() {
|
||||||
|
if (!credentials.client_email || !credentials.private_key) {
|
||||||
|
throw new Error('Service account credentials must include client_email and private_key');
|
||||||
|
}
|
||||||
|
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
const header = {alg: 'RS256', typ: 'JWT'};
|
||||||
|
const payload = {
|
||||||
|
iss: credentials.client_email,
|
||||||
|
scope: scopes.join(' '),
|
||||||
|
aud: tokenUri,
|
||||||
|
iat: now,
|
||||||
|
exp: now + 3600,
|
||||||
|
};
|
||||||
|
const unsignedToken = `${base64UrlEncode(JSON.stringify(header))}.${base64UrlEncode(JSON.stringify(payload))}`;
|
||||||
|
const signer = crypto.createSign('RSA-SHA256');
|
||||||
|
signer.update(unsignedToken);
|
||||||
|
signer.end();
|
||||||
|
const signature = signer.sign(credentials.private_key);
|
||||||
|
const assertion = `${unsignedToken}.${base64UrlEncode(signature)}`;
|
||||||
|
const body = `grant_type=${encodeURIComponent('urn:ietf:params:oauth:grant-type:jwt-bearer')}&assertion=${encodeURIComponent(assertion)}`;
|
||||||
|
|
||||||
|
const tokenUrl = new URL(tokenUri);
|
||||||
|
const requestOptions = {
|
||||||
|
protocol: tokenUrl.protocol,
|
||||||
|
hostname: tokenUrl.hostname,
|
||||||
|
port: tokenUrl.port || 443,
|
||||||
|
method: 'POST',
|
||||||
|
path: `${tokenUrl.pathname}${tokenUrl.search}`,
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/x-www-form-urlencoded',
|
||||||
|
'Content-Length': Buffer.byteLength(body),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const responseBody = await new Promise((resolve, reject) => {
|
||||||
|
const request = https.request(requestOptions, response => {
|
||||||
|
let responseText = '';
|
||||||
|
response.setEncoding('utf8');
|
||||||
|
|
||||||
|
response.on('data', chunk => {
|
||||||
|
responseText += chunk;
|
||||||
|
});
|
||||||
|
|
||||||
|
response.on('end', () => {
|
||||||
|
if (response.statusCode >= 200 && response.statusCode < 300) {
|
||||||
|
resolve(responseText);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let message = `Token request failed with status ${response.statusCode}`;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(responseText);
|
||||||
|
if (parsed && parsed.error && parsed.error_description) {
|
||||||
|
message = `${parsed.error}: ${parsed.error_description}`;
|
||||||
|
} else if (parsed && parsed.error && parsed.error.message) {
|
||||||
|
message = parsed.error.message;
|
||||||
|
}
|
||||||
|
} catch (_) {
|
||||||
|
if (responseText) {
|
||||||
|
message = responseText;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
reject(new Error(message));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
request.on('error', reject);
|
||||||
|
request.write(body);
|
||||||
|
request.end();
|
||||||
|
});
|
||||||
|
|
||||||
|
const tokenData = JSON.parse(responseBody);
|
||||||
|
if (!tokenData.access_token) {
|
||||||
|
throw new Error('Token response did not include access_token');
|
||||||
|
}
|
||||||
|
|
||||||
|
tokenCache.accessToken = tokenData.access_token;
|
||||||
|
tokenCache.expiryDate = Date.now() + Math.max(0, (tokenData.expires_in || 3600) - 60) * 1000;
|
||||||
|
return tokenData.access_token;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
scopes,
|
||||||
|
async getAccessToken() {
|
||||||
|
if (tokenCache.accessToken && Date.now() < tokenCache.expiryDate) {
|
||||||
|
return {token: tokenCache.accessToken, expiry_date: tokenCache.expiryDate};
|
||||||
|
}
|
||||||
|
|
||||||
|
const accessToken = await requestAccessToken();
|
||||||
|
return {token: accessToken, expiry_date: tokenCache.expiryDate};
|
||||||
|
},
|
||||||
|
async getRequestHeaders() {
|
||||||
|
const token = await this.getAccessToken();
|
||||||
|
return {Authorization: `Bearer ${token.token}`};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function readServiceAccountCredentials(resolvedPath) {
|
||||||
|
try {
|
||||||
|
const raw = fs.readFileSync(resolvedPath, 'utf8');
|
||||||
|
return JSON.parse(raw);
|
||||||
|
} catch (e) {
|
||||||
|
throw new Error(`Failed to read/parse service account credentials at ${resolvedPath}: ${e.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function loadApiKeys(config) {
|
function loadApiKeys(config) {
|
||||||
const apiKeys = [];
|
const apiKeys = [];
|
||||||
@@ -60,40 +205,18 @@ function createServiceAccountBatchFetcher(config) {
|
|||||||
// selecting a credential whose slot is available now. If none are
|
// selecting a credential whose slot is available now. If none are
|
||||||
// immediately available, wait the minimum required time.
|
// immediately available, wait the minimum required time.
|
||||||
const fetchers = accounts.map(({path: credentialsPath, rateLimitPerMinute}) => {
|
const fetchers = accounts.map(({path: credentialsPath, rateLimitPerMinute}) => {
|
||||||
const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath);
|
const resolvedPath = resolveConfigPath(config, credentialsPath);
|
||||||
if (!fs.existsSync(resolvedPath)) {
|
if (!fs.existsSync(resolvedPath)) {
|
||||||
throw new Error(`Service account credentials not found: ${resolvedPath}`);
|
throw new Error(`Service account credentials not found: ${resolvedPath}`);
|
||||||
}
|
}
|
||||||
|
const credentials = readServiceAccountCredentials(resolvedPath);
|
||||||
let credentials;
|
const authClient = createServiceAccountAuthClient(credentials);
|
||||||
try {
|
|
||||||
credentials = JSON.parse(fs.readFileSync(resolvedPath, 'utf8'));
|
|
||||||
} catch (e) {
|
|
||||||
throw new Error(`Failed to read/parse service account credentials at ${resolvedPath}: ${e.message}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
let authClient;
|
|
||||||
try {
|
|
||||||
authClient = google.auth.fromJSON(credentials);
|
|
||||||
} catch (e) {
|
|
||||||
throw new Error(`Failed to initialize auth client from credentials at ${resolvedPath}: ${e.message}`);
|
|
||||||
}
|
|
||||||
authClient.scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly'];
|
|
||||||
|
|
||||||
const bufferedRateLimit = applyRateLimitBuffer(rateLimitPerMinute);
|
const bufferedRateLimit = applyRateLimitBuffer(rateLimitPerMinute);
|
||||||
const interval = Math.ceil(60000 / bufferedRateLimit);
|
const interval = Math.ceil(60000 / bufferedRateLimit);
|
||||||
|
|
||||||
const fetchRaw = async (documentId, sheetNames) => {
|
const fetchRaw = async (documentId, sheetNames) => {
|
||||||
const sheets = google.sheets({version: 'v4', auth: authClient});
|
return fetchSpreadsheetValuesBatch({authClient}, documentId, sheetNames);
|
||||||
const request = {spreadsheetId: documentId, ranges: sheetNames};
|
|
||||||
const response = await sheets.spreadsheets.values.batchGet(request);
|
|
||||||
const valueRanges = response.data.valueRanges || [];
|
|
||||||
const rowsBySheet = {};
|
|
||||||
for (let i = 0; i < sheetNames.length; i += 1) {
|
|
||||||
const sheetName = sheetNames[i];
|
|
||||||
rowsBySheet[sheetName] = valueRanges[i]?.values || [];
|
|
||||||
}
|
|
||||||
return rowsBySheet;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
return {authClient, fetchRaw, interval, nextAvailable: 0};
|
return {authClient, fetchRaw, interval, nextAvailable: 0};
|
||||||
@@ -148,16 +271,7 @@ function createApiKeyBatchFetcher(config) {
|
|||||||
const interval = Math.ceil(60000 / bufferedRateLimit);
|
const interval = Math.ceil(60000 / bufferedRateLimit);
|
||||||
|
|
||||||
const fetchRaw = async (documentId, sheetNames) => {
|
const fetchRaw = async (documentId, sheetNames) => {
|
||||||
const sheets = google.sheets({version: 'v4'});
|
return fetchSpreadsheetValuesBatch({apiKey: key}, documentId, sheetNames);
|
||||||
const request = {spreadsheetId: documentId, ranges: sheetNames, key};
|
|
||||||
const response = await sheets.spreadsheets.values.batchGet(request);
|
|
||||||
const valueRanges = response.data.valueRanges || [];
|
|
||||||
const rowsBySheet = {};
|
|
||||||
for (let i = 0; i < sheetNames.length; i += 1) {
|
|
||||||
const sheetName = sheetNames[i];
|
|
||||||
rowsBySheet[sheetName] = valueRanges[i]?.values || [];
|
|
||||||
}
|
|
||||||
return rowsBySheet;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
return {key, fetchRaw, interval, nextAvailable: 0};
|
return {key, fetchRaw, interval, nextAvailable: 0};
|
||||||
@@ -215,15 +329,12 @@ function getFirstServiceAccountAuthClient(config) {
|
|||||||
const credentialsPath = entryObj.path || entryObj.credentialsPath;
|
const credentialsPath = entryObj.path || entryObj.credentialsPath;
|
||||||
if (!credentialsPath) return null;
|
if (!credentialsPath) return null;
|
||||||
|
|
||||||
const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath);
|
const resolvedPath = resolveConfigPath(config, credentialsPath);
|
||||||
if (!fs.existsSync(resolvedPath)) return null;
|
if (!fs.existsSync(resolvedPath)) return null;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const raw = fs.readFileSync(resolvedPath, 'utf8');
|
const credentials = readServiceAccountCredentials(resolvedPath);
|
||||||
const credentials = JSON.parse(raw);
|
return createServiceAccountAuthClient(credentials);
|
||||||
const authClient = google.auth.fromJSON(credentials);
|
|
||||||
authClient.scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly'];
|
|
||||||
return authClient;
|
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error(`Failed to load first service account from ${resolvedPath}: ${e.message}`);
|
console.error(`Failed to load first service account from ${resolvedPath}: ${e.message}`);
|
||||||
return null;
|
return null;
|
||||||
@@ -235,15 +346,12 @@ function loadAuth(config) {
|
|||||||
|
|
||||||
if (config.credentialsPath) {
|
if (config.credentialsPath) {
|
||||||
const credentialsPath = config.credentialsPath;
|
const credentialsPath = config.credentialsPath;
|
||||||
const resolvedPath = path.isAbsolute(credentialsPath) ? credentialsPath : path.resolve(process.cwd(), credentialsPath);
|
const resolvedPath = resolveConfigPath(config, credentialsPath);
|
||||||
if (!fs.existsSync(resolvedPath)) {
|
if (!fs.existsSync(resolvedPath)) {
|
||||||
throw new Error(`Credentials file not found: ${resolvedPath}`);
|
throw new Error(`Credentials file not found: ${resolvedPath}`);
|
||||||
}
|
}
|
||||||
const raw = fs.readFileSync(resolvedPath, 'utf8');
|
const credentials = readServiceAccountCredentials(resolvedPath);
|
||||||
const credentials = JSON.parse(raw);
|
return createServiceAccountAuthClient(credentials);
|
||||||
const authClient = google.auth.fromJSON(credentials);
|
|
||||||
authClient.scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly'];
|
|
||||||
return authClient;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return getFirstServiceAccountAuthClient(config);
|
return getFirstServiceAccountAuthClient(config);
|
||||||
@@ -255,6 +363,6 @@ module.exports = {
|
|||||||
createServiceAccountBatchFetcher,
|
createServiceAccountBatchFetcher,
|
||||||
createApiKeyBatchFetcher,
|
createApiKeyBatchFetcher,
|
||||||
getFirstApiKey,
|
getFirstApiKey,
|
||||||
getFirstServiceAccountAuthClient
|
getFirstServiceAccountAuthClient,
|
||||||
, loadAuth
|
loadAuth,
|
||||||
};
|
};
|
||||||
|
|||||||
+28
-3
@@ -1,6 +1,14 @@
|
|||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
|
|
||||||
|
let isSingleExecutableApp = false;
|
||||||
|
try {
|
||||||
|
const sea = require('node:sea');
|
||||||
|
isSingleExecutableApp = typeof sea.isSea === 'function' && sea.isSea();
|
||||||
|
} catch {
|
||||||
|
isSingleExecutableApp = false;
|
||||||
|
}
|
||||||
|
|
||||||
const CONFIG_FILE_NAME = 'config.json';
|
const CONFIG_FILE_NAME = 'config.json';
|
||||||
const DEFAULT_CONFIG_TEMPLATE = JSON.stringify({
|
const DEFAULT_CONFIG_TEMPLATE = JSON.stringify({
|
||||||
apiKeys: [
|
apiKeys: [
|
||||||
@@ -56,12 +64,12 @@ const DEFAULT_CONFIG_TEMPLATE = JSON.stringify({
|
|||||||
}, null, 2) + '\n';
|
}, null, 2) + '\n';
|
||||||
|
|
||||||
function getDefaultConfigPath() {
|
function getDefaultConfigPath() {
|
||||||
const packagedApp = Boolean(process.pkg);
|
const packagedApp = isSingleExecutableApp;
|
||||||
return path.resolve(packagedApp ? path.dirname(process.execPath) : process.cwd(), CONFIG_FILE_NAME);
|
return path.resolve(packagedApp ? path.dirname(process.execPath) : process.cwd(), CONFIG_FILE_NAME);
|
||||||
}
|
}
|
||||||
|
|
||||||
function getConfigPathCandidates() {
|
function getConfigPathCandidates() {
|
||||||
const packagedApp = Boolean(process.pkg);
|
const packagedApp = isSingleExecutableApp;
|
||||||
const externalDefault = getDefaultConfigPath();
|
const externalDefault = getDefaultConfigPath();
|
||||||
|
|
||||||
return [
|
return [
|
||||||
@@ -118,10 +126,25 @@ function loadConfig() {
|
|||||||
|
|
||||||
const json = fs.readFileSync(configPath, 'utf8');
|
const json = fs.readFileSync(configPath, 'utf8');
|
||||||
const config = JSON.parse(json);
|
const config = JSON.parse(json);
|
||||||
|
Object.defineProperty(config, '__configPath', {
|
||||||
|
value: configPath,
|
||||||
|
enumerable: false,
|
||||||
|
configurable: true,
|
||||||
|
writable: true,
|
||||||
|
});
|
||||||
validateConfig(config);
|
validateConfig(config);
|
||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getConfigBaseDir(config) {
|
||||||
|
const configPath = config && (config.__configPath || config.configPath);
|
||||||
|
if (configPath) {
|
||||||
|
return path.dirname(configPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
return process.cwd();
|
||||||
|
}
|
||||||
|
|
||||||
function resolveOutputPath(outputDir, filename) {
|
function resolveOutputPath(outputDir, filename) {
|
||||||
if (!path.isAbsolute(outputDir)) {
|
if (!path.isAbsolute(outputDir)) {
|
||||||
outputDir = path.resolve(process.cwd(), outputDir);
|
outputDir = path.resolve(process.cwd(), outputDir);
|
||||||
@@ -197,6 +220,8 @@ function validateConfig(config) {
|
|||||||
|
|
||||||
// Ensure service account credential paths are unique when using serviceAccounts
|
// Ensure service account credential paths are unique when using serviceAccounts
|
||||||
if (hasSvcAccounts) {
|
if (hasSvcAccounts) {
|
||||||
|
const baseDir = getConfigBaseDir(config);
|
||||||
|
|
||||||
for (const item of config.serviceAccounts) {
|
for (const item of config.serviceAccounts) {
|
||||||
if (typeof item === 'string') {
|
if (typeof item === 'string') {
|
||||||
if (!item.trim()) {
|
if (!item.trim()) {
|
||||||
@@ -225,7 +250,7 @@ function validateConfig(config) {
|
|||||||
if (typeof item === 'string') p = item.trim();
|
if (typeof item === 'string') p = item.trim();
|
||||||
else if (item && typeof item === 'object') p = (item.path || item.file || item.credentialsPath || '').toString().trim();
|
else if (item && typeof item === 'object') p = (item.path || item.file || item.credentialsPath || '').toString().trim();
|
||||||
if (!p) return '';
|
if (!p) return '';
|
||||||
const full = path.isAbsolute(p) ? p : path.resolve(process.cwd(), p);
|
const full = path.isAbsolute(p) ? p : path.resolve(baseDir, p);
|
||||||
if (!fs.existsSync(full)) {
|
if (!fs.existsSync(full)) {
|
||||||
throw new Error(`Service account file not found: ${p}`);
|
throw new Error(`Service account file not found: ${p}`);
|
||||||
}
|
}
|
||||||
|
|||||||
+133
-26
@@ -1,7 +1,9 @@
|
|||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
const readline = require('readline');
|
const readline = require('readline');
|
||||||
const {google} = require('googleapis');
|
|
||||||
const {createStarterConfig, getDefaultConfigPath, writeConfigFile} = require('./config');
|
const {createStarterConfig, getDefaultConfigPath, writeConfigFile} = require('./config');
|
||||||
const {getFirstServiceAccountAuthClient} = require('./auth');
|
const {getFirstServiceAccountAuthClient} = require('./auth');
|
||||||
|
const {fetchSpreadsheetTitle} = require('./sheets');
|
||||||
|
|
||||||
function createPrompt() {
|
function createPrompt() {
|
||||||
const rl = readline.createInterface({input: process.stdin, output: process.stdout});
|
const rl = readline.createInterface({input: process.stdin, output: process.stdout});
|
||||||
@@ -78,28 +80,81 @@ async function askOptionalPositiveInteger(ask, question, defaultValue) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function pauseForEnter(ask) {
|
function getServiceAccountIdentity(inputPath, baseDir) {
|
||||||
await ask('Press Enter to close this window...');
|
const value = (inputPath || '').trim();
|
||||||
|
if (!value) {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
const fullPath = path.isAbsolute(value) ? value : path.resolve(baseDir || process.cwd(), value);
|
||||||
|
if (!fs.existsSync(fullPath)) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
const raw = fs.readFileSync(fullPath, 'utf8').trim();
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(raw);
|
||||||
|
const canonicalize = obj => {
|
||||||
|
if (obj === null || typeof obj !== 'object') return JSON.stringify(obj);
|
||||||
|
if (Array.isArray(obj)) return '[' + obj.map(canonicalize).join(',') + ']';
|
||||||
|
const keys = Object.keys(obj).sort();
|
||||||
|
return '{' + keys.map(key => JSON.stringify(key) + ':' + canonicalize(obj[key])).join(',') + '}';
|
||||||
|
};
|
||||||
|
return canonicalize(parsed);
|
||||||
|
} catch (_) {
|
||||||
|
return raw;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchSpreadsheetTitle(context, documentId) {
|
function getServiceAccountsFolderPath(configPath) {
|
||||||
const sheetOptions = {version: 'v4'};
|
return path.join(path.dirname(configPath), 'serviceAccounts');
|
||||||
if (context.authClient) {
|
}
|
||||||
sheetOptions.auth = context.authClient;
|
|
||||||
|
function listServiceAccountFiles(folderPath) {
|
||||||
|
if (!fs.existsSync(folderPath)) {
|
||||||
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
const sheets = google.sheets(sheetOptions);
|
return fs.readdirSync(folderPath)
|
||||||
const request = {
|
.map(fileName => path.join(folderPath, fileName))
|
||||||
spreadsheetId: documentId,
|
.filter(filePath => {
|
||||||
fields: 'properties/title',
|
try {
|
||||||
};
|
return fs.statSync(filePath).isFile();
|
||||||
|
} catch (_) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
if (context.apiKey) {
|
function extractSpreadsheetId(input) {
|
||||||
request.key = context.apiKey;
|
const value = (input || '').trim();
|
||||||
|
if (!value) {
|
||||||
|
return '';
|
||||||
}
|
}
|
||||||
|
|
||||||
const response = await sheets.spreadsheets.get(request);
|
const urlMatch = value.match(/\/spreadsheets\/d\/([a-zA-Z0-9-_]+)/i);
|
||||||
return response.data.properties?.title || documentId;
|
if (urlMatch && urlMatch[1]) {
|
||||||
|
return urlMatch[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function askSpreadsheetIdOrUrl(ask, question) {
|
||||||
|
while (true) {
|
||||||
|
const answer = await ask(question);
|
||||||
|
const documentId = extractSpreadsheetId(answer);
|
||||||
|
|
||||||
|
if (documentId) {
|
||||||
|
return documentId;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Please enter a Google Sheets URL or spreadsheet ID.');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function pauseForEnter(ask) {
|
||||||
|
await ask('Press Enter to close this window...');
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildTitleContext(authMode, config) {
|
function buildTitleContext(authMode, config) {
|
||||||
@@ -152,45 +207,97 @@ async function runOnboarding() {
|
|||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
const credentialCount = await askPositiveInteger(
|
if (authMode === 'apiKeys') {
|
||||||
ask,
|
console.log('Warning: when using API keys, the spreadsheet must be publicly accessible.');
|
||||||
`How many ${authMode === 'apiKeys' ? 'API keys' : 'service accounts'} do you want to add? `
|
console.log('');
|
||||||
);
|
}
|
||||||
|
|
||||||
const config = {
|
const config = {
|
||||||
apiKeys: [],
|
apiKeys: [],
|
||||||
serviceAccounts: [],
|
serviceAccounts: [],
|
||||||
documents: [],
|
documents: [],
|
||||||
};
|
};
|
||||||
|
Object.defineProperty(config, '__configPath', {
|
||||||
|
value: configPath,
|
||||||
|
enumerable: false,
|
||||||
|
configurable: true,
|
||||||
|
writable: true,
|
||||||
|
});
|
||||||
|
|
||||||
if (authMode === 'apiKeys') {
|
if (authMode === 'apiKeys') {
|
||||||
|
const credentialCount = await askPositiveInteger(
|
||||||
|
ask,
|
||||||
|
'How many API keys do you want to add? '
|
||||||
|
);
|
||||||
|
|
||||||
|
const seenKeys = new Set();
|
||||||
|
|
||||||
for (let index = 0; index < credentialCount; index += 1) {
|
for (let index = 0; index < credentialCount; index += 1) {
|
||||||
const key = await askNonEmpty(ask, `Enter API key ${index + 1}: `);
|
let key;
|
||||||
|
while (true) {
|
||||||
|
key = await askNonEmpty(ask, `Enter API key ${index + 1}: `);
|
||||||
|
if (seenKeys.has(key)) {
|
||||||
|
console.log('That API key was already entered. Please enter a unique API key.');
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
const rateLimitPerMinute = await askOptionalPositiveInteger(
|
const rateLimitPerMinute = await askOptionalPositiveInteger(
|
||||||
ask,
|
ask,
|
||||||
`Rate limit per minute for API key ${index + 1} (press Enter for 50): `,
|
`Rate limit per minute for API key ${index + 1} (press Enter for 50): `,
|
||||||
50
|
50
|
||||||
);
|
);
|
||||||
|
|
||||||
|
seenKeys.add(key);
|
||||||
config.apiKeys.push({key, rateLimitPerMinute});
|
config.apiKeys.push({key, rateLimitPerMinute});
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
for (let index = 0; index < credentialCount; index += 1) {
|
const serviceAccountsFolder = getServiceAccountsFolderPath(configPath);
|
||||||
const pathValue = await askNonEmpty(ask, `Enter service account path ${index + 1}: `);
|
if (!fs.existsSync(serviceAccountsFolder)) {
|
||||||
|
fs.mkdirSync(serviceAccountsFolder, {recursive: true});
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`A serviceAccounts folder has been created at ${serviceAccountsFolder}.`);
|
||||||
|
console.log('Paste your service account files into that folder, then press Enter to continue.');
|
||||||
|
await pauseForEnter(ask);
|
||||||
|
|
||||||
|
const serviceAccountFiles = listServiceAccountFiles(serviceAccountsFolder);
|
||||||
|
console.log(`Found ${serviceAccountFiles.length} service account${serviceAccountFiles.length === 1 ? '' : 's'}.`);
|
||||||
|
|
||||||
|
if (serviceAccountFiles.length === 0) {
|
||||||
|
throw new Error(`No service account files were found in ${serviceAccountsFolder}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const seenServiceAccounts = new Set();
|
||||||
|
|
||||||
|
for (const filePath of serviceAccountFiles) {
|
||||||
|
const relativePath = path.relative(path.dirname(configPath), filePath).split(path.sep).join('/');
|
||||||
|
const serviceAccountIdentity = getServiceAccountIdentity(relativePath, path.dirname(configPath));
|
||||||
|
|
||||||
|
if (seenServiceAccounts.has(serviceAccountIdentity)) {
|
||||||
|
console.log(`Skipping duplicate service account file: ${path.basename(filePath)}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
seenServiceAccounts.add(serviceAccountIdentity);
|
||||||
const rateLimitPerMinute = await askOptionalPositiveInteger(
|
const rateLimitPerMinute = await askOptionalPositiveInteger(
|
||||||
ask,
|
ask,
|
||||||
`Rate limit per minute for service account ${index + 1} (press Enter for 50): `,
|
`Rate limit per minute for ${path.basename(filePath)} (press Enter for 50): `,
|
||||||
50
|
50
|
||||||
);
|
);
|
||||||
|
|
||||||
config.serviceAccounts.push({path: pathValue, rateLimitPerMinute});
|
config.serviceAccounts.push({path: relativePath, rateLimitPerMinute});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const documentCount = await askPositiveInteger(ask, 'How many documents do you want to pull from? ');
|
const documentCount = await askPositiveInteger(ask, 'How many documents do you want to pull from? ');
|
||||||
|
|
||||||
for (let documentIndex = 0; documentIndex < documentCount; documentIndex += 1) {
|
for (let documentIndex = 0; documentIndex < documentCount; documentIndex += 1) {
|
||||||
const documentId = await askNonEmpty(ask, `Enter document ID ${documentIndex + 1}: `);
|
const documentId = await askSpreadsheetIdOrUrl(
|
||||||
|
ask,
|
||||||
|
`Enter Google Sheets URL or document ID ${documentIndex + 1}: `
|
||||||
|
);
|
||||||
let documentTitle = documentId;
|
let documentTitle = documentId;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|||||||
+151
@@ -0,0 +1,151 @@
|
|||||||
|
const https = require('https');
|
||||||
|
const {URL} = require('url');
|
||||||
|
|
||||||
|
function parseResponseBody(body) {
|
||||||
|
if (!body) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return JSON.parse(body);
|
||||||
|
} catch (_) {
|
||||||
|
return body;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractErrorMessage(data) {
|
||||||
|
if (!data) {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof data === 'string') {
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (data.error && typeof data.error === 'object') {
|
||||||
|
if (typeof data.error.message === 'string' && data.error.message) {
|
||||||
|
return data.error.message;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Array.isArray(data.error.errors) && data.error.errors.length > 0) {
|
||||||
|
const firstError = data.error.errors[0];
|
||||||
|
if (firstError && typeof firstError.message === 'string' && firstError.message) {
|
||||||
|
return firstError.message;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof data.message === 'string' && data.message) {
|
||||||
|
return data.message;
|
||||||
|
}
|
||||||
|
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
function requestJson(urlString, headers = {}) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const url = new URL(urlString);
|
||||||
|
const requestOptions = {
|
||||||
|
protocol: url.protocol,
|
||||||
|
hostname: url.hostname,
|
||||||
|
port: url.port || 443,
|
||||||
|
method: 'GET',
|
||||||
|
path: `${url.pathname}${url.search}`,
|
||||||
|
headers: Object.assign({Accept: 'application/json'}, headers),
|
||||||
|
};
|
||||||
|
|
||||||
|
const request = https.request(requestOptions, response => {
|
||||||
|
let body = '';
|
||||||
|
response.setEncoding('utf8');
|
||||||
|
|
||||||
|
response.on('data', chunk => {
|
||||||
|
body += chunk;
|
||||||
|
});
|
||||||
|
|
||||||
|
response.on('end', () => {
|
||||||
|
const data = parseResponseBody(body);
|
||||||
|
|
||||||
|
if (response.statusCode >= 200 && response.statusCode < 300) {
|
||||||
|
resolve({status: response.statusCode, headers: response.headers, data});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const message = extractErrorMessage(data) || `Request failed with status ${response.statusCode}`;
|
||||||
|
const error = new Error(message);
|
||||||
|
error.response = {status: response.statusCode, headers: response.headers, data};
|
||||||
|
reject(error);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
request.on('error', reject);
|
||||||
|
request.end();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestSheetsJson(pathname, context, queryParams = {}) {
|
||||||
|
const url = new URL(`https://sheets.googleapis.com${pathname}`);
|
||||||
|
|
||||||
|
for (const [key, value] of Object.entries(queryParams)) {
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
for (const item of value) {
|
||||||
|
url.searchParams.append(key, item);
|
||||||
|
}
|
||||||
|
} else if (value !== undefined && value !== null) {
|
||||||
|
url.searchParams.set(key, String(value));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const headers = {};
|
||||||
|
|
||||||
|
if (context && context.authClient) {
|
||||||
|
if (typeof context.authClient.getAccessToken === 'function') {
|
||||||
|
const token = await context.authClient.getAccessToken();
|
||||||
|
const accessToken = token && typeof token === 'object' ? token.token || token.access_token : token;
|
||||||
|
if (accessToken) {
|
||||||
|
headers.Authorization = `Bearer ${accessToken}`;
|
||||||
|
}
|
||||||
|
} else if (typeof context.authClient.getRequestHeaders === 'function') {
|
||||||
|
const authHeaders = await context.authClient.getRequestHeaders(url.toString());
|
||||||
|
Object.assign(headers, authHeaders || {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (context && context.apiKey) {
|
||||||
|
url.searchParams.set('key', context.apiKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
return requestJson(url.toString(), headers);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchSpreadsheetTitle(context, documentId) {
|
||||||
|
const response = await requestSheetsJson(
|
||||||
|
`/v4/spreadsheets/${encodeURIComponent(documentId)}`,
|
||||||
|
context,
|
||||||
|
{fields: 'properties/title'}
|
||||||
|
);
|
||||||
|
|
||||||
|
return response.data && response.data.properties && response.data.properties.title || documentId;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchSpreadsheetValuesBatch(context, documentId, sheetNames) {
|
||||||
|
const response = await requestSheetsJson(
|
||||||
|
`/v4/spreadsheets/${encodeURIComponent(documentId)}/values:batchGet`,
|
||||||
|
context,
|
||||||
|
{ranges: sheetNames}
|
||||||
|
);
|
||||||
|
|
||||||
|
const valueRanges = response.data.valueRanges || [];
|
||||||
|
const rowsBySheet = {};
|
||||||
|
|
||||||
|
for (let i = 0; i < sheetNames.length; i += 1) {
|
||||||
|
const sheetName = sheetNames[i];
|
||||||
|
rowsBySheet[sheetName] = valueRanges[i] && valueRanges[i].values ? valueRanges[i].values : [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return rowsBySheet;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
fetchSpreadsheetTitle,
|
||||||
|
fetchSpreadsheetValuesBatch,
|
||||||
|
};
|
||||||
Generated
+459
-1522
File diff suppressed because it is too large
Load Diff
+2
-7
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "google-sheets-rate-assistant",
|
"name": "google-sheets-rate-assistant",
|
||||||
"version": "2.1.0",
|
"version": "2.1.11",
|
||||||
"description": "Pull Google Sheets data and export to CSV with configurable schedules and rate limiting.",
|
"description": "Pull Google Sheets data and export to CSV with configurable schedules and rate limiting.",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"bin": "index.js",
|
"bin": "index.js",
|
||||||
@@ -14,11 +14,6 @@
|
|||||||
"p-throttle": "^8.1.0"
|
"p-throttle": "^8.1.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"pkg": "^5.8.1"
|
"esbuild": "^0.25.9"
|
||||||
},
|
|
||||||
"pkg": {
|
|
||||||
"assets": [
|
|
||||||
"node_modules/csv-stringify/dist/cjs/sync.cjs"
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,9 +4,10 @@ This Node.js application pulls configured Google Sheets documents and exports th
|
|||||||
|
|
||||||
## Setup
|
## Setup
|
||||||
|
|
||||||
1. Download and run the latest version.
|
Download the latest version from the Releases and run it.
|
||||||
2. If `config.json` does not exist, the app will ask whether you want a wizard or a starter config file.
|
|
||||||
3. If you choose the config-file path instead of the wizard, configure authentication in `config.json` — choose one of:
|
1. If `config.json` does not exist, the app will ask whether you want a wizard or a starter config file.
|
||||||
|
2. If you choose the config-file path instead of the wizard, configure authentication in `config.json` — choose one of:
|
||||||
|
|
||||||
- Service account: place your service account JSON in `credentials.json` and set `credentialsPath`, or add one or more entries to the `serviceAccounts` array (each entry may be a path string or an object with `path` and optional `rateLimitPerMinute`).
|
- Service account: place your service account JSON in `credentials.json` and set `credentialsPath`, or add one or more entries to the `serviceAccounts` array (each entry may be a path string or an object with `path` and optional `rateLimitPerMinute`).
|
||||||
- API keys: add one or more API keys to the `apiKeys` array in `config.json` (each entry may be a string or an object with `key` and optional `rateLimitPerMinute`).
|
- API keys: add one or more API keys to the `apiKeys` array in `config.json` (each entry may be a string or an object with `key` and optional `rateLimitPerMinute`).
|
||||||
@@ -62,17 +63,9 @@ If `config.json` is missing, the app creates a starter file next to the executab
|
|||||||
|
|
||||||
If you choose the wizard, it will ask for the auth method, credential count, document count, and sheet names. It will not ask for output directories or file names.
|
If you choose the wizard, it will ask for the auth method, credential count, document count, and sheet names. It will not ask for output directories or file names.
|
||||||
|
|
||||||
## Windows Packaging
|
## Build Note
|
||||||
|
|
||||||
To build a Windows executable from source, run:
|
The Windows packaging script in `scripts/build-win.js` now builds a single executable using Node 26's SEA workflow.
|
||||||
|
|
||||||
```bash
|
|
||||||
npm run build:win
|
|
||||||
```
|
|
||||||
|
|
||||||
That creates `dist/gSheets-Rate-Assistant-v2.0.0.exe` using the current version in the filename. For distribution, ship the exe alongside a `config.json` and any service account credential files, or set `GSA_CONFIG` to point at a different config file.
|
|
||||||
|
|
||||||
Gitea now builds only on version tags like `v1.1.1` through the workflow in `.gitea/workflows/build.yml`.
|
|
||||||
|
|
||||||
## AI Disclosure
|
## AI Disclosure
|
||||||
|
|
||||||
|
|||||||
+28
-2
@@ -1,6 +1,8 @@
|
|||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
|
const os = require('os');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
const {exec} = require('pkg');
|
const {execFileSync} = require('child_process');
|
||||||
|
const esbuild = require('esbuild');
|
||||||
const {version: packageVersion} = require('../package.json');
|
const {version: packageVersion} = require('../package.json');
|
||||||
|
|
||||||
function normalizeBuildVersion(value) {
|
function normalizeBuildVersion(value) {
|
||||||
@@ -14,9 +16,33 @@ async function main() {
|
|||||||
fs.mkdirSync(distDir, {recursive: true});
|
fs.mkdirSync(distDir, {recursive: true});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsa-sea-'));
|
||||||
|
const bundlePath = path.join(tempDir, 'index.cjs');
|
||||||
|
const seaBlobPath = path.join(tempDir, 'sea-prep.blob');
|
||||||
|
const seaConfigPath = path.join(tempDir, 'sea-config.json');
|
||||||
const buildVersion = normalizeBuildVersion(process.env.BUILD_VERSION);
|
const buildVersion = normalizeBuildVersion(process.env.BUILD_VERSION);
|
||||||
const outputFile = path.join(distDir, `gSheets-Rate-Assistant-${buildVersion}.exe`);
|
const outputFile = path.join(distDir, `gSheets-Rate-Assistant-${buildVersion}.exe`);
|
||||||
await exec(['.', '--targets', 'latest-win-x64', '--output', outputFile]);
|
|
||||||
|
await esbuild.build({
|
||||||
|
entryPoints: [path.resolve(__dirname, '..', 'index.js')],
|
||||||
|
bundle: true,
|
||||||
|
platform: 'node',
|
||||||
|
format: 'cjs',
|
||||||
|
target: 'node26',
|
||||||
|
outfile: bundlePath,
|
||||||
|
logLevel: 'info',
|
||||||
|
});
|
||||||
|
|
||||||
|
fs.writeFileSync(seaConfigPath, JSON.stringify({
|
||||||
|
main: bundlePath,
|
||||||
|
output: outputFile,
|
||||||
|
mainFormat: 'commonjs',
|
||||||
|
disableExperimentalSEAWarning: true,
|
||||||
|
useCodeCache: false,
|
||||||
|
useSnapshot: false,
|
||||||
|
}, null, 2));
|
||||||
|
|
||||||
|
execFileSync(process.execPath, ['--build-sea', seaConfigPath], {stdio: 'inherit'});
|
||||||
}
|
}
|
||||||
|
|
||||||
main().catch(error => {
|
main().catch(error => {
|
||||||
|
|||||||
Reference in New Issue
Block a user