Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0d67a4eb25 | ||
|
|
f93065a377 | ||
|
|
91f6cd1aa2 | ||
|
|
e3f585222a | ||
|
|
e8e771aaf2 | ||
|
|
1a89020243 | ||
|
|
e92a3d364f | ||
|
|
1a714f6deb | ||
|
|
c4c470c073 | ||
|
|
694cc55eaa | ||
|
|
3a5cc191fe | ||
|
|
817af0e9ba | ||
|
|
3a4f1eed68 | ||
|
|
c1154a58d5 | ||
|
|
727df78b53 | ||
|
|
ba656eb679 | ||
|
|
00d0db8255 | ||
|
|
2dfe6aac03 | ||
|
|
5e6b5d37d1 |
@@ -15,6 +15,8 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@v4
|
||||||
@@ -28,6 +30,48 @@ jobs:
|
|||||||
- name: Clean dist
|
- name: Clean dist
|
||||||
run: rm -f dist/*.exe
|
run: rm -f dist/*.exe
|
||||||
|
|
||||||
|
- name: Generate release notes
|
||||||
|
run: |
|
||||||
|
python - <<'PY'
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
tag = os.environ["BUILD_VERSION"]
|
||||||
|
|
||||||
|
try:
|
||||||
|
previous_tag = subprocess.check_output(
|
||||||
|
["git", "describe", "--tags", "--abbrev=0", f"{tag}^"],
|
||||||
|
text=True,
|
||||||
|
).strip()
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
previous_tag = ""
|
||||||
|
|
||||||
|
log_range = f"{previous_tag}..{tag}" if previous_tag else tag
|
||||||
|
subjects = subprocess.check_output(
|
||||||
|
["git", "log", "--pretty=format:%s", log_range],
|
||||||
|
text=True,
|
||||||
|
).splitlines()
|
||||||
|
|
||||||
|
notes = []
|
||||||
|
for subject in subjects:
|
||||||
|
subject = subject.strip()
|
||||||
|
if not subject:
|
||||||
|
continue
|
||||||
|
if subject.startswith("chore: bump version"):
|
||||||
|
continue
|
||||||
|
if subject.lower() == "no message":
|
||||||
|
continue
|
||||||
|
notes.append(f"- {subject}")
|
||||||
|
|
||||||
|
if not notes:
|
||||||
|
notes = ["- Release build and version bump."]
|
||||||
|
|
||||||
|
body = "## What\'s new\n\n" + "\n".join(notes) + "\n"
|
||||||
|
pathlib.Path("release-notes.md").write_text(body, encoding="utf-8")
|
||||||
|
print(body)
|
||||||
|
PY
|
||||||
|
|
||||||
- name: Build Windows executable
|
- name: Build Windows executable
|
||||||
run: npm run build:win
|
run: npm run build:win
|
||||||
|
|
||||||
@@ -36,4 +80,110 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
tag_name: ${{ github.ref_name }}
|
tag_name: ${{ github.ref_name }}
|
||||||
name: gSheets-Rate-Assistant ${{ github.ref_name }}
|
name: gSheets-Rate-Assistant ${{ github.ref_name }}
|
||||||
|
body_path: release-notes.md
|
||||||
files: dist/*.exe
|
files: dist/*.exe
|
||||||
|
|
||||||
|
- name: Mirror release to GitHub
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.RELEASE_GITHUB_TOKEN }}
|
||||||
|
RELEASE_GITHUB_REPO: ${{ secrets.RELEASE_GITHUB_REPO }}
|
||||||
|
run: |
|
||||||
|
python - <<'PY'
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
token = os.environ.get("GITHUB_TOKEN", "")
|
||||||
|
repository = os.environ.get("RELEASE_GITHUB_REPO", "")
|
||||||
|
|
||||||
|
if not token or not repository:
|
||||||
|
print("Skipping GitHub mirror because RELEASE_GITHUB_REPO or RELEASE_GITHUB_TOKEN is missing.")
|
||||||
|
raise SystemExit(0)
|
||||||
|
|
||||||
|
tag = os.environ["BUILD_VERSION"]
|
||||||
|
release_name = f"gSheets-Rate-Assistant {tag}"
|
||||||
|
asset_path = next((os.path.join("dist", name) for name in os.listdir("dist") if name.endswith(".exe")), None)
|
||||||
|
with open("release-notes.md", "r", encoding="utf-8") as notes_file:
|
||||||
|
release_notes = notes_file.read()
|
||||||
|
|
||||||
|
if asset_path is None:
|
||||||
|
raise SystemExit("No Windows executable found in dist/")
|
||||||
|
|
||||||
|
def request_json(url, method="GET", payload=None, headers=None):
|
||||||
|
request_headers = {
|
||||||
|
"Authorization": f"Bearer {token}",
|
||||||
|
"Accept": "application/vnd.github+json",
|
||||||
|
"X-GitHub-Api-Version": "2022-11-28",
|
||||||
|
}
|
||||||
|
if headers:
|
||||||
|
request_headers.update(headers)
|
||||||
|
|
||||||
|
data = None if payload is None else json.dumps(payload).encode("utf-8")
|
||||||
|
if data is not None:
|
||||||
|
request_headers["Content-Type"] = "application/json"
|
||||||
|
|
||||||
|
request = urllib.request.Request(url, data=data, method=method, headers=request_headers)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request) as response:
|
||||||
|
body = response.read().decode("utf-8")
|
||||||
|
return json.loads(body) if body else {}
|
||||||
|
except urllib.error.HTTPError as error:
|
||||||
|
message = error.read().decode("utf-8", errors="replace")
|
||||||
|
print(message, file=sys.stderr)
|
||||||
|
raise
|
||||||
|
|
||||||
|
release_url = f"https://api.github.com/repos/{repository}/releases/tags/{urllib.parse.quote(tag)}"
|
||||||
|
release = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
release = request_json(release_url)
|
||||||
|
except urllib.error.HTTPError as error:
|
||||||
|
if error.code != 404:
|
||||||
|
raise
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"tag_name": tag,
|
||||||
|
"name": release_name,
|
||||||
|
"body": release_notes,
|
||||||
|
"draft": False,
|
||||||
|
"prerelease": False,
|
||||||
|
"generate_release_notes": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
if release is None:
|
||||||
|
release = request_json(f"https://api.github.com/repos/{repository}/releases", method="POST", payload=payload)
|
||||||
|
else:
|
||||||
|
release = request_json(release["url"], method="PATCH", payload=payload)
|
||||||
|
|
||||||
|
asset_name = os.path.basename(asset_path)
|
||||||
|
for asset in release.get("assets", []):
|
||||||
|
if asset.get("name") == asset_name:
|
||||||
|
request_json(asset["url"], method="DELETE")
|
||||||
|
|
||||||
|
upload_url = release["upload_url"].split("{", 1)[0]
|
||||||
|
with open(asset_path, "rb") as asset_file:
|
||||||
|
asset_data = asset_file.read()
|
||||||
|
|
||||||
|
upload_request = urllib.request.Request(
|
||||||
|
f"{upload_url}?name={urllib.parse.quote(asset_name)}",
|
||||||
|
data=asset_data,
|
||||||
|
method="POST",
|
||||||
|
headers={
|
||||||
|
"Authorization": f"Bearer {token}",
|
||||||
|
"Accept": "application/vnd.github+json",
|
||||||
|
"Content-Type": "application/octet-stream",
|
||||||
|
"X-GitHub-Api-Version": "2022-11-28",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(upload_request) as response:
|
||||||
|
response.read()
|
||||||
|
except urllib.error.HTTPError as error:
|
||||||
|
message = error.read().decode("utf-8", errors="replace")
|
||||||
|
print(message, file=sys.stderr)
|
||||||
|
raise
|
||||||
|
PY
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ dist/
|
|||||||
# Credentials and config (do NOT commit secrets)
|
# Credentials and config (do NOT commit secrets)
|
||||||
credentials.json
|
credentials.json
|
||||||
credentials-*.json
|
credentials-*.json
|
||||||
|
serviceAccounts/
|
||||||
config.json
|
config.json
|
||||||
config-*.json
|
config-*.json
|
||||||
.env
|
.env
|
||||||
|
|||||||
+132
-27
@@ -1,5 +1,7 @@
|
|||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
|
const https = require('https');
|
||||||
|
const crypto = require('crypto');
|
||||||
const {applyRateLimitBuffer} = require('./rate');
|
const {applyRateLimitBuffer} = require('./rate');
|
||||||
const {fetchSpreadsheetValuesBatch} = require('./sheets');
|
const {fetchSpreadsheetValuesBatch} = require('./sheets');
|
||||||
|
|
||||||
@@ -24,6 +26,128 @@ function resolveConfigPath(config, candidatePath) {
|
|||||||
return path.resolve(getConfigBaseDir(config), candidatePath);
|
return path.resolve(getConfigBaseDir(config), candidatePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function base64UrlEncode(value) {
|
||||||
|
return Buffer.from(value)
|
||||||
|
.toString('base64')
|
||||||
|
.replace(/=/g, '')
|
||||||
|
.replace(/\+/g, '-')
|
||||||
|
.replace(/\//g, '_');
|
||||||
|
}
|
||||||
|
|
||||||
|
function createServiceAccountAuthClient(credentials) {
|
||||||
|
const scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly'];
|
||||||
|
const tokenUri = credentials.token_uri || 'https://oauth2.googleapis.com/token';
|
||||||
|
const tokenCache = {accessToken: '', expiryDate: 0};
|
||||||
|
|
||||||
|
async function requestAccessToken() {
|
||||||
|
if (!credentials.client_email || !credentials.private_key) {
|
||||||
|
throw new Error('Service account credentials must include client_email and private_key');
|
||||||
|
}
|
||||||
|
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
const header = {alg: 'RS256', typ: 'JWT'};
|
||||||
|
const payload = {
|
||||||
|
iss: credentials.client_email,
|
||||||
|
scope: scopes.join(' '),
|
||||||
|
aud: tokenUri,
|
||||||
|
iat: now,
|
||||||
|
exp: now + 3600,
|
||||||
|
};
|
||||||
|
const unsignedToken = `${base64UrlEncode(JSON.stringify(header))}.${base64UrlEncode(JSON.stringify(payload))}`;
|
||||||
|
const signer = crypto.createSign('RSA-SHA256');
|
||||||
|
signer.update(unsignedToken);
|
||||||
|
signer.end();
|
||||||
|
const signature = signer.sign(credentials.private_key);
|
||||||
|
const assertion = `${unsignedToken}.${base64UrlEncode(signature)}`;
|
||||||
|
const body = `grant_type=${encodeURIComponent('urn:ietf:params:oauth:grant-type:jwt-bearer')}&assertion=${encodeURIComponent(assertion)}`;
|
||||||
|
|
||||||
|
const tokenUrl = new URL(tokenUri);
|
||||||
|
const requestOptions = {
|
||||||
|
protocol: tokenUrl.protocol,
|
||||||
|
hostname: tokenUrl.hostname,
|
||||||
|
port: tokenUrl.port || 443,
|
||||||
|
method: 'POST',
|
||||||
|
path: `${tokenUrl.pathname}${tokenUrl.search}`,
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/x-www-form-urlencoded',
|
||||||
|
'Content-Length': Buffer.byteLength(body),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const responseBody = await new Promise((resolve, reject) => {
|
||||||
|
const request = https.request(requestOptions, response => {
|
||||||
|
let responseText = '';
|
||||||
|
response.setEncoding('utf8');
|
||||||
|
|
||||||
|
response.on('data', chunk => {
|
||||||
|
responseText += chunk;
|
||||||
|
});
|
||||||
|
|
||||||
|
response.on('end', () => {
|
||||||
|
if (response.statusCode >= 200 && response.statusCode < 300) {
|
||||||
|
resolve(responseText);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let message = `Token request failed with status ${response.statusCode}`;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(responseText);
|
||||||
|
if (parsed && parsed.error && parsed.error_description) {
|
||||||
|
message = `${parsed.error}: ${parsed.error_description}`;
|
||||||
|
} else if (parsed && parsed.error && parsed.error.message) {
|
||||||
|
message = parsed.error.message;
|
||||||
|
}
|
||||||
|
} catch (_) {
|
||||||
|
if (responseText) {
|
||||||
|
message = responseText;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
reject(new Error(message));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
request.on('error', reject);
|
||||||
|
request.write(body);
|
||||||
|
request.end();
|
||||||
|
});
|
||||||
|
|
||||||
|
const tokenData = JSON.parse(responseBody);
|
||||||
|
if (!tokenData.access_token) {
|
||||||
|
throw new Error('Token response did not include access_token');
|
||||||
|
}
|
||||||
|
|
||||||
|
tokenCache.accessToken = tokenData.access_token;
|
||||||
|
tokenCache.expiryDate = Date.now() + Math.max(0, (tokenData.expires_in || 3600) - 60) * 1000;
|
||||||
|
return tokenData.access_token;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
scopes,
|
||||||
|
async getAccessToken() {
|
||||||
|
if (tokenCache.accessToken && Date.now() < tokenCache.expiryDate) {
|
||||||
|
return {token: tokenCache.accessToken, expiry_date: tokenCache.expiryDate};
|
||||||
|
}
|
||||||
|
|
||||||
|
const accessToken = await requestAccessToken();
|
||||||
|
return {token: accessToken, expiry_date: tokenCache.expiryDate};
|
||||||
|
},
|
||||||
|
async getRequestHeaders() {
|
||||||
|
const token = await this.getAccessToken();
|
||||||
|
return {Authorization: `Bearer ${token.token}`};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function readServiceAccountCredentials(resolvedPath) {
|
||||||
|
try {
|
||||||
|
const raw = fs.readFileSync(resolvedPath, 'utf8');
|
||||||
|
return JSON.parse(raw);
|
||||||
|
} catch (e) {
|
||||||
|
throw new Error(`Failed to read/parse service account credentials at ${resolvedPath}: ${e.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function loadApiKeys(config) {
|
function loadApiKeys(config) {
|
||||||
const apiKeys = [];
|
const apiKeys = [];
|
||||||
const defaultRateLimit = config.rateLimitPerMinute || 50;
|
const defaultRateLimit = config.rateLimitPerMinute || 50;
|
||||||
@@ -85,21 +209,8 @@ function createServiceAccountBatchFetcher(config) {
|
|||||||
if (!fs.existsSync(resolvedPath)) {
|
if (!fs.existsSync(resolvedPath)) {
|
||||||
throw new Error(`Service account credentials not found: ${resolvedPath}`);
|
throw new Error(`Service account credentials not found: ${resolvedPath}`);
|
||||||
}
|
}
|
||||||
|
const credentials = readServiceAccountCredentials(resolvedPath);
|
||||||
let credentials;
|
const authClient = createServiceAccountAuthClient(credentials);
|
||||||
try {
|
|
||||||
credentials = JSON.parse(fs.readFileSync(resolvedPath, 'utf8'));
|
|
||||||
} catch (e) {
|
|
||||||
throw new Error(`Failed to read/parse service account credentials at ${resolvedPath}: ${e.message}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
let authClient;
|
|
||||||
try {
|
|
||||||
authClient = google.auth.fromJSON(credentials);
|
|
||||||
} catch (e) {
|
|
||||||
throw new Error(`Failed to initialize auth client from credentials at ${resolvedPath}: ${e.message}`);
|
|
||||||
}
|
|
||||||
authClient.scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly'];
|
|
||||||
|
|
||||||
const bufferedRateLimit = applyRateLimitBuffer(rateLimitPerMinute);
|
const bufferedRateLimit = applyRateLimitBuffer(rateLimitPerMinute);
|
||||||
const interval = Math.ceil(60000 / bufferedRateLimit);
|
const interval = Math.ceil(60000 / bufferedRateLimit);
|
||||||
@@ -222,11 +333,8 @@ function getFirstServiceAccountAuthClient(config) {
|
|||||||
if (!fs.existsSync(resolvedPath)) return null;
|
if (!fs.existsSync(resolvedPath)) return null;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const raw = fs.readFileSync(resolvedPath, 'utf8');
|
const credentials = readServiceAccountCredentials(resolvedPath);
|
||||||
const credentials = JSON.parse(raw);
|
return createServiceAccountAuthClient(credentials);
|
||||||
const authClient = google.auth.fromJSON(credentials);
|
|
||||||
authClient.scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly'];
|
|
||||||
return authClient;
|
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error(`Failed to load first service account from ${resolvedPath}: ${e.message}`);
|
console.error(`Failed to load first service account from ${resolvedPath}: ${e.message}`);
|
||||||
return null;
|
return null;
|
||||||
@@ -242,11 +350,8 @@ function loadAuth(config) {
|
|||||||
if (!fs.existsSync(resolvedPath)) {
|
if (!fs.existsSync(resolvedPath)) {
|
||||||
throw new Error(`Credentials file not found: ${resolvedPath}`);
|
throw new Error(`Credentials file not found: ${resolvedPath}`);
|
||||||
}
|
}
|
||||||
const raw = fs.readFileSync(resolvedPath, 'utf8');
|
const credentials = readServiceAccountCredentials(resolvedPath);
|
||||||
const credentials = JSON.parse(raw);
|
return createServiceAccountAuthClient(credentials);
|
||||||
const authClient = google.auth.fromJSON(credentials);
|
|
||||||
authClient.scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly'];
|
|
||||||
return authClient;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return getFirstServiceAccountAuthClient(config);
|
return getFirstServiceAccountAuthClient(config);
|
||||||
@@ -258,6 +363,6 @@ module.exports = {
|
|||||||
createServiceAccountBatchFetcher,
|
createServiceAccountBatchFetcher,
|
||||||
createApiKeyBatchFetcher,
|
createApiKeyBatchFetcher,
|
||||||
getFirstApiKey,
|
getFirstApiKey,
|
||||||
getFirstServiceAccountAuthClient
|
getFirstServiceAccountAuthClient,
|
||||||
, loadAuth
|
loadAuth,
|
||||||
};
|
};
|
||||||
|
|||||||
+12
-1
@@ -128,6 +128,15 @@ function loadConfig() {
|
|||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getConfigBaseDir(config) {
|
||||||
|
const configPath = config && (config.__configPath || config.configPath);
|
||||||
|
if (configPath) {
|
||||||
|
return path.dirname(configPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
return process.cwd();
|
||||||
|
}
|
||||||
|
|
||||||
function resolveOutputPath(outputDir, filename) {
|
function resolveOutputPath(outputDir, filename) {
|
||||||
if (!path.isAbsolute(outputDir)) {
|
if (!path.isAbsolute(outputDir)) {
|
||||||
outputDir = path.resolve(process.cwd(), outputDir);
|
outputDir = path.resolve(process.cwd(), outputDir);
|
||||||
@@ -203,6 +212,8 @@ function validateConfig(config) {
|
|||||||
|
|
||||||
// Ensure service account credential paths are unique when using serviceAccounts
|
// Ensure service account credential paths are unique when using serviceAccounts
|
||||||
if (hasSvcAccounts) {
|
if (hasSvcAccounts) {
|
||||||
|
const baseDir = getConfigBaseDir(config);
|
||||||
|
|
||||||
for (const item of config.serviceAccounts) {
|
for (const item of config.serviceAccounts) {
|
||||||
if (typeof item === 'string') {
|
if (typeof item === 'string') {
|
||||||
if (!item.trim()) {
|
if (!item.trim()) {
|
||||||
@@ -231,7 +242,7 @@ function validateConfig(config) {
|
|||||||
if (typeof item === 'string') p = item.trim();
|
if (typeof item === 'string') p = item.trim();
|
||||||
else if (item && typeof item === 'object') p = (item.path || item.file || item.credentialsPath || '').toString().trim();
|
else if (item && typeof item === 'object') p = (item.path || item.file || item.credentialsPath || '').toString().trim();
|
||||||
if (!p) return '';
|
if (!p) return '';
|
||||||
const full = path.isAbsolute(p) ? p : path.resolve(process.cwd(), p);
|
const full = path.isAbsolute(p) ? p : path.resolve(baseDir, p);
|
||||||
if (!fs.existsSync(full)) {
|
if (!fs.existsSync(full)) {
|
||||||
throw new Error(`Service account file not found: ${p}`);
|
throw new Error(`Service account file not found: ${p}`);
|
||||||
}
|
}
|
||||||
|
|||||||
+131
-10
@@ -1,3 +1,5 @@
|
|||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
const readline = require('readline');
|
const readline = require('readline');
|
||||||
const {createStarterConfig, getDefaultConfigPath, writeConfigFile} = require('./config');
|
const {createStarterConfig, getDefaultConfigPath, writeConfigFile} = require('./config');
|
||||||
const {getFirstServiceAccountAuthClient} = require('./auth');
|
const {getFirstServiceAccountAuthClient} = require('./auth');
|
||||||
@@ -78,6 +80,79 @@ async function askOptionalPositiveInteger(ask, question, defaultValue) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getServiceAccountIdentity(inputPath, baseDir) {
|
||||||
|
const value = (inputPath || '').trim();
|
||||||
|
if (!value) {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
const fullPath = path.isAbsolute(value) ? value : path.resolve(baseDir || process.cwd(), value);
|
||||||
|
if (!fs.existsSync(fullPath)) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
const raw = fs.readFileSync(fullPath, 'utf8').trim();
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(raw);
|
||||||
|
const canonicalize = obj => {
|
||||||
|
if (obj === null || typeof obj !== 'object') return JSON.stringify(obj);
|
||||||
|
if (Array.isArray(obj)) return '[' + obj.map(canonicalize).join(',') + ']';
|
||||||
|
const keys = Object.keys(obj).sort();
|
||||||
|
return '{' + keys.map(key => JSON.stringify(key) + ':' + canonicalize(obj[key])).join(',') + '}';
|
||||||
|
};
|
||||||
|
return canonicalize(parsed);
|
||||||
|
} catch (_) {
|
||||||
|
return raw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getServiceAccountsFolderPath(configPath) {
|
||||||
|
return path.join(path.dirname(configPath), 'serviceAccounts');
|
||||||
|
}
|
||||||
|
|
||||||
|
function listServiceAccountFiles(folderPath) {
|
||||||
|
if (!fs.existsSync(folderPath)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return fs.readdirSync(folderPath)
|
||||||
|
.map(fileName => path.join(folderPath, fileName))
|
||||||
|
.filter(filePath => {
|
||||||
|
try {
|
||||||
|
return fs.statSync(filePath).isFile();
|
||||||
|
} catch (_) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractSpreadsheetId(input) {
|
||||||
|
const value = (input || '').trim();
|
||||||
|
if (!value) {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
const urlMatch = value.match(/\/spreadsheets\/d\/([a-zA-Z0-9-_]+)/i);
|
||||||
|
if (urlMatch && urlMatch[1]) {
|
||||||
|
return urlMatch[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function askSpreadsheetIdOrUrl(ask, question) {
|
||||||
|
while (true) {
|
||||||
|
const answer = await ask(question);
|
||||||
|
const documentId = extractSpreadsheetId(answer);
|
||||||
|
|
||||||
|
if (documentId) {
|
||||||
|
return documentId;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Please enter a Google Sheets URL or spreadsheet ID.');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function pauseForEnter(ask) {
|
async function pauseForEnter(ask) {
|
||||||
await ask('Press Enter to close this window...');
|
await ask('Press Enter to close this window...');
|
||||||
}
|
}
|
||||||
@@ -132,10 +207,10 @@ async function runOnboarding() {
|
|||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
const credentialCount = await askPositiveInteger(
|
if (authMode === 'apiKeys') {
|
||||||
ask,
|
console.log('Warning: when using API keys, the spreadsheet must be publicly accessible.');
|
||||||
`How many ${authMode === 'apiKeys' ? 'API keys' : 'service accounts'} do you want to add? `
|
console.log('');
|
||||||
);
|
}
|
||||||
|
|
||||||
const config = {
|
const config = {
|
||||||
apiKeys: [],
|
apiKeys: [],
|
||||||
@@ -150,33 +225,79 @@ async function runOnboarding() {
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (authMode === 'apiKeys') {
|
if (authMode === 'apiKeys') {
|
||||||
|
const credentialCount = await askPositiveInteger(
|
||||||
|
ask,
|
||||||
|
'How many API keys do you want to add? '
|
||||||
|
);
|
||||||
|
|
||||||
|
const seenKeys = new Set();
|
||||||
|
|
||||||
for (let index = 0; index < credentialCount; index += 1) {
|
for (let index = 0; index < credentialCount; index += 1) {
|
||||||
const key = await askNonEmpty(ask, `Enter API key ${index + 1}: `);
|
let key;
|
||||||
|
while (true) {
|
||||||
|
key = await askNonEmpty(ask, `Enter API key ${index + 1}: `);
|
||||||
|
if (seenKeys.has(key)) {
|
||||||
|
console.log('That API key was already entered. Please enter a unique API key.');
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
const rateLimitPerMinute = await askOptionalPositiveInteger(
|
const rateLimitPerMinute = await askOptionalPositiveInteger(
|
||||||
ask,
|
ask,
|
||||||
`Rate limit per minute for API key ${index + 1} (press Enter for 50): `,
|
`Rate limit per minute for API key ${index + 1} (press Enter for 50): `,
|
||||||
50
|
50
|
||||||
);
|
);
|
||||||
|
|
||||||
|
seenKeys.add(key);
|
||||||
config.apiKeys.push({key, rateLimitPerMinute});
|
config.apiKeys.push({key, rateLimitPerMinute});
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
for (let index = 0; index < credentialCount; index += 1) {
|
const serviceAccountsFolder = getServiceAccountsFolderPath(configPath);
|
||||||
const pathValue = await askNonEmpty(ask, `Enter service account path ${index + 1}: `);
|
if (!fs.existsSync(serviceAccountsFolder)) {
|
||||||
|
fs.mkdirSync(serviceAccountsFolder, {recursive: true});
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`A serviceAccounts folder has been created at ${serviceAccountsFolder}.`);
|
||||||
|
console.log('Paste your service account files into that folder, then press Enter to continue.');
|
||||||
|
await pauseForEnter(ask);
|
||||||
|
|
||||||
|
const serviceAccountFiles = listServiceAccountFiles(serviceAccountsFolder);
|
||||||
|
console.log(`Found ${serviceAccountFiles.length} service account${serviceAccountFiles.length === 1 ? '' : 's'}.`);
|
||||||
|
|
||||||
|
if (serviceAccountFiles.length === 0) {
|
||||||
|
throw new Error(`No service account files were found in ${serviceAccountsFolder}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const seenServiceAccounts = new Set();
|
||||||
|
|
||||||
|
for (const filePath of serviceAccountFiles) {
|
||||||
|
const relativePath = path.relative(path.dirname(configPath), filePath).split(path.sep).join('/');
|
||||||
|
const serviceAccountIdentity = getServiceAccountIdentity(relativePath, path.dirname(configPath));
|
||||||
|
|
||||||
|
if (seenServiceAccounts.has(serviceAccountIdentity)) {
|
||||||
|
console.log(`Skipping duplicate service account file: ${path.basename(filePath)}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
seenServiceAccounts.add(serviceAccountIdentity);
|
||||||
const rateLimitPerMinute = await askOptionalPositiveInteger(
|
const rateLimitPerMinute = await askOptionalPositiveInteger(
|
||||||
ask,
|
ask,
|
||||||
`Rate limit per minute for service account ${index + 1} (press Enter for 50): `,
|
`Rate limit per minute for ${path.basename(filePath)} (press Enter for 50): `,
|
||||||
50
|
50
|
||||||
);
|
);
|
||||||
|
|
||||||
config.serviceAccounts.push({path: pathValue, rateLimitPerMinute});
|
config.serviceAccounts.push({path: relativePath, rateLimitPerMinute});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const documentCount = await askPositiveInteger(ask, 'How many documents do you want to pull from? ');
|
const documentCount = await askPositiveInteger(ask, 'How many documents do you want to pull from? ');
|
||||||
|
|
||||||
for (let documentIndex = 0; documentIndex < documentCount; documentIndex += 1) {
|
for (let documentIndex = 0; documentIndex < documentCount; documentIndex += 1) {
|
||||||
const documentId = await askNonEmpty(ask, `Enter document ID ${documentIndex + 1}: `);
|
const documentId = await askSpreadsheetIdOrUrl(
|
||||||
|
ask,
|
||||||
|
`Enter Google Sheets URL or document ID ${documentIndex + 1}: `
|
||||||
|
);
|
||||||
let documentTitle = documentId;
|
let documentTitle = documentId;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|||||||
+31
-29
@@ -45,35 +45,37 @@ function extractErrorMessage(data) {
|
|||||||
function requestJson(urlString, headers = {}) {
|
function requestJson(urlString, headers = {}) {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const url = new URL(urlString);
|
const url = new URL(urlString);
|
||||||
const request = https.request(
|
const requestOptions = {
|
||||||
url,
|
protocol: url.protocol,
|
||||||
{
|
hostname: url.hostname,
|
||||||
method: 'GET',
|
port: url.port || 443,
|
||||||
headers: Object.assign({Accept: 'application/json'}, headers),
|
method: 'GET',
|
||||||
},
|
path: `${url.pathname}${url.search}`,
|
||||||
response => {
|
headers: Object.assign({Accept: 'application/json'}, headers),
|
||||||
let body = '';
|
};
|
||||||
response.setEncoding('utf8');
|
|
||||||
|
|
||||||
response.on('data', chunk => {
|
const request = https.request(requestOptions, response => {
|
||||||
body += chunk;
|
let body = '';
|
||||||
});
|
response.setEncoding('utf8');
|
||||||
|
|
||||||
response.on('end', () => {
|
response.on('data', chunk => {
|
||||||
const data = parseResponseBody(body);
|
body += chunk;
|
||||||
|
});
|
||||||
|
|
||||||
if (response.statusCode >= 200 && response.statusCode < 300) {
|
response.on('end', () => {
|
||||||
resolve({status: response.statusCode, headers: response.headers, data});
|
const data = parseResponseBody(body);
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const message = extractErrorMessage(data) || `Request failed with status ${response.statusCode}`;
|
if (response.statusCode >= 200 && response.statusCode < 300) {
|
||||||
const error = new Error(message);
|
resolve({status: response.statusCode, headers: response.headers, data});
|
||||||
error.response = {status: response.statusCode, headers: response.headers, data};
|
return;
|
||||||
reject(error);
|
}
|
||||||
});
|
|
||||||
}
|
const message = extractErrorMessage(data) || `Request failed with status ${response.statusCode}`;
|
||||||
);
|
const error = new Error(message);
|
||||||
|
error.response = {status: response.statusCode, headers: response.headers, data};
|
||||||
|
reject(error);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
request.on('error', reject);
|
request.on('error', reject);
|
||||||
request.end();
|
request.end();
|
||||||
@@ -96,15 +98,15 @@ async function requestSheetsJson(pathname, context, queryParams = {}) {
|
|||||||
const headers = {};
|
const headers = {};
|
||||||
|
|
||||||
if (context && context.authClient) {
|
if (context && context.authClient) {
|
||||||
if (typeof context.authClient.getRequestHeaders === 'function') {
|
if (typeof context.authClient.getAccessToken === 'function') {
|
||||||
const authHeaders = await context.authClient.getRequestHeaders(url.toString());
|
|
||||||
Object.assign(headers, authHeaders || {});
|
|
||||||
} else if (typeof context.authClient.getAccessToken === 'function') {
|
|
||||||
const token = await context.authClient.getAccessToken();
|
const token = await context.authClient.getAccessToken();
|
||||||
const accessToken = token && typeof token === 'object' ? token.token || token.access_token : token;
|
const accessToken = token && typeof token === 'object' ? token.token || token.access_token : token;
|
||||||
if (accessToken) {
|
if (accessToken) {
|
||||||
headers.Authorization = `Bearer ${accessToken}`;
|
headers.Authorization = `Bearer ${accessToken}`;
|
||||||
}
|
}
|
||||||
|
} else if (typeof context.authClient.getRequestHeaders === 'function') {
|
||||||
|
const authHeaders = await context.authClient.getRequestHeaders(url.toString());
|
||||||
|
Object.assign(headers, authHeaders || {});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "google-sheets-rate-assistant",
|
"name": "google-sheets-rate-assistant",
|
||||||
"version": "2.1.0",
|
"version": "2.1.10",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "google-sheets-rate-assistant",
|
"name": "google-sheets-rate-assistant",
|
||||||
"version": "2.1.0",
|
"version": "2.1.10",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"csv-stringify": "^6.0.0",
|
"csv-stringify": "^6.0.0",
|
||||||
"googleapis": "^173.0.0",
|
"googleapis": "^173.0.0",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "google-sheets-rate-assistant",
|
"name": "google-sheets-rate-assistant",
|
||||||
"version": "2.1.3",
|
"version": "2.1.10",
|
||||||
"description": "Pull Google Sheets data and export to CSV with configurable schedules and rate limiting.",
|
"description": "Pull Google Sheets data and export to CSV with configurable schedules and rate limiting.",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"bin": "index.js",
|
"bin": "index.js",
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ This Node.js application pulls configured Google Sheets documents and exports th
|
|||||||
|
|
||||||
## Setup
|
## Setup
|
||||||
|
|
||||||
Download the latest version from the [Releases tab](./releases) and run it.
|
Download the latest version from the Releases and run it.
|
||||||
|
|
||||||
1. If `config.json` does not exist, the app will ask whether you want a wizard or a starter config file.
|
1. If `config.json` does not exist, the app will ask whether you want a wizard or a starter config file.
|
||||||
2. If you choose the config-file path instead of the wizard, configure authentication in `config.json` — choose one of:
|
2. If you choose the config-file path instead of the wizard, configure authentication in `config.json` — choose one of:
|
||||||
@@ -63,18 +63,6 @@ If `config.json` is missing, the app creates a starter file next to the executab
|
|||||||
|
|
||||||
If you choose the wizard, it will ask for the auth method, credential count, document count, and sheet names. It will not ask for output directories or file names.
|
If you choose the wizard, it will ask for the auth method, credential count, document count, and sheet names. It will not ask for output directories or file names.
|
||||||
|
|
||||||
## Windows Packaging
|
|
||||||
|
|
||||||
To build a Windows executable from source, run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm run build:win
|
|
||||||
```
|
|
||||||
|
|
||||||
That creates `dist/gSheets-Rate-Assistant-v2.0.0.exe` using the current version in the filename. For distribution, ship the exe alongside a `config.json` and any service account credential files, or set `GSA_CONFIG` to point at a different config file.
|
|
||||||
|
|
||||||
Gitea now builds only on version tags like `v1.1.1` through the workflow in `.gitea/workflows/build.yml`.
|
|
||||||
|
|
||||||
## AI Disclosure
|
## AI Disclosure
|
||||||
|
|
||||||
A large chunk of the project was coded using AI, this is the first pass at integrating some AI assistance into my projects. However all code written has been verified and checked before submission. AI use has only been included since the v2 rewrite.
|
A large chunk of the project was coded using AI, this is the first pass at integrating some AI assistance into my projects. However all code written has been verified and checked before submission. AI use has only been included since the v2 rewrite.
|
||||||
|
|||||||
Reference in New Issue
Block a user