124 lines
3.3 KiB
TypeScript
124 lines
3.3 KiB
TypeScript
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import * as https from 'https';
|
|
|
|
interface GitHubFile {
|
|
name: string;
|
|
download_url: string;
|
|
type: string;
|
|
}
|
|
|
|
export class GitHubDownloader {
|
|
private apiBase = 'https://api.github.com/repos/csound/manual/contents';
|
|
private branch = 'develop';
|
|
private downloadDir: string;
|
|
|
|
constructor(downloadDir: string) {
|
|
this.downloadDir = downloadDir;
|
|
}
|
|
|
|
private async fetchJson<T>(url: string): Promise<T> {
|
|
return new Promise((resolve, reject) => {
|
|
https.get(
|
|
url,
|
|
{
|
|
headers: {
|
|
'User-Agent': 'Csound-Parser',
|
|
},
|
|
},
|
|
(res) => {
|
|
let data = '';
|
|
|
|
res.on('data', (chunk) => {
|
|
data += chunk;
|
|
});
|
|
|
|
res.on('end', () => {
|
|
if (res.statusCode === 200) {
|
|
try {
|
|
resolve(JSON.parse(data));
|
|
} catch (err) {
|
|
reject(new Error(`Failed to parse JSON: ${err}`));
|
|
}
|
|
} else {
|
|
reject(new Error(`HTTP ${res.statusCode}: ${data}`));
|
|
}
|
|
});
|
|
}
|
|
).on('error', reject);
|
|
});
|
|
}
|
|
|
|
private async downloadFile(url: string, destPath: string): Promise<void> {
|
|
return new Promise((resolve, reject) => {
|
|
https.get(url, (res) => {
|
|
if (res.statusCode === 302 || res.statusCode === 301) {
|
|
if (res.headers.location) {
|
|
this.downloadFile(res.headers.location, destPath).then(resolve).catch(reject);
|
|
return;
|
|
}
|
|
}
|
|
|
|
const fileStream = fs.createWriteStream(destPath);
|
|
res.pipe(fileStream);
|
|
|
|
fileStream.on('finish', () => {
|
|
fileStream.close();
|
|
resolve();
|
|
});
|
|
|
|
fileStream.on('error', (err) => {
|
|
fs.unlink(destPath, () => {});
|
|
reject(err);
|
|
});
|
|
}).on('error', reject);
|
|
});
|
|
}
|
|
|
|
async downloadOpcodes(): Promise<number> {
|
|
console.log('Fetching opcode list from GitHub...');
|
|
|
|
const url = `${this.apiBase}/docs/opcodes?ref=${this.branch}`;
|
|
const files = await this.fetchJson<GitHubFile[]>(url);
|
|
|
|
const markdownFiles = files.filter(f => f.type === 'file' && f.name.endsWith('.md'));
|
|
|
|
console.log(`Found ${markdownFiles.length} markdown files`);
|
|
|
|
if (!fs.existsSync(this.downloadDir)) {
|
|
fs.mkdirSync(this.downloadDir, { recursive: true });
|
|
}
|
|
|
|
let downloaded = 0;
|
|
let failed = 0;
|
|
|
|
for (const file of markdownFiles) {
|
|
const destPath = path.join(this.downloadDir, file.name);
|
|
|
|
try {
|
|
await this.downloadFile(file.download_url, destPath);
|
|
downloaded++;
|
|
|
|
if (downloaded % 50 === 0) {
|
|
console.log(`Downloaded ${downloaded}/${markdownFiles.length}...`);
|
|
}
|
|
} catch (err) {
|
|
console.error(`Failed to download ${file.name}:`, err);
|
|
failed++;
|
|
}
|
|
}
|
|
|
|
console.log(`\nDownloaded ${downloaded} files, ${failed} failed`);
|
|
return downloaded;
|
|
}
|
|
|
|
async downloadCategories(): Promise<void> {
|
|
console.log('Downloading categories.py...');
|
|
const url = 'https://raw.githubusercontent.com/csound/manual/develop/categories.py';
|
|
const destPath = path.join(this.downloadDir, 'categories.py');
|
|
|
|
await this.downloadFile(url, destPath);
|
|
console.log('Categories file downloaded');
|
|
}
|
|
}
|