From e61c631fc5ec279be8a9d5129773406a3e2f6e5b Mon Sep 17 00:00:00 2001 From: fscarmen2 Date: Mon, 16 Dec 2024 08:19:57 +0000 Subject: [PATCH 01/13] Read all accounts from Worker. No need config.yml any more. --- README.md | 19 ++- cloudflare_worker/github_gitlab.js | 111 +++--------------- cloudflare_worker/github_gitlab_r2.js | 106 +++-------------- cloudflare_worker/github_gitlab_s3.js | 106 +++-------------- .../.github/workflows/cluster_sync.yml | 8 +- github_to_gitlab/config.yml | 33 ------ gitlab_to_github/config.yml | 33 ------ .../.github/workflows/r2_to_github.yml | 8 +- r2_to_github/config.yml | 33 ------ .../.github/workflows/s3_to_github.yml | 8 +- s3_to_github/config.yml | 48 -------- 11 files changed, 80 insertions(+), 433 deletions(-) delete mode 100644 github_to_gitlab/config.yml delete mode 100644 gitlab_to_github/config.yml delete mode 100644 r2_to_github/config.yml delete mode 100644 s3_to_github/config.yml diff --git a/README.md b/README.md index fbe3bb2..8d79294 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,26 @@ - **CI/CD 脚本**: `.gitlab-ci.yml` ## R2 --→ GitHub 目录下2个文件，放到 GitHub 库: -- **配置文件**: `config.yml` - **AC 脚本**: `./github/workflows/r2_to_github.yml` +- **设置3个secrets**: `ACCOUNT_ID`, `WORKER_NAME` 和 `API_TOKEN` ## S2 (R2+B2) --→ GitHub 目录下2个文件，放到 GitHub 库: -- **配置文件**: `config.yml` - **AC 脚本**: `./github/workflows/s3_to_github.yml` +- **设置3个secrets**: `ACCOUNT_ID`, `WORKER_NAME` 和 `API_TOKEN` + +## ACCOUNT_ID,WORKER_NAME,API_TOKEN 获取方式 + +### 在 Cloudflare 面板创建可以读取 Worker 项目的 API, https://dash.cloudflare.com/profile/api-tokens + +![image](https://github.com/user-attachments/assets/9e49b29a-54ae-46f0-aeda-28d95f4a9041) +![image](https://github.com/user-attachments/assets/11dceb4b-ab2e-41a8-b8e4-7317bcf4b50f) +![image](https://github.com/user-attachments/assets/b1e6f1c3-3d8d-4ba3-8d98-35ab4f061b14) +![image](https://github.com/user-attachments/assets/81e66642-cd5c-43d3-bb72-7fecf24e16a3) +![image](https://github.com/user-attachments/assets/3c832e81-bfc6-480d-939c-1d0731a07c17) + +### 在 Action 处设置 3 个 secret 变量 + +![image](https://github.com/user-attachments/assets/25b8d0fa-8302-4cb9-a6db-83e449e9664c) ## Cloudflare worker 目录下5个文件，复制代码到 worker 处： - **只使用 GitHub**: `github_only.js` @@ -26,7 +40,6 @@ - **同时使用 GitHub, GitLab 和 R2**: `github_gitlab_r2.js` - **同时使用 GitHub, GitLab，R2 和 B2**: `github_gitlab_s3.js` - ## 检测节点状态 `https://<自定义域名>/` image diff --git a/cloudflare_worker/github_gitlab.js b/cloudflare_worker/github_gitlab.js index f63738e..3e3fb16 100644 --- a/cloudflare_worker/github_gitlab.js +++ b/cloudflare_worker/github_gitlab.js @@ -8,9 +8,6 @@ const GITLAB_CONFIGS = [ { name: 'repo4', id: 'repoID4', token: 'repoToken4' }, // 节点4配置 ]; -// GitHub 仓库列表，留空表示与 GitLab 的仓库名称一致 -const GITHUB_REPOS = ['']; // 如果需要指定 GitHub 仓库名，可以在此处填写。如果 GitHub 节点与 GitLab 不一样，可以用 ['ghRepo1','ghRepo2','ghRepo3','ghRepo4'] - // GitHub 用户名和个人访问令牌（PAT） const GITHUB_USERNAME = ''; // 填写 GitHub 用户名 const GITHUB_PAT = ''; // 填写 GitHub 访问令牌 @@ -24,6 +21,10 @@ const CHECK_PASSWORD = '' || GITHUB_PAT; // 定义全局缓存时间，单位为秒，默认值为一年（31556952秒） const CACHE_MAX_AGE = 31556952; // 一年 +// GitHub 备份策略 +const STRATEGY = 'size' // 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 +const DELETE = 'true' // 可选 [true (默认) | false]，已复制到 GitHub 的文件，是否从 R2 删除 + // 用户配置区域结束 ================================= // Cloudflare Worker 主函数，处理 HTTP 请求 @@ -46,15 +47,8 @@ export default { } } - // 验证 GitHub 仓库列表是否有效 - const isValidGithubRepos = Array.isArray(GITHUB_REPOS) && - GITHUB_REPOS.length > 0 && - GITHUB_REPOS.some(repo => repo.trim() !== ''); - - // 如果 GitHub 仓库列表无效，使用 GitLab 的仓库名称 - const githubRepos = isValidGithubRepos - ? GITHUB_REPOS.filter(repo => repo.trim() !== '') - : GITLAB_CONFIGS.map(config => config.name); + // 直接使用 GITLAB_CONFIGS 中的 name 作为 GitHub 仓库名 + const githubRepos = GITLAB_CONFIGS.map(config => config.name); const FILE = url.pathname.split('/').pop(); // 获取请求文件名 @@ -286,6 +280,7 @@ export default { // 列出所有节点仓库状态 async function listProjects(gitlabConfigs, githubRepos, githubUsername, githubPat) { + let result = 'GitHub and GitLab Nodes status:\n\n'; try { @@ -317,8 +312,8 @@ async function listProjects(gitlabConfigs, githubRepos, githubUsername, githubPa // 添加 GitLab 结果 gitlabConfigs.forEach((config, index) => { - const [status, username, fileCount, totalSize] = gitlabResults[index]; - result += `GitLab: Project ID ${config.id} - ${status} (Username: ${username}, Files: ${fileCount}, Size: ${totalSize})\n`; + const [status, username, fileCount] = gitlabResults[index]; + result += `GitLab: Project ID ${config.id} - ${status} (Username: ${username}, Files: ${fileCount})\n`; }); } catch (error) { @@ -420,67 +415,10 @@ async function checkGitHubRepo(owner, repo, pat) { } } -// 获取单个文件大小的辅助函数 -async function getFileSizeFromGitLab(projectId, filePath, pat, retries = 3) { - for (let i = 0; i < retries; i++) { - try { - // 使用 raw 端点和 HEAD 请求获取文件大小 - const fileUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/files${filePath}/raw?ref=main`; - const response = await fetch(fileUrl, { - method: 'HEAD', - headers: { 'PRIVATE-TOKEN': pat } - }); - - if (response.status === 200) { - const contentLength = response.headers.get('content-length'); - return contentLength ? parseInt(contentLength, 10) : 0; - } else if (response.status === 429) { - // 遇到限流时等待后重试 - await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1))); - continue; - } - console.error(`Failed to get file size: ${response.status}`); - return 0; - } catch (error) { - if (i === retries - 1) { - console.error(`Error fetching file size:`, error); - } - if (i < retries - 1) { - await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1))); - continue; - } - } - return 0; - } - return 0; -} - -// 添加并发控制的辅助函数 -async function asyncPool(concurrency, iterable, iteratorFn) { - const ret = []; - const executing = new Set(); - - for (const item of iterable) { - const p = Promise.resolve().then(() => iteratorFn(item)); - ret.push(p); - executing.add(p); - - const clean = () => executing.delete(p); - p.then(clean).catch(clean); - - if (executing.size >= concurrency) { - await Promise.race(executing); - } - } - - return Promise.all(ret); -} - // 检查 GitLab 项目的异步函数 async function checkGitLabProject(projectId, pat) { const projectUrl = `https://gitlab.com/api/v4/projects/${projectId}`; - // 步骤1: 获取文件列表 - const filesUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?ref=main&path=${DIR}`; + const filesUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?ref=main&path=${DIR}&recursive=true&per_page=10000`; try { const [projectResponse, filesResponse] = await Promise.all([ @@ -494,44 +432,25 @@ async function checkGitLabProject(projectId, pat) { if (projectResponse.status === 200) { const projectData = await projectResponse.json(); - let totalSize = 0; let fileCount = 0; if (filesResponse.status === 200) { const filesData = await filesResponse.json(); - fileCount = filesData.length; - - if (fileCount > 0) { - console.log(`Found ${fileCount} files in ${DIR} directory`); - - // 步骤2: 并发获取每个文件的大小 - const CONCURRENT_REQUESTS = 100; - const sizes = await asyncPool(CONCURRENT_REQUESTS, filesData, async (file) => { - // 构造文件路径，格式为 /files%2Ffilename - const encodedPath = `/${encodeURIComponent(file.path)}`; - const size = await getFileSizeFromGitLab(projectId, encodedPath, pat); - console.log(`File: ${file.path}, Size: ${formatSize(size)}`); - return size; - }); - - totalSize = sizes.reduce((acc, size) => acc + size, 0); - console.log(`Total size: ${formatSize(totalSize)}`); - } + fileCount = filesData.filter(item => item.type === 'blob').length; } return [ `working (${projectData.visibility})`, projectData.owner.username, - fileCount, - formatSize(totalSize) + fileCount ]; } else if (projectResponse.status === 404) { - return ['not found', 'Unknown', 0, '0 B']; + return ['not found', 'Unknown', 0]; } else { - return ['disconnect', 'Unknown', 0, '0 B']; + return ['disconnect', 'Unknown', 0]; } } catch (error) { console.error('GitLab project check error:', error); - return ['disconnect', 'Error', 0, '0 B']; + return ['disconnect', 'Error', 0]; } } \ No newline at end of file diff --git a/cloudflare_worker/github_gitlab_r2.js b/cloudflare_worker/github_gitlab_r2.js index bb3b56f..aa6e9cf 100644 --- a/cloudflare_worker/github_gitlab_r2.js +++ b/cloudflare_worker/github_gitlab_r2.js @@ -9,7 +9,6 @@ const GITLAB_CONFIGS = [ ]; // GitHub 配置 -const GITHUB_REPOS = ['']; // GitHub 仓库名列表 const GITHUB_USERNAME = ''; // GitHub 用户名 const GITHUB_PAT = ''; // GitHub 个人访问令牌 @@ -38,6 +37,10 @@ const DIR = ''; // 定义集群里全部节点连接状态的密码验证，区分大小写（优先使用自定义密码，若为空则使用 GITHUB_PAT） const CHECK_PASSWORD = '' || GITHUB_PAT; +// GitHub 备份策略 +const STRATEGY = 'size' // 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 +const DELETE = 'true' // 可选 [true (默认) | false]，已复制到 GitHub 的文件，是否从 R2 删除 + // 用户配置区域结束 ================================= // AWS SDK 签名相关函数开始 ================================= @@ -156,13 +159,8 @@ export default { } } - const isValidGithubRepos = Array.isArray(GITHUB_REPOS) && - GITHUB_REPOS.length > 0 && - GITHUB_REPOS.some(repo => repo.trim() !== ''); - - const githubRepos = isValidGithubRepos - ? GITHUB_REPOS.filter(repo => repo.trim() !== '') - : GITLAB_CONFIGS.map(config => config.name); + // 直接使用 GITLAB_CONFIGS 中的 name 作为 GitHub 仓库名 + const githubRepos = GITLAB_CONFIGS.map(config => config.name); const FILE = url.pathname.split('/').pop(); @@ -457,8 +455,8 @@ async function listProjects(gitlabConfigs, githubRepos, githubUsername, githubPa // 添加 GitLab 结果 gitlabConfigs.forEach((config, index) => { - const [status, username, fileCount, totalSize] = gitlabResults[index]; - result += `GitLab: Project ID ${config.id} - ${status} (Username: ${username}, Files: ${fileCount}, Size: ${totalSize})\n`; + const [status, username, fileCount] = gitlabResults[index]; + result += `GitLab: Project ID ${config.id} - ${status} (Username: ${username}, Files: ${fileCount})\n`; }); // 添加 R2 结果 @@ -565,67 +563,10 @@ async function checkGitHubRepo(owner, repo, pat) { } } -// 获取单个文件大小的辅助函数 -async function getFileSizeFromGitLab(projectId, filePath, pat, retries = 3) { - for (let i = 0; i < retries; i++) { - try { - // 使用 raw 端点和 HEAD 请求获取文件大小 - const fileUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/files${filePath}/raw?ref=main`; - const response = await fetch(fileUrl, { - method: 'HEAD', - headers: { 'PRIVATE-TOKEN': pat } - }); - - if (response.status === 200) { - const contentLength = response.headers.get('content-length'); - return contentLength ? parseInt(contentLength, 10) : 0; - } else if (response.status === 429) { - // 遇到限流时等待后重试 - await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1))); - continue; - } - console.error(`Failed to get file size: ${response.status}`); - return 0; - } catch (error) { - if (i === retries - 1) { - console.error(`Error fetching file size:`, error); - } - if (i < retries - 1) { - await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1))); - continue; - } - } - return 0; - } - return 0; -} - -// 添加并发控制的辅助函数 -async function asyncPool(concurrency, iterable, iteratorFn) { - const ret = []; - const executing = new Set(); - - for (const item of iterable) { - const p = Promise.resolve().then(() => iteratorFn(item)); - ret.push(p); - executing.add(p); - - const clean = () => executing.delete(p); - p.then(clean).catch(clean); - - if (executing.size >= concurrency) { - await Promise.race(executing); - } - } - - return Promise.all(ret); -} - // 检查 GitLab 项目的异步函数 async function checkGitLabProject(projectId, pat) { const projectUrl = `https://gitlab.com/api/v4/projects/${projectId}`; - // 步骤1: 获取文件列表 - const filesUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?ref=main&path=${DIR}`; + const filesUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?ref=main&path=${DIR}&recursive=true&per_page=10000`; try { const [projectResponse, filesResponse] = await Promise.all([ @@ -639,45 +580,26 @@ async function checkGitLabProject(projectId, pat) { if (projectResponse.status === 200) { const projectData = await projectResponse.json(); - let totalSize = 0; let fileCount = 0; if (filesResponse.status === 200) { const filesData = await filesResponse.json(); - fileCount = filesData.length; - - if (fileCount > 0) { - console.log(`Found ${fileCount} files in ${DIR} directory`); - - // 步骤2: 并发获取每个文件的大小 - const CONCURRENT_REQUESTS = 100; - const sizes = await asyncPool(CONCURRENT_REQUESTS, filesData, async (file) => { - // 构造文件路径，格式为 /files%2Ffilename - const encodedPath = `/${encodeURIComponent(file.path)}`; - const size = await getFileSizeFromGitLab(projectId, encodedPath, pat); - console.log(`File: ${file.path}, Size: ${formatSize(size)}`); - return size; - }); - - totalSize = sizes.reduce((acc, size) => acc + size, 0); - console.log(`Total size: ${formatSize(totalSize)}`); - } + fileCount = filesData.filter(item => item.type === 'blob').length; } return [ `working (${projectData.visibility})`, projectData.owner.username, - fileCount, - formatSize(totalSize) + fileCount ]; } else if (projectResponse.status === 404) { - return ['not found', 'Unknown', 0, '0 B']; + return ['not found', 'Unknown', 0]; } else { - return ['disconnect', 'Unknown', 0, '0 B']; + return ['disconnect', 'Unknown', 0]; } } catch (error) { console.error('GitLab project check error:', error); - return ['disconnect', 'Error', 0, '0 B']; + return ['disconnect', 'Error', 0]; } } diff --git a/cloudflare_worker/github_gitlab_s3.js b/cloudflare_worker/github_gitlab_s3.js index ddbc39e..4795513 100644 --- a/cloudflare_worker/github_gitlab_s3.js +++ b/cloudflare_worker/github_gitlab_s3.js @@ -9,7 +9,6 @@ const GITLAB_CONFIGS = [ ]; // GitHub 配置 -const GITHUB_REPOS = ['']; // GitHub 仓库名列表 const GITHUB_USERNAME = ''; // GitHub 用户名 const GITHUB_PAT = ''; // GitHub 个人访问令牌 @@ -57,6 +56,10 @@ const DIR = ''; // 定义集群里全部节点连接状态的密码验证，区分大小写（优先使用自定义密码，若为空则使用 GITHUB_PAT） const CHECK_PASSWORD = '' || GITHUB_PAT; +// GitHub 备份策略 +const STRATEGY = 'size' // 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 +const DELETE = 'true' // 可选 [true (默认) | false]，已复制到 GitHub 的文件，是否从 R2 删除 + // 用户配置区域结束 ================================= // AWS SDK 签名相关函数开始 ================================= @@ -185,13 +188,8 @@ export default { } } - const isValidGithubRepos = Array.isArray(GITHUB_REPOS) && - GITHUB_REPOS.length > 0 && - GITHUB_REPOS.some(repo => repo.trim() !== ''); - - const githubRepos = isValidGithubRepos - ? GITHUB_REPOS.filter(repo => repo.trim() !== '') - : GITLAB_CONFIGS.map(config => config.name); + // 直接使用 GITLAB_CONFIGS 中的 name 作为 GitHub 仓库名 + const githubRepos = GITLAB_CONFIGS.map(config => config.name); const FILE = url.pathname.split('/').pop(); @@ -516,8 +514,8 @@ async function listProjects(gitlabConfigs, githubRepos, githubUsername, githubPa // 添加 GitLab 结果 gitlabConfigs.forEach((config, index) => { - const [status, username, fileCount, totalSize] = gitlabResults[index]; - result += `GitLab: Project ID ${config.id} - ${status} (Username: ${username}, Files: ${fileCount}, Size: ${totalSize})\n`; + const [status, username, fileCount] = gitlabResults[index]; + result += `GitLab: Project ID ${config.id} - ${status} (Username: ${username}, Files: ${fileCount})\n`; }); // 添加 R2 结果 @@ -632,67 +630,10 @@ async function checkGitHubRepo(owner, repo, pat) { } } -// 获取单个文件大小的辅助函数 -async function getFileSizeFromGitLab(projectId, filePath, pat, retries = 3) { - for (let i = 0; i < retries; i++) { - try { - // 使用 raw 端点和 HEAD 请求获取文件大小 - const fileUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/files${filePath}/raw?ref=main`; - const response = await fetch(fileUrl, { - method: 'HEAD', - headers: { 'PRIVATE-TOKEN': pat } - }); - - if (response.status === 200) { - const contentLength = response.headers.get('content-length'); - return contentLength ? parseInt(contentLength, 10) : 0; - } else if (response.status === 429) { - // 遇到限流时等待后重试 - await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1))); - continue; - } - console.error(`Failed to get file size: ${response.status}`); - return 0; - } catch (error) { - if (i === retries - 1) { - console.error(`Error fetching file size:`, error); - } - if (i < retries - 1) { - await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1))); - continue; - } - } - return 0; - } - return 0; -} - -// 添加并发控制的辅助函数 -async function asyncPool(concurrency, iterable, iteratorFn) { - const ret = []; - const executing = new Set(); - - for (const item of iterable) { - const p = Promise.resolve().then(() => iteratorFn(item)); - ret.push(p); - executing.add(p); - - const clean = () => executing.delete(p); - p.then(clean).catch(clean); - - if (executing.size >= concurrency) { - await Promise.race(executing); - } - } - - return Promise.all(ret); -} - // 检查 GitLab 项目的异步函数 async function checkGitLabProject(projectId, pat) { const projectUrl = `https://gitlab.com/api/v4/projects/${projectId}`; - // 步骤1: 获取文件列表 - const filesUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?ref=main&path=${DIR}`; + const filesUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?ref=main&path=${DIR}&recursive=true&per_page=10000`; try { const [projectResponse, filesResponse] = await Promise.all([ @@ -706,45 +647,26 @@ async function checkGitLabProject(projectId, pat) { if (projectResponse.status === 200) { const projectData = await projectResponse.json(); - let totalSize = 0; let fileCount = 0; if (filesResponse.status === 200) { const filesData = await filesResponse.json(); - fileCount = filesData.length; - - if (fileCount > 0) { - console.log(`Found ${fileCount} files in ${DIR} directory`); - - // 步骤2: 并发获取每个文件的大小 - const CONCURRENT_REQUESTS = 100; - const sizes = await asyncPool(CONCURRENT_REQUESTS, filesData, async (file) => { - // 构造文件路径，格式为 /files%2Ffilename - const encodedPath = `/${encodeURIComponent(file.path)}`; - const size = await getFileSizeFromGitLab(projectId, encodedPath, pat); - console.log(`File: ${file.path}, Size: ${formatSize(size)}`); - return size; - }); - - totalSize = sizes.reduce((acc, size) => acc + size, 0); - console.log(`Total size: ${formatSize(totalSize)}`); - } + fileCount = filesData.filter(item => item.type === 'blob').length; } return [ `working (${projectData.visibility})`, projectData.owner.username, - fileCount, - formatSize(totalSize) + fileCount ]; } else if (projectResponse.status === 404) { - return ['not found', 'Unknown', 0, '0 B']; + return ['not found', 'Unknown', 0]; } else { - return ['disconnect', 'Unknown', 0, '0 B']; + return ['disconnect', 'Unknown', 0]; } } catch (error) { console.error('GitLab project check error:', error); - return ['disconnect', 'Error', 0, '0 B']; + return ['disconnect', 'Error', 0]; } } diff --git a/github_to_gitlab/.github/workflows/cluster_sync.yml b/github_to_gitlab/.github/workflows/cluster_sync.yml index 6cd25ed..c372440 100644 --- a/github_to_gitlab/.github/workflows/cluster_sync.yml +++ b/github_to_gitlab/.github/workflows/cluster_sync.yml @@ -8,9 +8,15 @@ on: jobs: GitHub_To_GitLab : runs-on: ubuntu-latest + + env: + ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} + WORKER_NAME: ${{ secrets.WORKER_NAME }} + API_TOKEN: ${{ secrets.API_TOKEN }} + steps: - name: Checkout code uses: actions/checkout@v4.2.2 - name: Clone Repo to GitLab - uses: fscarmen2/github_to_gitlab@v1.0.0 \ No newline at end of file + uses: fscarmen2/github_to_gitlab@v1.0.1 \ No newline at end of file diff --git a/github_to_gitlab/config.yml b/github_to_gitlab/config.yml deleted file mode 100644 index 6c00a46..0000000 --- a/github_to_gitlab/config.yml +++ /dev/null @@ -1,33 +0,0 @@ -# 以下所有的 KV 填写格式要求: 1. 修改的地方需要把尖括号一并去掉；2. : 冒号后面有空格 -## GitHub 部分 -github_pat: <改为 GitHub PAT获取到的 Token> -github_repo_prefix: <改为 GitHub 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> - -## GitLab 部分 -gitlab_repo_prefix: <改为 GitLab 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> -gitlab_username: <改为 GitLab 用户名> -gitlab_pats: - # 注意，下面只填仓库名数字序号，比如 repo1,repo2,repo3,repo4，就只填 1,2,3,4 即可 - 1: <改为 GitLab 仓库名1 api> - 2: <改为 GitLab 仓库名2 api> - 3: <改为 GitLab 仓库名3 api> - 4: <改为 GitLab 仓库名4 api> - -## CloudFlare R2 部分 -r2_accounts: - - name: <改为 R2 帐户1 名，建议用 email 方便管理> - account_id: <改为 R2 账户1 ID> - access_key_id: <改为 R2 账户1 访问密钥 ID> - secret_access_key: <改为 R2 账户1 机密访问密钥> - bucket: <改为 R2 账户1 存储桶名称> - dir: <改为 R2 账户1 文件存放目录> - - name: <改为 R2 帐户2 名，建议用 email 方便管理> - account_id: <改为 R2 账户2 ID> - access_key_id: <改为 R2 账户2 访问密钥 ID> - secret_access_key: <改为 R2 账户2 机密访问密钥> - bucket: <改为 R2 账户2 存储桶名称> - dir: <改为 R2 账户2 文件存放目录> - -## 备份策略 -strategy: size # 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 -delete: true # 可选 [true (默认) | false]，已复制到 GitHub 的文件，是否从 R2 删除 \ No newline at end of file diff --git a/gitlab_to_github/config.yml b/gitlab_to_github/config.yml deleted file mode 100644 index 6c00a46..0000000 --- a/gitlab_to_github/config.yml +++ /dev/null @@ -1,33 +0,0 @@ -# 以下所有的 KV 填写格式要求: 1. 修改的地方需要把尖括号一并去掉；2. : 冒号后面有空格 -## GitHub 部分 -github_pat: <改为 GitHub PAT获取到的 Token> -github_repo_prefix: <改为 GitHub 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> - -## GitLab 部分 -gitlab_repo_prefix: <改为 GitLab 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> -gitlab_username: <改为 GitLab 用户名> -gitlab_pats: - # 注意，下面只填仓库名数字序号，比如 repo1,repo2,repo3,repo4，就只填 1,2,3,4 即可 - 1: <改为 GitLab 仓库名1 api> - 2: <改为 GitLab 仓库名2 api> - 3: <改为 GitLab 仓库名3 api> - 4: <改为 GitLab 仓库名4 api> - -## CloudFlare R2 部分 -r2_accounts: - - name: <改为 R2 帐户1 名，建议用 email 方便管理> - account_id: <改为 R2 账户1 ID> - access_key_id: <改为 R2 账户1 访问密钥 ID> - secret_access_key: <改为 R2 账户1 机密访问密钥> - bucket: <改为 R2 账户1 存储桶名称> - dir: <改为 R2 账户1 文件存放目录> - - name: <改为 R2 帐户2 名，建议用 email 方便管理> - account_id: <改为 R2 账户2 ID> - access_key_id: <改为 R2 账户2 访问密钥 ID> - secret_access_key: <改为 R2 账户2 机密访问密钥> - bucket: <改为 R2 账户2 存储桶名称> - dir: <改为 R2 账户2 文件存放目录> - -## 备份策略 -strategy: size # 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 -delete: true # 可选 [true (默认) | false]，已复制到 GitHub 的文件，是否从 R2 删除 \ No newline at end of file diff --git a/r2_to_github/.github/workflows/r2_to_github.yml b/r2_to_github/.github/workflows/r2_to_github.yml index 63480a4..7d6e8b3 100644 --- a/r2_to_github/.github/workflows/r2_to_github.yml +++ b/r2_to_github/.github/workflows/r2_to_github.yml @@ -8,9 +8,15 @@ on: jobs: R2_To_GitHub: runs-on: ubuntu-latest + + env: + ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} + WORKER_NAME: ${{ secrets.WORKER_NAME }} + API_TOKEN: ${{ secrets.API_TOKEN }} + steps: - name: Checkout code uses: actions/checkout@v4.2.2 - name: Sync files to GitHub - uses: fscarmen2/r2_to_github@v1.0.0 \ No newline at end of file + uses: fscarmen2/r2_to_github@v1.0.2 \ No newline at end of file diff --git a/r2_to_github/config.yml b/r2_to_github/config.yml deleted file mode 100644 index 6c00a46..0000000 --- a/r2_to_github/config.yml +++ /dev/null @@ -1,33 +0,0 @@ -# 以下所有的 KV 填写格式要求: 1. 修改的地方需要把尖括号一并去掉；2. : 冒号后面有空格 -## GitHub 部分 -github_pat: <改为 GitHub PAT获取到的 Token> -github_repo_prefix: <改为 GitHub 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> - -## GitLab 部分 -gitlab_repo_prefix: <改为 GitLab 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> -gitlab_username: <改为 GitLab 用户名> -gitlab_pats: - # 注意，下面只填仓库名数字序号，比如 repo1,repo2,repo3,repo4，就只填 1,2,3,4 即可 - 1: <改为 GitLab 仓库名1 api> - 2: <改为 GitLab 仓库名2 api> - 3: <改为 GitLab 仓库名3 api> - 4: <改为 GitLab 仓库名4 api> - -## CloudFlare R2 部分 -r2_accounts: - - name: <改为 R2 帐户1 名，建议用 email 方便管理> - account_id: <改为 R2 账户1 ID> - access_key_id: <改为 R2 账户1 访问密钥 ID> - secret_access_key: <改为 R2 账户1 机密访问密钥> - bucket: <改为 R2 账户1 存储桶名称> - dir: <改为 R2 账户1 文件存放目录> - - name: <改为 R2 帐户2 名，建议用 email 方便管理> - account_id: <改为 R2 账户2 ID> - access_key_id: <改为 R2 账户2 访问密钥 ID> - secret_access_key: <改为 R2 账户2 机密访问密钥> - bucket: <改为 R2 账户2 存储桶名称> - dir: <改为 R2 账户2 文件存放目录> - -## 备份策略 -strategy: size # 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 -delete: true # 可选 [true (默认) | false]，已复制到 GitHub 的文件，是否从 R2 删除 \ No newline at end of file diff --git a/s3_to_github/.github/workflows/s3_to_github.yml b/s3_to_github/.github/workflows/s3_to_github.yml index 4bdcf05..c26b050 100644 --- a/s3_to_github/.github/workflows/s3_to_github.yml +++ b/s3_to_github/.github/workflows/s3_to_github.yml @@ -8,9 +8,15 @@ on: jobs: S3_To_GitHub: runs-on: ubuntu-latest + + env: + ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} + WORKER_NAME: ${{ secrets.WORKER_NAME }} + API_TOKEN: ${{ secrets.API_TOKEN }} + steps: - name: Checkout code uses: actions/checkout@v4.2.2 - name: Sync files to GitHub - uses: fscarmen2/r2_to_github@v1.0.1 \ No newline at end of file + uses: fscarmen2/r2_to_github@v1.0.2 \ No newline at end of file diff --git a/s3_to_github/config.yml b/s3_to_github/config.yml deleted file mode 100644 index b1b4825..0000000 --- a/s3_to_github/config.yml +++ /dev/null @@ -1,48 +0,0 @@ -# 以下所有的 KV 填写格式要求: 1. 修改的地方需要把尖括号一并去掉；2. : 冒号后面有空格 -## GitHub 部分 -github_pat: <改为 GitHub PAT获取到的 Token> -github_repo_prefix: <改为 GitHub 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> - -## GitLab 部分 -gitlab_repo_prefix: <改为 GitLab 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> -gitlab_username: <改为 GitLab 用户名> -gitlab_pats: - # 注意，下面只填仓库名数字序号，比如 repo1,repo2,repo3,repo4，就只填 1,2,3,4 即可 - 1: <改为 GitLab 仓库名1 api> - 2: <改为 GitLab 仓库名2 api> - 3: <改为 GitLab 仓库名3 api> - 4: <改为 GitLab 仓库名4 api> - -## CloudFlare R2 部分 -r2_accounts: - - name: <改为 R2 帐户1 名，建议用 email 方便管理> - account_id: <改为 R2 账户1 ID> - access_key_id: <改为 R2 账户1 访问密钥 ID> - secret_access_key: <改为 R2 账户1 机密访问密钥> - bucket: <改为 R2 账户1 存储桶名称> - dir: <改为 R2 账户1 文件存放目录> - - name: <改为 R2 帐户2 名，建议用 email 方便管理> - account_id: <改为 R2 账户2 ID> - access_key_id: <改为 R2 账户2 访问密钥 ID> - secret_access_key: <改为 R2 账户2 机密访问密钥> - bucket: <改为 R2 账户2 存储桶名称> - dir: <改为 R2 账户2 文件存放目录> - -## Backblaze B2 部分 -b2_accounts: - - name: <改为 B2 帐户1 名，建议用 email 方便管理> - endpoint: <改为 B2 账户1 Endpoint，以 s3 开头的，不要以 https:// 开头> - key_id: <改为 B2 账户1 keyID> - application_key: <改为 B2 账户1 applicationKey> - bucket: <改为 B2 账户1 bucketName，这个是全 Backblaze 唯一的> - dir: <改为 B2 账户1 文件存放目录> - - name: <改为 B2 帐户2 名，建议用 email 方便管理> - endpoint: <改为 B2 账户2 Endpoint，以 s3 开头的，不要以 https:// 开头> - key_id: <改为 B2 账户2 keyID> - application_key: <改为 B2 账户2 applicationKey> - bucket: <改为 B2 账户2 bucketName，这个是全 Backblaze 唯一的> - dir: <改为 B2 账户2 文件存放目录> - -## 备份策略 -strategy: size # 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 -delete: true # 可选 [true (默认) | false]，已复制到 GitHub 的文件，是否从 R2 删除 \ No newline at end of file From 78f9d5cfca420ab8722f6795c06be28eb28a0767 Mon Sep 17 00:00:00 2001 From: fscarmen2 Date: Tue, 17 Dec 2024 03:11:34 +0000 Subject: [PATCH 02/13] Add Template Repos. --- README.md | 16 ++++++++++++---- gitlab_to_github/config.yml | 8 ++++++++ gitlab_to_github/sync_to_github.sh | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 gitlab_to_github/config.yml diff --git a/README.md b/README.md index 8d79294..043a87f 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,17 @@ -# R2，GitHub 和 GitLab 分布式存储集群 +# 分布式存储集群 -- **方案的配置文件都是 `config.yml`，只需要根据实际修改** +## 更新日期 2024-12-17 -## GitHub --→ GitLab 目录下2个文件，放到 GitHub 库: -- **配置文件**: `config.yml` +## 各方案的独立分仓库 +| 方案 | worker 文件 | 同步仓库模版 | 视频教程 | +| --- |--- |--- |--- | +| GitHub only | [worker.js](https://raw.githubusercontent.com/fscarmen2/pic-hosting-cluster/refs/heads/main/cloudflare_worker/github_only.js) | [博文 7.2 Github 设置](https://www.fscarmen.com/2024/10/blog-post.html) | https://youtu.be/eRqIpeeo9SA | +| GitLab only |[worker.js](https://raw.githubusercontent.com/fscarmen2/pic-hosting-cluster/refs/heads/main/cloudflare_worker/gitlab_only.js) | 使用 GitLab 平台自带的镜像功能 | https://youtu.be/tjiI3I3MkaQ | +| GitHub + GitLab | [worker.js](https://raw.githubusercontent.com/fscarmen2/files-hosting-template-1/refs/heads/main/worker.js) | [点击使用模板库，注意需要手动改为私有仓库](https://github.com/new?template_name=files-hosting-template-1&template_owner=fscarmen2) | https://youtu.be/SGex7xJ9YdQ | +| R2 + GitHub + GitLab | [worker.js](https://raw.githubusercontent.com/fscarmen2/files-hosting-template-2/refs/heads/main/worker.js) | [点击使用模板库，注意需要手动改为私有仓库](https://github.com/new?template_name=files-hosting-template-2&template_owner=fscarmen2) | https://youtu.be/5i-86oBLWP8 | +| B2 + R2 + GitHub + GitLab | [worker.js](https://raw.githubusercontent.com/fscarmen2/files-hosting-template-3/refs/heads/main/worker.js) | [点击使用模板库，注意需要手动改为私有仓库](https://github.com/new?template_name=files-hosting-template-3&template_owner=fscarmen2) | https://youtu.be/4X1FjLCAckI | + +## GitHub --→ GitLab 目录下1个文件，放到 GitHub 库: - **AC 脚本**: `./github/workflows/cluster_sync.yml` ## GitLab --→ GitHub 目录下3个文件，放到 GitLab 库: diff --git a/gitlab_to_github/config.yml b/gitlab_to_github/config.yml new file mode 100644 index 0000000..2be4c75 --- /dev/null +++ b/gitlab_to_github/config.yml @@ -0,0 +1,8 @@ +# 以下所有的 KV 填写格式要求: 1. 修改的地方需要把尖括号一并去掉；2. : 冒号后面有空格 +## GitHub 部分 +github_pat: <改为 GitHub PAT获取到的 Token> +github_repo_prefix: <改为 GitHub 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> + +## GitLab 部分 +gitlab_repo_prefix: <改为 GitLab 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> +gitlab_username: <改为 GitLab 用户名> \ No newline at end of file diff --git a/gitlab_to_github/sync_to_github.sh b/gitlab_to_github/sync_to_github.sh index 6bbbfa6..8e565c8 100644 --- a/gitlab_to_github/sync_to_github.sh +++ b/gitlab_to_github/sync_to_github.sh @@ -3,7 +3,7 @@ set -e # 从文件读取配置，为防止用户没有去掉尖括号，故用 sed 处理强制去掉所有的 < > 符号 -CONFIG=$(sed "s/[<>]//g" config.yml) +CONFIG=$(sed 's/[<>]//g; s/:/: /g; s/:[[:space:]]\+/: /g' config.yml) GITHUB_PAT=$(awk '$1 ~ "^github_pat" {sub(/[^:]*:[[:space:]]*/, ""); print}' <<< "$CONFIG") GITLAB_USERNAME=$(awk '$1 ~ "^gitlab_username" {sub(/[^:]*:[[:space:]]*/, ""); print}' <<< "$CONFIG") GITHUB_REPO_PREFIX=$(awk '$1 ~ "^github_repo_prefix" {sub(/[^:]*:[[:space:]]*/, ""); print}' <<< "$CONFIG") From 68c65e602db7aec8f16cab173b38e5eccc4fbb5d Mon Sep 17 00:00:00 2001 From: fscarmen2 Date: Wed, 18 Dec 2024 04:40:26 +0000 Subject: [PATCH 03/13] Merge some Worker js. --- README.md | 20 +- cloudflare_worker/github_gitlab.js | 456 --------- cloudflare_worker/github_gitlab_r2.js | 651 ------------- ...ub_gitlab_s3.js => github_gitlab_r2_b2.js} | 882 +++++++++--------- 4 files changed, 471 insertions(+), 1538 deletions(-) delete mode 100644 cloudflare_worker/github_gitlab.js delete mode 100644 cloudflare_worker/github_gitlab_r2.js rename cloudflare_worker/{github_gitlab_s3.js => github_gitlab_r2_b2.js} (59%) diff --git a/README.md b/README.md index 043a87f..5adb308 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ # 分布式存储集群 -## 更新日期 2024-12-17 +## 更新日期 2024-12-18 ## 各方案的独立分仓库 | 方案 | worker 文件 | 同步仓库模版 | 视频教程 | | --- |--- |--- |--- | | GitHub only | [worker.js](https://raw.githubusercontent.com/fscarmen2/pic-hosting-cluster/refs/heads/main/cloudflare_worker/github_only.js) | [博文 7.2 Github 设置](https://www.fscarmen.com/2024/10/blog-post.html) | https://youtu.be/eRqIpeeo9SA | | GitLab only |[worker.js](https://raw.githubusercontent.com/fscarmen2/pic-hosting-cluster/refs/heads/main/cloudflare_worker/gitlab_only.js) | 使用 GitLab 平台自带的镜像功能 | https://youtu.be/tjiI3I3MkaQ | -| GitHub + GitLab | [worker.js](https://raw.githubusercontent.com/fscarmen2/files-hosting-template-1/refs/heads/main/worker.js) | [点击使用模板库，注意需要手动改为私有仓库](https://github.com/new?template_name=files-hosting-template-1&template_owner=fscarmen2) | https://youtu.be/SGex7xJ9YdQ | -| R2 + GitHub + GitLab | [worker.js](https://raw.githubusercontent.com/fscarmen2/files-hosting-template-2/refs/heads/main/worker.js) | [点击使用模板库，注意需要手动改为私有仓库](https://github.com/new?template_name=files-hosting-template-2&template_owner=fscarmen2) | https://youtu.be/5i-86oBLWP8 | -| B2 + R2 + GitHub + GitLab | [worker.js](https://raw.githubusercontent.com/fscarmen2/files-hosting-template-3/refs/heads/main/worker.js) | [点击使用模板库，注意需要手动改为私有仓库](https://github.com/new?template_name=files-hosting-template-3&template_owner=fscarmen2) | https://youtu.be/4X1FjLCAckI | +| GitHub + GitLab | [worker.js](https://raw.githubusercontent.com/fscarmen2/pic-hosting-cluster/refs/heads/main/cloudflare_worker/github_gitlab_r2_b2.js) | [点击使用模板库，注意需要手动改为私有仓库](https://github.com/new?template_name=files-hosting-template-1&template_owner=fscarmen2) | https://youtu.be/SGex7xJ9YdQ | +| R2 + GitHub + GitLab | [worker.js](https://raw.githubusercontent.com/fscarmen2/pic-hosting-cluster/refs/heads/main/cloudflare_worker/github_gitlab_r2_b2.js) | [点击使用模板库，注意需要手动改为私有仓库](https://github.com/new?template_name=files-hosting-template-2&template_owner=fscarmen2) | https://youtu.be/5i-86oBLWP8 | +| B2 + R2 + GitHub + GitLab | [worker.js](https://raw.githubusercontent.com/fscarmen2/pic-hosting-cluster/refs/heads/main/cloudflare_worker/github_gitlab_r2_b2.js) | [点击使用模板库，注意需要手动改为私有仓库](https://github.com/new?template_name=files-hosting-template-2&template_owner=fscarmen2) | https://youtu.be/4X1FjLCAckI | ## GitHub --→ GitLab 目录下1个文件，放到 GitHub 库: - **AC 脚本**: `./github/workflows/cluster_sync.yml` @@ -19,11 +19,11 @@ - **同步脚本**: `sync_to_github.sh` - **CI/CD 脚本**: `.gitlab-ci.yml` -## R2 --→ GitHub 目录下2个文件，放到 GitHub 库: +## R2 --→ GitHub 目录下1个文件，放到 GitHub 库: - **AC 脚本**: `./github/workflows/r2_to_github.yml` - **设置3个secrets**: `ACCOUNT_ID`, `WORKER_NAME` 和 `API_TOKEN` -## S2 (R2+B2) --→ GitHub 目录下2个文件，放到 GitHub 库: +## S3 (R2+B2) --→ GitHub 目录下1个文件，放到 GitHub 库: - **AC 脚本**: `./github/workflows/s3_to_github.yml` - **设置3个secrets**: `ACCOUNT_ID`, `WORKER_NAME` 和 `API_TOKEN` @@ -41,12 +41,12 @@ ![image](https://github.com/user-attachments/assets/25b8d0fa-8302-4cb9-a6db-83e449e9664c) -## Cloudflare worker 目录下5个文件，复制代码到 worker 处： +## Cloudflare worker 目录下3个文件，复制代码到 worker 处： - **只使用 GitHub**: `github_only.js` - **只使用 GitLab**: `gitlab_only.js` -- **同时使用 GitHub 和 GitLab**: `github_gitlab.js` -- **同时使用 GitHub, GitLab 和 R2**: `github_gitlab_r2.js` -- **同时使用 GitHub, GitLab，R2 和 B2**: `github_gitlab_s3.js` +- **同时使用 GitHub 和 GitLab**: `github_gitlab_r2_b2.js` +- **同时使用 GitHub, GitLab 和 R2**: `github_gitlab_r2_b2.js` +- **同时使用 GitHub, GitLab，R2 和 B2**: `github_gitlab_r2_b2.js` ## 检测节点状态 `https://<自定义域名>/` diff --git a/cloudflare_worker/github_gitlab.js b/cloudflare_worker/github_gitlab.js deleted file mode 100644 index 3e3fb16..0000000 --- a/cloudflare_worker/github_gitlab.js +++ /dev/null @@ -1,456 +0,0 @@ -// 用户配置区域开始 ================================= - -// GitLab 仓库配置列表，包含仓库名称、ID 和访问 Token -const GITLAB_CONFIGS = [ - { name: 'repo1', id: 'repoID1', token: 'repoToken1' }, // 节点1配置 - { name: 'repo2', id: 'repoID2', token: 'repoToken2' }, // 节点2配置 - { name: 'repo3', id: 'repoID3', token: 'repoToken3' }, // 节点3配置 - { name: 'repo4', id: 'repoID4', token: 'repoToken4' }, // 节点4配置 -]; - -// GitHub 用户名和个人访问令牌（PAT） -const GITHUB_USERNAME = ''; // 填写 GitHub 用户名 -const GITHUB_PAT = ''; // 填写 GitHub 访问令牌 - -// 定义集群访问目录 -const DIR = ''; // 存储的文件目录，可以为空，表示根目录 - -// 定义集群里全部节点连接状态的密码验证，区分大小写（优先使用自定义密码，若为空则使用 GITHUB_PAT） -const CHECK_PASSWORD = '' || GITHUB_PAT; - -// 定义全局缓存时间，单位为秒，默认值为一年（31556952秒） -const CACHE_MAX_AGE = 31556952; // 一年 - -// GitHub 备份策略 -const STRATEGY = 'size' // 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 -const DELETE = 'true' // 可选 [true (默认) | false]，已复制到 GitHub 的文件，是否从 R2 删除 - -// 用户配置区域结束 ================================= - -// Cloudflare Worker 主函数，处理 HTTP 请求 -export default { - async fetch(request, env, ctx) { - const url = new URL(request.url); // 获取请求的 URL - const from = url.searchParams.get('from')?.toLowerCase(); // 获取来源参数 - - // 只在没有 from 参数时才检查和使用缓存 - let cacheResponse; - if (!from) { - const cacheUrl = new URL(request.url); // 获取请求 URL 的对象表示 - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - cacheResponse = await cache.match(cacheKey); - - if (cacheResponse) { - // 如果有缓存，直接返回缓存内容 - return cacheResponse; - } - } - - // 直接使用 GITLAB_CONFIGS 中的 name 作为 GitHub 仓库名 - const githubRepos = GITLAB_CONFIGS.map(config => config.name); - - const FILE = url.pathname.split('/').pop(); // 获取请求文件名 - - if (url.pathname === `/${CHECK_PASSWORD}`) { - const result = await listProjects(GITLAB_CONFIGS, githubRepos, GITHUB_USERNAME, GITHUB_PAT); - // 如果有 from 参数，添加禁止缓存的头部 - if (from) { - result.headers.append('Cache-Control', 'no-store, no-cache, must-revalidate, proxy-revalidate'); - result.headers.append('Pragma', 'no-cache'); - result.headers.append('Expires', '0'); - } else { - result.headers.append('Cache-Control', `public, max-age=${CACHE_MAX_AGE}`); - const cacheUrl = new URL(request.url); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - ctx.waitUntil(cache.put(cacheKey, result.clone())); // 异步存入缓存 - } - return result; - } - - const startTime = Date.now(); // 记录请求开始时间 - let requests = []; // 初始化请求数组 - - // 构建 API 请求数组，根据请求来源（GitHub 或 GitLab 或混合） - if (from === 'where') { - // 获取文件信息模式 - const githubRequests = githubRepos.map(repo => ({ - url: `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found') - const data = await response.json() - return { - size: data.size, - exists: true - } - } - })) - - const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found') - const data = await response.json() - const size = atob(data.content).length - return { - size: size, - exists: true - } - } - })) - - requests = [...githubRequests, ...gitlabRequests] - } else { - // 获取文件内容模式 - if (from === 'github') { - requests = githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo - })) - } else if (from === 'gitlab') { - requests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name - })) - } else { - requests = [ - ...githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo - })), - ...GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name - })) - ] - } - } - - const fetchPromises = requests.map(({ url, headers, source, repo, processResponse }) => { - return fetch(new Request(url, { - method: 'GET', - headers: headers - })).then(async response => { - if (from === 'where') { - // 查询文件所在位置 - try { - const result = await processResponse(response); - const endTime = Date.now(); // 记录请求结束时间 - const duration = endTime - startTime; // 计算请求耗时 - - // 格式化文件大小 - const formattedSize = result.size > 1024 * 1024 - ? `${(result.size / (1024 * 1024)).toFixed(2)} MB` - : `${(result.size / 1024).toFixed(2)} kB`; - - // 返回文件信息 - return { - fileName: FILE, - size: formattedSize, - source: `${source} (${repo})`, - duration: `${duration}ms` - }; - } catch (error) { - throw new Error(`Not found in ${source} (${repo})`); - } - } else { - // 如果不是 "where" 查询，直接返回响应内容 - if (!response.ok) { - throw new Error(`Not found in ${source} (${repo})`); - } - return response; - } - }).catch(error => { - throw new Error(`Error in ${source} (${repo}): ${error.message}`); - }); - }); - - try { - if (requests.length === 0) { - throw new Error('No valid source specified'); - } - - const result = await Promise.any(fetchPromises); - - let response; - if (from === 'where') { - // 如果是 where 查询，返回 JSON 格式响应 - response = new Response(JSON.stringify(result, null, 2), { - headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-store, no-cache, must-revalidate, proxy-revalidate', - 'Pragma': 'no-cache', - 'Expires': '0' - } - }); - } else if (result instanceof Response) { - // 先读取响应体 - const blob = await result.blob(); - const headers = { - 'Content-Type': result.headers.get('Content-Type') || 'application/octet-stream', - 'Access-Control-Allow-Origin': '*' - }; - - // 如果有 from 参数，添加禁止缓存的头部 - if (from) { - headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, proxy-revalidate'; - headers['Pragma'] = 'no-cache'; - headers['Expires'] = '0'; - } else { - // 如果没有 from 参数，使用配置的缓存时间 - headers['Cache-Control'] = `public, max-age=${CACHE_MAX_AGE}`; - } - - // 创建新的响应，只使用最基本的必要头部 - response = new Response(blob, { - status: 200, - headers: headers - }); - } else { - throw new Error("Unexpected result type"); - } - - // 只在没有 from 参数且不是 where 查询时才缓存响应 - if (!from && from !== 'where') { - const cacheUrl = new URL(request.url); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - ctx.waitUntil(cache.put(cacheKey, response.clone())); - } - - return response; - - } catch (error) { - const sourceText = from === 'where' - ? 'in any repository' - : from - ? `from ${from}` - : 'in the GitHub and GitLab picture cluster'; - - const errorResponse = new Response( - `404: Cannot find the ${FILE} ${sourceText}.`, - { - status: 404, - headers: { - 'Content-Type': 'text/plain', - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-store, no-cache, must-revalidate, proxy-revalidate', - 'Pragma': 'no-cache', - 'Expires': '0' - } - } - ); - - return errorResponse; - } - } -} - -// 列出所有节点仓库状态 -async function listProjects(gitlabConfigs, githubRepos, githubUsername, githubPat) { - - let result = 'GitHub and GitLab Nodes status:\n\n'; - - try { - // 并发执行所有检查 - const [username, ...allChecks] = await Promise.all([ - getGitHubUsername(githubPat), - ...githubRepos.map(repo => - checkGitHubRepo(githubUsername, repo, githubPat) - ), - ...gitlabConfigs.map(config => - checkGitLabProject(config.id, config.token) - ), - ]); - - // 计算各类检查结果的数量 - const githubCount = githubRepos.length; - const gitlabCount = gitlabConfigs.length; - - // 分割检查结果 - const githubResults = allChecks.slice(0, githubCount); - const gitlabResults = allChecks.slice(githubCount, githubCount + gitlabCount); - - // 添加 GitHub 结果 - githubRepos.forEach((repo, index) => { - const [status, fileCount, totalSize] = githubResults[index]; - const formattedSize = formatSize(totalSize); - result += `GitHub: ${repo} - ${status} (Username: ${username}, Files: ${fileCount}, Size: ${formattedSize})\n`; - }); - - // 添加 GitLab 结果 - gitlabConfigs.forEach((config, index) => { - const [status, username, fileCount] = gitlabResults[index]; - result += `GitLab: Project ID ${config.id} - ${status} (Username: ${username}, Files: ${fileCount})\n`; - }); - - } catch (error) { - result += `Error during status check: ${error.message}\n`; - } - - return new Response(result, { - headers: { 'Content-Type': 'text/plain' } - }); -} - -// 文件大小格式化函数 -function formatSize(sizeInBytes) { - if (sizeInBytes >= 1024 * 1024 * 1024) { - return `${(sizeInBytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; - } else if (sizeInBytes >= 1024 * 1024) { - return `${(sizeInBytes / (1024 * 1024)).toFixed(2)} MB`; - } else { - return `${(sizeInBytes / 1024).toFixed(2)} kB`; - } -} - -// 获取 GitHub 用户名的异步函数 -async function getGitHubUsername(pat) { - const url = 'https://api.github.com/user'; // GitHub 用户信息 API 地址 - try { - const response = await fetch(url, { - headers: { - 'Authorization': `token ${pat}`, // 使用个人访问令牌进行授权 - 'Accept': 'application/vnd.github.v3+json', // 指定接受的响应格式 - 'User-Agent': 'Cloudflare Worker' // 用户代理 - } - }); - - // 如果响应状态为 200，表示成功 - if (response.status === 200) { - const data = await response.json(); // 解析 JSON 数据 - return data.login; // 返回用户登录名 - } else { - console.error('GitHub API Error:', response.status); // 记录错误状态 - return 'Unknown'; // 返回未知状态 - } - } catch (error) { - console.error('GitHub request error:', error); // 记录请求错误 - return 'Error'; // 返回错误状态 - } -} - -// 检查 GitHub 仓库的异步函数 -async function checkGitHubRepo(owner, repo, pat) { - const repoUrl = `https://api.github.com/repos/${owner}/${repo}`; - const contentsUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${DIR}`; // 直接检查指定目录 - - const headers = { - 'Authorization': `token ${pat}`, - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'Cloudflare Worker' - }; - - try { - // 并发请求获取仓库信息和目录内容 - const [repoResponse, contentsResponse] = await Promise.all([ - fetch(repoUrl, { headers }), - fetch(contentsUrl, { headers }) - ]); - - const repoData = await repoResponse.json(); - - if (repoResponse.status !== 200) { - throw new Error(`Repository error: ${repoData.message}`); - } - - if (contentsResponse.status !== 200) { - return [`working (${repoData.private ? 'private' : 'public'})`, 0, 0]; - } - - const contentsData = await contentsResponse.json(); - - // 计算文件数量和总大小 - const fileStats = contentsData.reduce((acc, item) => { - if (item.type === 'file') { - return { - count: acc.count + 1, - size: acc.size + (item.size || 0) - }; - } - return acc; - }, { count: 0, size: 0 }); - - return [ - `working (${repoData.private ? 'private' : 'public'})`, - fileStats.count, - fileStats.size - ]; - - } catch (error) { - console.error(`Error checking GitHub repo ${repo}:`, error); - return [`error: ${error.message}`, 0, 0]; - } -} - -// 检查 GitLab 项目的异步函数 -async function checkGitLabProject(projectId, pat) { - const projectUrl = `https://gitlab.com/api/v4/projects/${projectId}`; - const filesUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?ref=main&path=${DIR}&recursive=true&per_page=10000`; - - try { - const [projectResponse, filesResponse] = await Promise.all([ - fetch(projectUrl, { - headers: { 'PRIVATE-TOKEN': pat } - }), - fetch(filesUrl, { - headers: { 'PRIVATE-TOKEN': pat } - }) - ]); - - if (projectResponse.status === 200) { - const projectData = await projectResponse.json(); - let fileCount = 0; - - if (filesResponse.status === 200) { - const filesData = await filesResponse.json(); - fileCount = filesData.filter(item => item.type === 'blob').length; - } - - return [ - `working (${projectData.visibility})`, - projectData.owner.username, - fileCount - ]; - } else if (projectResponse.status === 404) { - return ['not found', 'Unknown', 0]; - } else { - return ['disconnect', 'Unknown', 0]; - } - } catch (error) { - console.error('GitLab project check error:', error); - return ['disconnect', 'Error', 0]; - } -} \ No newline at end of file diff --git a/cloudflare_worker/github_gitlab_r2.js b/cloudflare_worker/github_gitlab_r2.js deleted file mode 100644 index aa6e9cf..0000000 --- a/cloudflare_worker/github_gitlab_r2.js +++ /dev/null @@ -1,651 +0,0 @@ -// 用户配置区域开始 ================================= - -// GitLab 节点配置 -const GITLAB_CONFIGS = [ - { name: '', id: '', token: '' }, // GitLab 账户1 - { name: '', id: '', token: '' }, // GitLab 账户2 - { name: '', id: '', token: '' }, // GitLab 账户3 - { name: '', id: '', token: '' }, // GitLab 账户4 -]; - -// GitHub 配置 -const GITHUB_USERNAME = ''; // GitHub 用户名 -const GITHUB_PAT = ''; // GitHub 个人访问令牌 - -// R2 存储配置 -const R2_CONFIGS = [ - { - name: '', // 帐户1 ID - accountId: '', // 帐户1 访问密钥 ID - accessKeyId: '', // 帐户1 机密访问密钥 - secretAccessKey: '', // 帐户1 机密访问密钥 - bucket: '' // 帐户1 R2 存储桶名称 - }, - { - name: '', // 帐户2 ID - accountId: '', // 帐户2 访问密钥 ID - accessKeyId: '', // 帐户2 机密访问密钥 - secretAccessKey: '', // 帐户2 机密访问密钥 - bucket: '' // 帐户2 R2 存储桶名称 - }, - // 可以添加更多 R2 配置 -]; - -// 定义集群访问目录 -const DIR = ''; - -// 定义集群里全部节点连接状态的密码验证，区分大小写（优先使用自定义密码，若为空则使用 GITHUB_PAT） -const CHECK_PASSWORD = '' || GITHUB_PAT; - -// GitHub 备份策略 -const STRATEGY = 'size' // 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 -const DELETE = 'true' // 可选 [true (默认) | false]，已复制到 GitHub 的文件，是否从 R2 删除 - -// 用户配置区域结束 ================================= - -// AWS SDK 签名相关函数开始 ================================= - -// 获取签名URL -async function getSignedUrl(r2Config, method, path) { - const region = 'auto'; - const service = 's3'; - const host = `${r2Config.accountId}.r2.cloudflarestorage.com`; - const datetime = new Date().toISOString().replace(/[:-]|\.\d{3}/g, ''); - const date = datetime.substr(0, 8); - - const canonicalRequest = [ - method, - '/' + path, - '', - `host:${host}`, - 'x-amz-content-sha256:UNSIGNED-PAYLOAD', - `x-amz-date:${datetime}`, - '', - 'host;x-amz-content-sha256;x-amz-date', - 'UNSIGNED-PAYLOAD' - ].join('\n'); - - const stringToSign = [ - 'AWS4-HMAC-SHA256', - datetime, - `${date}/${region}/${service}/aws4_request`, - await sha256(canonicalRequest) - ].join('\n'); - - const signature = await getSignature( - r2Config.secretAccessKey, - date, - region, - service, - stringToSign - ); - - const authorization = [ - `AWS4-HMAC-SHA256 Credential=${r2Config.accessKeyId}/${date}/${region}/${service}/aws4_request`, - `SignedHeaders=host;x-amz-content-sha256;x-amz-date`, - `Signature=${signature}` - ].join(', '); - - return { - url: `https://${host}/${path}`, - headers: { - 'Authorization': authorization, - 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - 'x-amz-date': datetime, - 'Host': host - } - }; -} - -// SHA256 哈希函数 -async function sha256(message) { - const msgBuffer = new TextEncoder().encode(message); - const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer); - return Array.from(new Uint8Array(hashBuffer)) - .map(b => b.toString(16).padStart(2, '0')) - .join(''); -} - -// HMAC-SHA256 函数 -async function hmacSha256(key, message) { - const keyBuffer = key instanceof ArrayBuffer ? key : new TextEncoder().encode(key); - const messageBuffer = new TextEncoder().encode(message); - - const cryptoKey = await crypto.subtle.importKey( - 'raw', - keyBuffer, - { name: 'HMAC', hash: 'SHA-256' }, - false, - ['sign'] - ); - - const signature = await crypto.subtle.sign( - 'HMAC', - cryptoKey, - messageBuffer - ); - - return signature; -} - -// 获取签名 -async function getSignature(secret, date, region, service, stringToSign) { - const kDate = await hmacSha256('AWS4' + secret, date); - const kRegion = await hmacSha256(kDate, region); - const kService = await hmacSha256(kRegion, service); - const kSigning = await hmacSha256(kService, 'aws4_request'); - const signature = await hmacSha256(kSigning, stringToSign); - - return Array.from(new Uint8Array(signature)) - .map(b => b.toString(16).padStart(2, '0')) - .join(''); -} - -export default { - async fetch(request, env, ctx) { - const url = new URL(request.url); - const from = url.searchParams.get('from')?.toLowerCase(); - - // 只在没有 from 参数时才检查和使用缓存 - let cacheResponse; - if (!from) { - const cacheUrl = new URL(request.url); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - cacheResponse = await cache.match(cacheKey); - - if (cacheResponse) { - return cacheResponse; - } - } - - // 直接使用 GITLAB_CONFIGS 中的 name 作为 GitHub 仓库名 - const githubRepos = GITLAB_CONFIGS.map(config => config.name); - - const FILE = url.pathname.split('/').pop(); - - if (url.pathname === `/${CHECK_PASSWORD}`) { - const response = await listProjects(GITLAB_CONFIGS, githubRepos, GITHUB_USERNAME, GITHUB_PAT); - // 如果有 from 参数，添加禁止缓存的头部 - if (from) { - response.headers.append('Cache-Control', 'no-store, no-cache, must-revalidate, proxy-revalidate'); - response.headers.append('Pragma', 'no-cache'); - response.headers.append('Expires', '0'); - } else { - const cacheUrl = new URL(request.url); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - ctx.waitUntil(cache.put(cacheKey, response.clone())); - } - return response; - } - - const startTime = Date.now(); - // 根据不同的访问方式构建请求 - let requests = []; - - // R2 请求生成函数 - const generateR2Requests = async () => { - return Promise.all(R2_CONFIGS.map(async (r2Config) => { - const r2Path = `${r2Config.bucket}/${DIR}/${FILE}`; - const signedRequest = await getSignedUrl(r2Config, 'GET', r2Path); - return { - url: signedRequest.url, - headers: signedRequest.headers, - source: 'r2', - repo: `${r2Config.name} (${r2Config.bucket})` - }; - })); - }; - - if (from === 'where') { - // 获取文件信息模式 - const githubRequests = githubRepos.map(repo => ({ - url: `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found'); - const data = await response.json(); - return { - size: data.size, - exists: true - }; - } - })); - - const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found'); - const data = await response.json(); - const size = atob(data.content).length; - return { - size: size, - exists: true - }; - } - })); - - const r2Requests = await generateR2Requests(); - const r2WhereRequests = r2Requests.map(request => ({ - ...request, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found'); - const size = response.headers.get('content-length'); - return { - size: parseInt(size), - exists: true - }; - } - })); - - requests = [...githubRequests, ...gitlabRequests, ...r2WhereRequests]; - - } else { - // 获取文件内容模式 - if (from === 'github') { - requests = githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo - })); - } else if (from === 'gitlab') { - requests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name - })); - } else if (from === 'r2') { - requests = await generateR2Requests(); - } else { - // 如果没有指定来源，则从所有源获取 - const githubRequests = githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo - })); - - const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name - })); - - const r2Requests = await generateR2Requests(); - - requests = [...githubRequests, ...gitlabRequests, ...r2Requests]; - } - } - - // 发送请求并处理响应 - const fetchPromises = requests.map(request => { - const { url, headers, source, repo, processResponse } = request; - - return fetch(new Request(url, { - method: 'GET', - headers: headers - })).then(async response => { - if (from === 'where' && typeof processResponse === 'function') { - // 使用 `processResponse` 处理 where 查询逻辑 - try { - const result = await processResponse(response); - const endTime = Date.now(); - const duration = endTime - startTime; - - const formattedSize = result.size > 1024 * 1024 - ? `${(result.size / (1024 * 1024)).toFixed(2)} MB` - : `${(result.size / 1024).toFixed(2)} kB`; - - return { - fileName: FILE, - size: formattedSize, - source: `${source} (${repo})`, - duration: `${duration}ms` - }; - } catch (error) { - throw new Error(`Not found in ${source} (${repo})`); - } - } else { - // 对于内容获取，直接返回响应 - if (!response.ok) { - throw new Error(`Not found in ${source} (${repo})`); - } - return response; - } - }).catch(error => { - throw new Error(`Error in ${source} (${repo}): ${error.message}`); - }); - }); - - try { - if (requests.length === 0) { - throw new Error('No valid source specified'); - } - - const result = await Promise.any(fetchPromises); - - let response; - if (from === 'where') { - response = new Response(JSON.stringify(result, null, 2), { - headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-store, no-cache, must-revalidate, proxy-revalidate', - 'Pragma': 'no-cache', - 'Expires': '0' - } - }); - } else if (result instanceof Response) { - const blob = await result.blob(); - const headers = { - 'Content-Type': result.headers.get('Content-Type') || 'application/octet-stream', - 'Access-Control-Allow-Origin': '*' - }; - - // 如果有 from 参数，添加禁止缓存的头部 - if (from) { - headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, proxy-revalidate'; - headers['Pragma'] = 'no-cache'; - headers['Expires'] = '0'; - } - - response = new Response(blob, { - status: 200, - headers: headers - }); - } else { - throw new Error("Unexpected result type"); - } - - // 只在没有 from 参数时才缓存响应 - if (!from) { - const cacheUrl = new URL(request.url); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - ctx.waitUntil(cache.put(cacheKey, response.clone())); - } - - return response; - - } catch (error) { - const sourceText = from === 'where' - ? 'in any repository' - : from - ? `from ${from}` - : 'in the GitHub, GitLab and R2 storage'; - - const errorResponse = new Response( - `404: Cannot find the ${FILE} ${sourceText}.`, - { - status: 404, - headers: { - 'Content-Type': 'text/plain', - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-store, no-cache, must-revalidate, proxy-revalidate', - 'Pragma': 'no-cache', - 'Expires': '0' - } - } - ); - - return errorResponse; - } - } -}; - -// 列出所有节点仓库状态 -async function listProjects(gitlabConfigs, githubRepos, githubUsername, githubPat) { - let result = 'GitHub, GitLab and R2 Storage status:\n\n'; - - try { - // 并发执行所有检查 - const [username, ...allChecks] = await Promise.all([ - getGitHubUsername(githubPat), - ...githubRepos.map(repo => - checkGitHubRepo(githubUsername, repo, githubPat) - ), - ...gitlabConfigs.map(config => - checkGitLabProject(config.id, config.token) - ), - ...R2_CONFIGS.map(config => - checkR2Storage(config) - ) - ]); - - // 计算各类检查结果的数量 - const githubCount = githubRepos.length; - const gitlabCount = gitlabConfigs.length; - - // 分割检查结果 - const githubResults = allChecks.slice(0, githubCount); - const gitlabResults = allChecks.slice(githubCount, githubCount + gitlabCount); - const r2Results = allChecks.slice(githubCount + gitlabCount); - - // 添加 GitHub 结果 - githubRepos.forEach((repo, index) => { - const [status, fileCount, totalSize] = githubResults[index]; - const formattedSize = formatSize(totalSize); - result += `GitHub: ${repo} - ${status} (Username: ${username}, Files: ${fileCount}, Size: ${formattedSize})\n`; - }); - - // 添加 GitLab 结果 - gitlabConfigs.forEach((config, index) => { - const [status, username, fileCount] = gitlabResults[index]; - result += `GitLab: Project ID ${config.id} - ${status} (Username: ${username}, Files: ${fileCount})\n`; - }); - - // 添加 R2 结果 - r2Results.forEach(([status, name, bucket, fileCount, totalSize]) => { - result += `R2 Storage: ${name} - ${status} (Bucket: ${bucket}, Files: ${fileCount}, Size: ${totalSize})\n`; - }); - - } catch (error) { - result += `Error during status check: ${error.message}\n`; - } - - return new Response(result, { - headers: { 'Content-Type': 'text/plain' } - }); -} - -// 文件大小格式化函数 -function formatSize(sizeInBytes) { - if (sizeInBytes >= 1024 * 1024 * 1024) { - return `${(sizeInBytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; - } else if (sizeInBytes >= 1024 * 1024) { - return `${(sizeInBytes / (1024 * 1024)).toFixed(2)} MB`; - } else { - return `${(sizeInBytes / 1024).toFixed(2)} kB`; - } -} - -// 获取 GitHub 用户名的异步函数 -async function getGitHubUsername(pat) { - const url = 'https://api.github.com/user'; // GitHub 用户信息 API 地址 - try { - const response = await fetch(url, { - headers: { - 'Authorization': `token ${pat}`, // 使用个人访问令牌进行授权 - 'Accept': 'application/vnd.github.v3+json', // 指定接受的响应格式 - 'User-Agent': 'Cloudflare Worker' // 用户代理 - } - }); - - // 如果响应状态为 200，表示成功 - if (response.status === 200) { - const data = await response.json(); // 解析 JSON 数据 - return data.login; // 返回用户登录名 - } else { - console.error('GitHub API Error:', response.status); // 记录错误状态 - return 'Unknown'; // 返回未知状态 - } - } catch (error) { - console.error('GitHub request error:', error); // 记录请求错误 - return 'Error'; // 返回错误状态 - } -} - -// 检查 GitHub 仓库的异步函数 -async function checkGitHubRepo(owner, repo, pat) { - const repoUrl = `https://api.github.com/repos/${owner}/${repo}`; - const contentsUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${DIR}`; // 直接检查指定目录 - - const headers = { - 'Authorization': `token ${pat}`, - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'Cloudflare Worker' - }; - - try { - // 并发请求获取仓库信息和目录内容 - const [repoResponse, contentsResponse] = await Promise.all([ - fetch(repoUrl, { headers }), - fetch(contentsUrl, { headers }) - ]); - - const repoData = await repoResponse.json(); - - if (repoResponse.status !== 200) { - throw new Error(`Repository error: ${repoData.message}`); - } - - if (contentsResponse.status !== 200) { - return [`working (${repoData.private ? 'private' : 'public'})`, 0, 0]; - } - - const contentsData = await contentsResponse.json(); - - // 计算文件数量和总大小 - const fileStats = contentsData.reduce((acc, item) => { - if (item.type === 'file') { - return { - count: acc.count + 1, - size: acc.size + (item.size || 0) - }; - } - return acc; - }, { count: 0, size: 0 }); - - return [ - `working (${repoData.private ? 'private' : 'public'})`, - fileStats.count, - fileStats.size - ]; - - } catch (error) { - console.error(`Error checking GitHub repo ${repo}:`, error); - return [`error: ${error.message}`, 0, 0]; - } -} - -// 检查 GitLab 项目的异步函数 -async function checkGitLabProject(projectId, pat) { - const projectUrl = `https://gitlab.com/api/v4/projects/${projectId}`; - const filesUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?ref=main&path=${DIR}&recursive=true&per_page=10000`; - - try { - const [projectResponse, filesResponse] = await Promise.all([ - fetch(projectUrl, { - headers: { 'PRIVATE-TOKEN': pat } - }), - fetch(filesUrl, { - headers: { 'PRIVATE-TOKEN': pat } - }) - ]); - - if (projectResponse.status === 200) { - const projectData = await projectResponse.json(); - let fileCount = 0; - - if (filesResponse.status === 200) { - const filesData = await filesResponse.json(); - fileCount = filesData.filter(item => item.type === 'blob').length; - } - - return [ - `working (${projectData.visibility})`, - projectData.owner.username, - fileCount - ]; - } else if (projectResponse.status === 404) { - return ['not found', 'Unknown', 0]; - } else { - return ['disconnect', 'Unknown', 0]; - } - } catch (error) { - console.error('GitLab project check error:', error); - return ['disconnect', 'Error', 0]; - } -} - -// 检查 R2 存储状态 -async function checkR2Storage(r2Config) { - try { - // 1. 列出目录下所有文件 - const listPath = `${r2Config.bucket}`; // 列出根目录 - const signedRequest = await getSignedUrl(r2Config, 'GET', listPath); - - const response = await fetch(signedRequest.url, { - headers: signedRequest.headers - }); - - let fileCount = 0; - let totalSize = 0; - - if (response.ok) { - const data = await response.text(); - // 解析 XML 响应 - const keys = data.match(/([^<]+)<\/Key>/g) || []; - const sizes = data.match(/(\d+)<\/Size>/g) || []; - - // 只计算指定目录下的文件 - keys.forEach((key, index) => { - const filePath = key.replace(/|<\/Key>/g, ''); - if (filePath.startsWith(DIR + '/')) { - fileCount++; - const size = parseInt(sizes[index]?.replace(/|<\/Size>/g, '') || String(0), 10); - totalSize += size; - } - }); - } - - // 即使文件不存在，只要能访问到存储桶就认为是正常的 - const status = response.ok ? 'working' : 'error'; - - return [ - status, - r2Config.name, - r2Config.bucket, - fileCount, - formatSize(totalSize) - ]; - } catch (error) { - console.error('R2 Storage error:', error); - return ['error', r2Config.name, 'connection failed', 0, '0 B']; - } -} \ No newline at end of file diff --git a/cloudflare_worker/github_gitlab_s3.js b/cloudflare_worker/github_gitlab_r2_b2.js similarity index 59% rename from cloudflare_worker/github_gitlab_s3.js rename to cloudflare_worker/github_gitlab_r2_b2.js index 4795513..71adc77 100644 --- a/cloudflare_worker/github_gitlab_s3.js +++ b/cloudflare_worker/github_gitlab_r2_b2.js @@ -8,11 +8,11 @@ const GITLAB_CONFIGS = [ { name: '', id: '', token: '' }, // GitLab 账户4 ]; -// GitHub 配置 +// GitHub 配置，仓库名与 GitLab 的相同，故创建仓库时需要注意一定要对称 const GITHUB_USERNAME = ''; // GitHub 用户名 const GITHUB_PAT = ''; // GitHub 个人访问令牌 -// R2 存储配置 +// R2 存储配置，没有可以留空不填，但不要删除 const R2_CONFIGS = [ { name: '', // 帐户1 ID @@ -31,7 +31,7 @@ const R2_CONFIGS = [ // 可以添加更多 R2 配置 ]; -// B2 存储配置 +// B2 存储配置，没有可以留空不填，但不要删除 const B2_CONFIGS = [ { name: '', // 帐户1 名 @@ -62,6 +62,48 @@ const DELETE = 'true' // 可选 [true (默认) | false]，已复制到 GitHub // 用户配置区域结束 ================================= +// 检查配置是否有效 +function hasValidConfig() { + // 检查 GitHub 配置 + const hasGithub = GITHUB_PAT && GITHUB_USERNAME && GITLAB_CONFIGS && + GITLAB_CONFIGS.length > 0 && + GITLAB_CONFIGS.some(config => config.name && config.id && config.token); + + // 检查 GitLab 配置 + const hasGitlab = GITLAB_CONFIGS && + GITLAB_CONFIGS.length > 0 && + GITLAB_CONFIGS.some(config => config.name && config.id && config.token); + + // 检查 R2 配置 + const hasR2 = R2_CONFIGS && + R2_CONFIGS.length > 0 && + R2_CONFIGS.some(config => + config.name && + config.accountId && + config.accessKeyId && + config.secretAccessKey && + config.bucket + ); + + // 检查 B2 配置 + const hasB2 = B2_CONFIGS && + B2_CONFIGS.length > 0 && + B2_CONFIGS.some(config => + config.name && + config.endPoint && + config.keyId && + config.applicationKey && + config.bucket + ); + + return { + github: hasGithub, + gitlab: hasGitlab, + r2: hasR2, + b2: hasB2 + }; +} + // AWS SDK 签名相关函数开始 ================================= // 获取签名URL @@ -101,7 +143,7 @@ async function getSignedUrl(config, method, path) { ].join('\n'); const signature = await getSignature( - secretKey, // 使用映射后的密钥 + secretKey, date, region, service, @@ -109,7 +151,7 @@ async function getSignedUrl(config, method, path) { ); const authorization = [ - `AWS4-HMAC-SHA256 Credential=${accessKeyId}/${date}/${region}/${service}/aws4_request`, // 使用映射后的密钥ID + `AWS4-HMAC-SHA256 Credential=${accessKeyId}/${date}/${region}/${service}/aws4_request`, `SignedHeaders=host;x-amz-content-sha256;x-amz-date`, `Signature=${signature}` ].join(', '); @@ -132,10 +174,10 @@ async function sha256(message) { return Array.from(new Uint8Array(hashBuffer)) .map(b => b.toString(16).padStart(2, '0')) .join(''); - } +} - // HMAC-SHA256 函数 - async function hmacSha256(key, message) { +// HMAC-SHA256 函数 +async function hmacSha256(key, message) { const keyBuffer = key instanceof ArrayBuffer ? key : new TextEncoder().encode(key); const messageBuffer = new TextEncoder().encode(message); @@ -154,10 +196,10 @@ async function sha256(message) { ); return signature; - } +} - // 获取签名 - async function getSignature(secret, date, region, service, stringToSign) { +// 获取签名 +async function getSignature(secret, date, region, service, stringToSign) { const kDate = await hmacSha256('AWS4' + secret, date); const kRegion = await hmacSha256(kDate, region); const kService = await hmacSha256(kRegion, service); @@ -165,422 +207,41 @@ async function sha256(message) { const signature = await hmacSha256(kSigning, stringToSign); return Array.from(new Uint8Array(signature)) - .map(b => b.toString(16).padStart(2, '0')) - .join(''); - } - -export default { - async fetch(request, env, ctx) { - const url = new URL(request.url); - const from = url.searchParams.get('from')?.toLowerCase(); - - // 只在没有 from 参数时才检查和使用缓存 - let cacheResponse; - if (!from) { - // 检查缓存 - const cacheUrl = new URL(request.url); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - cacheResponse = await cache.match(cacheKey); - - if (cacheResponse) { - return cacheResponse; - } - } - - // 直接使用 GITLAB_CONFIGS 中的 name 作为 GitHub 仓库名 - const githubRepos = GITLAB_CONFIGS.map(config => config.name); - - const FILE = url.pathname.split('/').pop(); - - if (url.pathname === `/${CHECK_PASSWORD}`) { - const response = await listProjects(GITLAB_CONFIGS, githubRepos, GITHUB_USERNAME, GITHUB_PAT); - // 不缓存状态检查页面 - return response; - } - - const startTime = Date.now(); - - // 根据不同的访问方式构建请求 - let requests = []; - - // 在生成请求时需要合并 R2 和 B2 的请求 - const generateStorageRequests = async () => { - const r2Requests = await Promise.all(R2_CONFIGS.map(async (r2Config) => { - const r2Path = `${r2Config.bucket}/${DIR}/${FILE}`; - const signedRequest = await getSignedUrl(r2Config, 'GET', r2Path); - return { - url: signedRequest.url, - headers: signedRequest.headers, - source: 'r2', - repo: `${r2Config.name} (${r2Config.bucket})` - }; - })); - - const b2Requests = await Promise.all(B2_CONFIGS.map(async (b2Config) => { - const b2Path = `${b2Config.bucket}/${DIR}/${FILE}`; - const signedRequest = await getSignedUrl(b2Config, 'GET', b2Path); - return { - url: signedRequest.url, - headers: signedRequest.headers, - source: 'b2', - repo: `${b2Config.name} (${b2Config.bucket})` - }; - })); - - return [...r2Requests, ...b2Requests]; - }; - - if (from === 'where') { - // 获取文件信息模式 - const githubRequests = githubRepos.map(repo => ({ - url: `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found'); - const data = await response.json(); - return { - size: data.size, - exists: true - }; - } - })); - - const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found'); - const data = await response.json(); - const size = atob(data.content).length; - return { - size: size, - exists: true - }; - } - })); - - const r2Requests = await generateStorageRequests(); - const r2WhereRequests = r2Requests.map(request => ({ - ...request, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found'); - const size = response.headers.get('content-length'); - return { - size: parseInt(size), - exists: true - }; - } - })); - - requests = [...githubRequests, ...gitlabRequests, ...r2WhereRequests]; - - } else { - // 获取文件内容模式 - if (from === 'github') { - requests = githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo - })); - } else if (from === 'gitlab') { - requests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name - })); - } else if (from === 'r2') { - // 只从 R2 存储获取 - requests = await Promise.all(R2_CONFIGS.map(async (r2Config) => { - const r2Path = `${r2Config.bucket}/${DIR}/${FILE}`; - const signedRequest = await getSignedUrl(r2Config, 'GET', r2Path); - return { - url: signedRequest.url, - headers: signedRequest.headers, - source: 'r2', - repo: `${r2Config.name} (${r2Config.bucket})` - }; - })); - } else if (from === 'b2') { - // 只从 B2 存储获取 - requests = await Promise.all(B2_CONFIGS.map(async (b2Config) => { - const b2Path = `${b2Config.bucket}/${DIR}/${FILE}`; - const signedRequest = await getSignedUrl(b2Config, 'GET', b2Path); - return { - url: signedRequest.url, - headers: signedRequest.headers, - source: 'b2', - repo: `${b2Config.name} (${b2Config.bucket})` - }; - })); - } else { - // 如果没有指定来源，则从所有源获取 - const githubRequests = githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo - })); - - const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name - })); - - const r2Requests = await generateStorageRequests(); - - requests = [...githubRequests, ...gitlabRequests, ...r2Requests]; - } - } - - // 发送请求并处理响应 - const fetchPromises = requests.map(request => { - const { url, headers, source, repo, processResponse } = request; - - return fetch(new Request(url, { - method: 'GET', - headers: headers - })).then(async response => { - if (from === 'where' && typeof processResponse === 'function') { - // 使用 `processResponse` 处理 where 查询逻辑 - try { - const result = await processResponse(response); - const endTime = Date.now(); - const duration = endTime - startTime; - - const formattedSize = result.size > 1024 * 1024 - ? `${(result.size / (1024 * 1024)).toFixed(2)} MB` - : `${(result.size / 1024).toFixed(2)} kB`; - - return { - fileName: FILE, - size: formattedSize, - source: `${source} (${repo})`, - duration: `${duration}ms` - }; - } catch (error) { - throw new Error(`Not found in ${source} (${repo})`); - } - } else { - // 对于内容获取，直接返回响应 - if (!response.ok) { - throw new Error(`Not found in ${source} (${repo})`); - } - return response; - } - }).catch(error => { - throw new Error(`Error in ${source} (${repo}): ${error.message}`); - }); - }); - - try { - if (requests.length === 0) { - throw new Error('No valid source specified'); - } - - const result = await Promise.any(fetchPromises); - - let response; - if (from === 'where') { - // 如果是 where 查询，返回 JSON 格式响应 - response = new Response(JSON.stringify(result, null, 2), { - headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*' - } - }); - } else if (result instanceof Response) { - // 先读取响应体 - const blob = await result.blob(); - - // 创建新的响应，设置适当的头部 - const headers = { - 'Content-Type': result.headers.get('Content-Type') || 'application/octet-stream', - 'Access-Control-Allow-Origin': '*' - }; - - // 如果有 from 参数，添加禁止缓存的头部 - if (from) { - headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, proxy-revalidate'; - headers['Pragma'] = 'no-cache'; - headers['Expires'] = '0'; - } - - response = new Response(blob, { - status: 200, - headers: headers - }); - } else { - throw new Error("Unexpected result type"); - } - - // 只在没有 from 参数且不是 where 查询时才缓存响应 - if (!from && from !== 'where') { - const cacheUrl = new URL(request.url); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - ctx.waitUntil(cache.put(cacheKey, response.clone())); - } - - return response; - - } catch (error) { - const sourceText = from === 'where' - ? 'in any repository' - : from - ? `from ${from}` - : 'in the GitHub, GitLab, R2, and B2 storage'; - - const errorResponse = new Response( - `404: Cannot find the ${FILE} ${sourceText}.`, - { - status: 404, - headers: { - 'Content-Type': 'text/plain', - 'Access-Control-Allow-Origin': '*' - } - } - ); - - return errorResponse; - } - } -}; - -// 列出所有节点仓库状态 -async function listProjects(gitlabConfigs, githubRepos, githubUsername, githubPat) { - let result = 'GitHub, GitLab, R2 and B2 Storage status:\n\n'; - - try { - // 并发执行所有检查 - const [username, ...allChecks] = await Promise.all([ - getGitHubUsername(githubPat), - ...githubRepos.map(repo => - checkGitHubRepo(githubUsername, repo, githubPat) - ), - ...gitlabConfigs.map(config => - checkGitLabProject(config.id, config.token) - ), - ...R2_CONFIGS.map(config => - checkR2Storage(config) - ), - ...B2_CONFIGS.map(config => - checkB2Storage(config) - ) - ]); - - // 计算各类检查结果的数量和起始位置 - const githubCount = githubRepos.length; - const gitlabCount = gitlabConfigs.length; - const r2Count = R2_CONFIGS.length; - const b2Count = B2_CONFIGS.length; - - // 分割检查结果 - const githubResults = allChecks.slice(0, githubCount); - const gitlabResults = allChecks.slice(githubCount, githubCount + gitlabCount); - const r2Results = allChecks.slice(githubCount + gitlabCount, githubCount + gitlabCount + r2Count); - const b2Results = allChecks.slice(githubCount + gitlabCount + r2Count); - - // 添加 GitHub 结果 - githubRepos.forEach((repo, index) => { - const [status, fileCount, totalSize] = githubResults[index]; - const formattedSize = formatSize(totalSize); - result += `GitHub: ${repo} - ${status} (Username: ${username}, Files: ${fileCount}, Size: ${formattedSize})\n`; - }); - - // 添加 GitLab 结果 - gitlabConfigs.forEach((config, index) => { - const [status, username, fileCount] = gitlabResults[index]; - result += `GitLab: Project ID ${config.id} - ${status} (Username: ${username}, Files: ${fileCount})\n`; - }); - - // 添加 R2 结果 - r2Results.forEach(([status, name, bucket, fileCount, totalSize]) => { - result += `R2 Storage: ${name} - ${status} (Bucket: ${bucket}, Files: ${fileCount}, Size: ${totalSize})\n`; - }); - - // 添加 B2 结果 - b2Results.forEach(([status, name, bucket, fileCount, totalSize]) => { - result += `B2 Storage: ${name} - ${status} (Bucket: ${bucket}, Files: ${fileCount}, Size: ${totalSize})\n`; - }); - - } catch (error) { - result += `Error during status check: ${error.message}\n`; - } - - return new Response(result, { - headers: { - 'Content-Type': 'text/plain', - 'Access-Control-Allow-Origin': '*' - } - }); + .map(b => b.toString(16).padStart(2, '0')) + .join(''); } -// 文件大小格式化函数 -function formatSize(sizeInBytes) { - if (sizeInBytes >= 1024 * 1024 * 1024) { - return `${(sizeInBytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; - } else if (sizeInBytes >= 1024 * 1024) { - return `${(sizeInBytes / (1024 * 1024)).toFixed(2)} MB`; - } else { - return `${(sizeInBytes / 1024).toFixed(2)} kB`; - } -} +// AWS SDK 签名相关函数结束 ================================= -// 获取 GitHub 用户名的异步函数 +// 检查服务函数 async function getGitHubUsername(pat) { - const url = 'https://api.github.com/user'; // GitHub 用户信息 API 地址 + const url = 'https://api.github.com/user'; try { const response = await fetch(url, { headers: { - 'Authorization': `token ${pat}`, // 使用个人访问令牌进行授权 - 'Accept': 'application/vnd.github.v3+json', // 指定接受的响应格式 - 'User-Agent': 'Cloudflare Worker' // 用户代理 + 'Authorization': `token ${pat}`, + 'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'Cloudflare Worker' } }); - // 如果响应状态为 200，表示成功 if (response.status === 200) { - const data = await response.json(); // 解析 JSON 数据 - return data.login; // 返回用户登录名 + const data = await response.json(); + return data.login; } else { - console.error('GitHub API Error:', response.status); // 记录错误状态 - return 'Unknown'; // 返回未知状态 + console.error('GitHub API Error:', response.status); + return 'Unknown'; } } catch (error) { - console.error('GitHub request error:', error); // 记录请求错误 - return 'Error'; // 返回错误状态 + console.error('GitHub request error:', error); + return 'Error'; } } -// 检查 GitHub 仓库的异步函数 +// 检查 GitHub 仓库 async function checkGitHubRepo(owner, repo, pat) { const repoUrl = `https://api.github.com/repos/${owner}/${repo}`; - const contentsUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${DIR}`; // 直接检查指定目录 + const contentsUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${DIR}`; const headers = { 'Authorization': `token ${pat}`, @@ -630,7 +291,7 @@ async function checkGitHubRepo(owner, repo, pat) { } } -// 检查 GitLab 项目的异步函数 +// 检查 GitLab 项目 async function checkGitLabProject(projectId, pat) { const projectUrl = `https://gitlab.com/api/v4/projects/${projectId}`; const filesUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?ref=main&path=${DIR}&recursive=true&per_page=10000`; @@ -670,11 +331,10 @@ async function checkGitLabProject(projectId, pat) { } } -// 检查 R2 存储状态 +// 检查 R2 存储 async function checkR2Storage(r2Config) { try { - // 1. 列出目录下所有文件 - const listPath = `${r2Config.bucket}`; // 列出根目录 + const listPath = `${r2Config.bucket}`; const signedRequest = await getSignedUrl(r2Config, 'GET', listPath); const response = await fetch(signedRequest.url, { @@ -686,11 +346,9 @@ async function checkR2Storage(r2Config) { if (response.ok) { const data = await response.text(); - // 解析 XML 响应 const keys = data.match(/([^<]+)<\/Key>/g) || []; const sizes = data.match(/(\d+)<\/Size>/g) || []; - // 只计算指定目录下的文件 keys.forEach((key, index) => { const filePath = key.replace(/|<\/Key>/g, ''); if (filePath.startsWith(DIR + '/')) { @@ -701,7 +359,6 @@ async function checkR2Storage(r2Config) { }); } - // 即使文件不存在，只要能访问到存储桶就认为是正常的 const status = response.ok ? 'working' : 'error'; return [ @@ -717,11 +374,10 @@ async function checkR2Storage(r2Config) { } } -// 检查 B2 存储状态 +// 检查 B2 存储 async function checkB2Storage(b2Config) { try { - // 1. 列出目录下所有文件 - const listPath = `${b2Config.bucket}`; // 列出根目录 + const listPath = `${b2Config.bucket}`; const signedRequest = await getSignedUrl(b2Config, 'GET', listPath); const response = await fetch(signedRequest.url, { @@ -733,11 +389,9 @@ async function checkB2Storage(b2Config) { if (response.ok) { const data = await response.text(); - // 解析 XML 响应 const keys = data.match(/([^<]+)<\/Key>/g) || []; const sizes = data.match(/(\d+)<\/Size>/g) || []; - // 只计算指定目录下的文件 keys.forEach((key, index) => { const filePath = key.replace(/|<\/Key>/g, ''); if (filePath.startsWith(DIR + '/')) { @@ -748,7 +402,6 @@ async function checkB2Storage(b2Config) { }); } - // 即使文件不存在，只要能访问到存储桶就认为是正常的 const status = (response.status === 404 || response.status === 403 || response.ok) ? 'working' : 'error'; return [ @@ -762,4 +415,391 @@ async function checkB2Storage(b2Config) { console.error('B2 Storage error:', error); return ['error', b2Config.name, 'connection failed', 0, '0 B']; } +} + +// 文件大小格式化函数 +function formatSize(sizeInBytes) { + if (sizeInBytes >= 1024 * 1024 * 1024) { + return `${(sizeInBytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; + } else if (sizeInBytes >= 1024 * 1024) { + return `${(sizeInBytes / (1024 * 1024)).toFixed(2)} MB`; + } else { + return `${(sizeInBytes / 1024).toFixed(2)} kB`; + } +} + +export default { + async fetch(request, env, ctx) { + const url = new URL(request.url); + const from = url.searchParams.get('from')?.toLowerCase(); + const validConfigs = hasValidConfig(); + + // 直接使用 GITLAB_CONFIGS 中的 name 作为 GitHub 仓库名 + const githubRepos = GITLAB_CONFIGS.map(config => config.name); + const FILE = url.pathname.split('/').pop(); + + // 只在没有 from 参数时才检查和使用缓存 + let cacheResponse; + if (!from) { + const cacheUrl = new URL(request.url); + const cacheKey = new Request(cacheUrl.toString(), request); + const cache = caches.default; + cacheResponse = await cache.match(cacheKey); + + if (cacheResponse) { + return cacheResponse; + } + } + + // 检查状态页面 + if (url.pathname === `/${CHECK_PASSWORD}`) { + let result = ''; + let hasAnyValidConfig = false; + + try { + // GitHub 状态检查 + if (validConfigs.github) { + hasAnyValidConfig = true; + result += '=== GitHub Status ===\n'; + const username = await getGitHubUsername(GITHUB_PAT); + for (const repo of githubRepos) { + const [status, fileCount, totalSize] = await checkGitHubRepo(GITHUB_USERNAME, repo, GITHUB_PAT); + const formattedSize = formatSize(totalSize); + result += `GitHub: ${repo} - ${status} (Username: ${username}, Files: ${fileCount}, Size: ${formattedSize})\n`; + } + } + + // GitLab 状态检查 + if (validConfigs.gitlab) { + hasAnyValidConfig = true; + result += result ? '\n=== GitLab Status ===\n' : '=== GitLab Status ===\n'; + for (const config of GITLAB_CONFIGS) { + const [status, username, fileCount] = await checkGitLabProject(config.id, config.token); + result += `GitLab: Project ID ${config.id} - ${status} (Username: ${username}, Files: ${fileCount})\n`; + } + } + + // R2 状态检查 + if (validConfigs.r2) { + hasAnyValidConfig = true; + result += result ? '\n=== R2 Storage Status ===\n' : '=== R2 Storage Status ===\n'; + for (const config of R2_CONFIGS) { + const [status, name, bucket, fileCount, totalSize] = await checkR2Storage(config); + result += `R2 Storage: ${name} - ${status} (Bucket: ${bucket}, Files: ${fileCount}, Size: ${totalSize})\n`; + } + } + + // B2 状态检查 + if (validConfigs.b2) { + hasAnyValidConfig = true; + result += result ? '\n=== B2 Storage Status ===\n' : '=== B2 Storage Status ===\n'; + for (const config of B2_CONFIGS) { + const [status, name, bucket, fileCount, totalSize] = await checkB2Storage(config); + result += `B2 Storage: ${name} - ${status} (Bucket: ${bucket}, Files: ${fileCount}, Size: ${totalSize})\n`; + } + } + + // 如果没有任何有效配置 + if (!hasAnyValidConfig) { + result = 'No storage services configured.\n'; + } else { + result = 'Storage status:\n\n' + result; + } + + } catch (error) { + result += `Error during status check: ${error.message}\n`; + } + + return new Response(result, { + headers: { + 'Content-Type': 'text/plain', + 'Access-Control-Allow-Origin': '*' + } + }); + } + + const startTime = Date.now(); + let requests = []; + + // 检查特定服务的请求是否有效 + if (from) { + if (from === 'github' && !validConfigs.github) { + return new Response('GitHub service is not configured.', { + status: 400, + headers: { 'Content-Type': 'text/plain', 'Access-Control-Allow-Origin': '*' } + }); + } + if (from === 'gitlab' && !validConfigs.gitlab) { + return new Response('GitLab service is not configured.', { + status: 400, + headers: { 'Content-Type': 'text/plain', 'Access-Control-Allow-Origin': '*' } + }); + } + if (from === 'r2' && !validConfigs.r2) { + return new Response('R2 storage service is not configured.', { + status: 400, + headers: { 'Content-Type': 'text/plain', 'Access-Control-Allow-Origin': '*' } + }); + } + if (from === 'b2' && !validConfigs.b2) { + return new Response('B2 storage service is not configured.', { + status: 400, + headers: { 'Content-Type': 'text/plain', 'Access-Control-Allow-Origin': '*' } + }); + } + } + + // 生成存储请求 + const generateStorageRequests = async () => { + let requests = []; + + if (validConfigs.r2) { + const r2Requests = await Promise.all(R2_CONFIGS.map(async (r2Config) => { + const r2Path = `${r2Config.bucket}/${DIR}/${FILE}`; + const signedRequest = await getSignedUrl(r2Config, 'GET', r2Path); + return { + url: signedRequest.url, + headers: signedRequest.headers, + source: 'r2', + repo: `${r2Config.name} (${r2Config.bucket})` + }; + })); + requests = [...requests, ...r2Requests]; + } + + if (validConfigs.b2) { + const b2Requests = await Promise.all(B2_CONFIGS.map(async (b2Config) => { + const b2Path = `${b2Config.bucket}/${DIR}/${FILE}`; + const signedRequest = await getSignedUrl(b2Config, 'GET', b2Path); + return { + url: signedRequest.url, + headers: signedRequest.headers, + source: 'b2', + repo: `${b2Config.name} (${b2Config.bucket})` + }; + })); + requests = [...requests, ...b2Requests]; + } + + return requests; + }; + + // 处理不同类型的请求 + if (from === 'where') { + if (validConfigs.github) { + const githubRequests = githubRepos.map(repo => ({ + url: `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${DIR}/${FILE}`, + headers: { + 'Authorization': `token ${GITHUB_PAT}`, + 'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'Cloudflare Worker' + }, + source: 'github', + repo: repo, + processResponse: async (response) => { + if (!response.ok) throw new Error('Not found'); + const data = await response.json(); + return { + size: data.size, + exists: true + }; + } + })); + requests = [...requests, ...githubRequests]; + } + + if (validConfigs.gitlab) { + const gitlabRequests = GITLAB_CONFIGS.map(config => ({ + url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}?ref=main`, + headers: { + 'PRIVATE-TOKEN': config.token + }, + source: 'gitlab', + repo: config.name, + processResponse: async (response) => { + if (!response.ok) throw new Error('Not found'); + const data = await response.json(); + const size = atob(data.content).length; + return { + size: size, + exists: true + }; + } + })); + requests = [...requests, ...gitlabRequests]; + } + + const storageRequests = await generateStorageRequests(); + const storageWhereRequests = storageRequests.map(request => ({ + ...request, + processResponse: async (response) => { + if (!response.ok) throw new Error('Not found'); + const size = response.headers.get('content-length'); + return { + size: parseInt(size), + exists: true + }; + } + })); + requests = [...requests, ...storageWhereRequests]; + + } else { + // 获取文件内容模式 + if (from === 'github' && validConfigs.github) { + requests = githubRepos.map(repo => ({ + url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, + headers: { + 'Authorization': `token ${GITHUB_PAT}`, + 'User-Agent': 'Cloudflare Worker' + }, + source: 'github', + repo: repo + })); + } else if (from === 'gitlab' && validConfigs.gitlab) { + requests = GITLAB_CONFIGS.map(config => ({ + url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, + headers: { + 'PRIVATE-TOKEN': config.token + }, + source: 'gitlab', + repo: config.name + })); + } else if ((from === 'r2' && validConfigs.r2) || (from === 'b2' && validConfigs.b2)) { + requests = await generateStorageRequests(); + requests = requests.filter(req => req.source === from); + } else if (!from) { + if (validConfigs.github) { + const githubRequests = githubRepos.map(repo => ({ + url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, + headers: { + 'Authorization': `token ${GITHUB_PAT}`, + 'User-Agent': 'Cloudflare Worker' + }, + source: 'github', + repo: repo + })); + requests = [...requests, ...githubRequests]; + } + + if (validConfigs.gitlab) { + const gitlabRequests = GITLAB_CONFIGS.map(config => ({ + url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, + headers: { + 'PRIVATE-TOKEN': config.token + }, + source: 'gitlab', + repo: config.name + })); + requests = [...requests, ...gitlabRequests]; + } + + const storageRequests = await generateStorageRequests(); + requests = [...requests, ...storageRequests]; + } + } + + // 处理请求和响应 + try { + if (requests.length === 0) { + throw new Error('No valid source specified or no valid configurations found'); + } + + const fetchPromises = requests.map(request => { + const { url, headers, source, repo, processResponse } = request; + + return fetch(new Request(url, { + method: 'GET', + headers: headers + })).then(async response => { + if (from === 'where' && typeof processResponse === 'function') { + try { + const result = await processResponse(response); + const endTime = Date.now(); + const duration = endTime - startTime; + + const formattedSize = result.size > 1024 * 1024 + ? `${(result.size / (1024 * 1024)).toFixed(2)} MB` + : `${(result.size / 1024).toFixed(2)} kB`; + + return { + fileName: FILE, + size: formattedSize, + source: `${source} (${repo})`, + duration: `${duration}ms` + }; + } catch (error) { + throw new Error(`Not found in ${source} (${repo})`); + } + } else { + if (!response.ok) { + throw new Error(`Not found in ${source} (${repo})`); + } + return response; + } + }).catch(error => { + throw new Error(`Error in ${source} (${repo}): ${error.message}`); + }); + }); + + const result = await Promise.any(fetchPromises); + + let response; + if (from === 'where') { + response = new Response(JSON.stringify(result, null, 2), { + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*' + } + }); + } else if (result instanceof Response) { + const blob = await result.blob(); + const headers = { + 'Content-Type': result.headers.get('Content-Type') || 'application/octet-stream', + 'Access-Control-Allow-Origin': '*' + }; + + if (from) { + headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, proxy-revalidate'; + headers['Pragma'] = 'no-cache'; + headers['Expires'] = '0'; + } + + response = new Response(blob, { + status: 200, + headers: headers + }); + } else { + throw new Error("Unexpected result type"); + } + + if (!from && from !== 'where') { + const cacheUrl = new URL(request.url); + const cacheKey = new Request(cacheUrl.toString(), request); + const cache = caches.default; + ctx.waitUntil(cache.put(cacheKey, response.clone())); + } + + return response; + + } catch (error) { + const sourceText = from === 'where' + ? 'in any repository' + : from + ? `from ${from}` + : 'in any configured storage'; + + const errorResponse = new Response( + `404: Cannot find ${FILE} ${sourceText}. ${error.message}`, + { + status: 404, + headers: { + 'Content-Type': 'text/plain', + 'Access-Control-Allow-Origin': '*' + } + } + ); + + return errorResponse; + } + } } \ No newline at end of file From bcffcde7ab8c5c2078720a7402875e7196f40955 Mon Sep 17 00:00:00 2001 From: fscarmen2 Date: Thu, 19 Dec 2024 23:58:11 +0800 Subject: [PATCH 04/13] Support subdirectories. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 使用示例： 如果设置 DIR = 'dirA' 访问 https://mydomain.workers.dev/dirB/dirC/file.jpg 实际会访问存储中的 dirA/dirB/dirC/file.jpg --- cloudflare_worker/github_gitlab_r2_b2.js | 38 +++++++++++++++++------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/cloudflare_worker/github_gitlab_r2_b2.js b/cloudflare_worker/github_gitlab_r2_b2.js index 71adc77..76fc15f 100644 --- a/cloudflare_worker/github_gitlab_r2_b2.js +++ b/cloudflare_worker/github_gitlab_r2_b2.js @@ -238,6 +238,19 @@ async function getGitHubUsername(pat) { } } +// 修改文件路径处理函数 +function getFilePath(basePath, requestPath) { + // 移除开头的斜杠 + const cleanRequestPath = requestPath.replace(/^\//, ''); + + // 如果没有设置 basePath，直接返回请求路径 + if (!basePath) return cleanRequestPath; + + // 组合基础路径和请求路径 + return `${basePath}/${cleanRequestPath}`; +} + + // 检查 GitHub 仓库 async function checkGitHubRepo(owner, repo, pat) { const repoUrl = `https://api.github.com/repos/${owner}/${repo}`; @@ -434,9 +447,14 @@ export default { const from = url.searchParams.get('from')?.toLowerCase(); const validConfigs = hasValidConfig(); + // 获取完整的请求路径 + const requestPath = decodeURIComponent(url.pathname); + const FILE = requestPath.split('/').pop(); + const subPath = requestPath.substring(1, requestPath.lastIndexOf('/')); + const fullPath = DIR ? `${DIR}/${subPath}` : subPath; + // 直接使用 GITLAB_CONFIGS 中的 name 作为 GitHub 仓库名 const githubRepos = GITLAB_CONFIGS.map(config => config.name); - const FILE = url.pathname.split('/').pop(); // 只在没有 from 参数时才检查和使用缓存 let cacheResponse; @@ -555,7 +573,7 @@ export default { if (validConfigs.r2) { const r2Requests = await Promise.all(R2_CONFIGS.map(async (r2Config) => { - const r2Path = `${r2Config.bucket}/${DIR}/${FILE}`; + const r2Path = `${r2Config.bucket}/${fullPath}/${FILE}`; const signedRequest = await getSignedUrl(r2Config, 'GET', r2Path); return { url: signedRequest.url, @@ -569,7 +587,7 @@ export default { if (validConfigs.b2) { const b2Requests = await Promise.all(B2_CONFIGS.map(async (b2Config) => { - const b2Path = `${b2Config.bucket}/${DIR}/${FILE}`; + const b2Path = `${b2Config.bucket}/${fullPath}/${FILE}`; const signedRequest = await getSignedUrl(b2Config, 'GET', b2Path); return { url: signedRequest.url, @@ -588,7 +606,7 @@ export default { if (from === 'where') { if (validConfigs.github) { const githubRequests = githubRepos.map(repo => ({ - url: `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${DIR}/${FILE}`, + url: `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${fullPath}/${FILE}`, headers: { 'Authorization': `token ${GITHUB_PAT}`, 'Accept': 'application/vnd.github.v3+json', @@ -610,7 +628,7 @@ export default { if (validConfigs.gitlab) { const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}?ref=main`, + url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${fullPath}/${FILE}`)}?ref=main`, headers: { 'PRIVATE-TOKEN': config.token }, @@ -647,7 +665,7 @@ export default { // 获取文件内容模式 if (from === 'github' && validConfigs.github) { requests = githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, + url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${fullPath}/${FILE}`, headers: { 'Authorization': `token ${GITHUB_PAT}`, 'User-Agent': 'Cloudflare Worker' @@ -657,7 +675,7 @@ export default { })); } else if (from === 'gitlab' && validConfigs.gitlab) { requests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, + url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${fullPath}/${FILE}`)}/raw?ref=main`, headers: { 'PRIVATE-TOKEN': config.token }, @@ -670,7 +688,7 @@ export default { } else if (!from) { if (validConfigs.github) { const githubRequests = githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, + url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${fullPath}/${FILE}`, headers: { 'Authorization': `token ${GITHUB_PAT}`, 'User-Agent': 'Cloudflare Worker' @@ -683,7 +701,7 @@ export default { if (validConfigs.gitlab) { const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, + url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${fullPath}/${FILE}`)}/raw?ref=main`, headers: { 'PRIVATE-TOKEN': config.token }, @@ -802,4 +820,4 @@ export default { return errorResponse; } } -} \ No newline at end of file +} From 30ce3830f469f033fb1a6018108bc03dff562be3 Mon Sep 17 00:00:00 2001 From: fscarmen2 Date: Sat, 1 Feb 2025 09:42:46 +0000 Subject: [PATCH 05/13] Add interface for deleting a file. --- README.md | 18 +- cloudflare_worker/github_gitlab_r2_b2.js | 615 ++++++++++++++++++----- 2 files changed, 514 insertions(+), 119 deletions(-) diff --git a/README.md b/README.md index 5adb308..b20df29 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # 分布式存储集群 -## 更新日期 2024-12-18 +## 更新日期 2025-01-31 ## 各方案的独立分仓库 | 方案 | worker 文件 | 同步仓库模版 | 视频教程 | @@ -62,4 +62,18 @@ - **从 Cloudflare R2 获取** `https://<自定义域名>/<文件名>?from=r2` -- **从 Backblaze B2 获取** `https://<自定义域名>/<文件名>?from=b2` \ No newline at end of file +- **从 Backblaze B2 获取** `https://<自定义域名>/<文件名>?from=b2` + +## 从所有的平台删除指定文件 + +- **支持同时在 GitHub / GitLab / R2 / B2 多级子目录下的文件** ``https://<自定义域名>/delete?file=<文件名>` + +- **举例** 定义的节点目录为 files，而需要删除 `<节点>/files/a/b/test.jpg` + +``` +# 以下两个路径都可以 +https://<自定义域名>/delete?file=a/b/test.jpg +https://<自定义域名>/delete?file=/a/b/test.jpg +``` + +![image](https://github.com/user-attachments/assets/ccbd96df-f930-490b-a947-8df9dd9b8459) \ No newline at end of file diff --git a/cloudflare_worker/github_gitlab_r2_b2.js b/cloudflare_worker/github_gitlab_r2_b2.js index 76fc15f..8fa1902 100644 --- a/cloudflare_worker/github_gitlab_r2_b2.js +++ b/cloudflare_worker/github_gitlab_r2_b2.js @@ -65,34 +65,34 @@ const DELETE = 'true' // 可选 [true (默认) | false]，已复制到 GitHub // 检查配置是否有效 function hasValidConfig() { // 检查 GitHub 配置 - const hasGithub = GITHUB_PAT && GITHUB_USERNAME && GITLAB_CONFIGS && - GITLAB_CONFIGS.length > 0 && + const hasGithub = GITHUB_PAT && GITHUB_USERNAME && GITLAB_CONFIGS && + GITLAB_CONFIGS.length > 0 && GITLAB_CONFIGS.some(config => config.name && config.id && config.token); // 检查 GitLab 配置 - const hasGitlab = GITLAB_CONFIGS && - GITLAB_CONFIGS.length > 0 && + const hasGitlab = GITLAB_CONFIGS && + GITLAB_CONFIGS.length > 0 && GITLAB_CONFIGS.some(config => config.name && config.id && config.token); // 检查 R2 配置 - const hasR2 = R2_CONFIGS && - R2_CONFIGS.length > 0 && - R2_CONFIGS.some(config => - config.name && - config.accountId && - config.accessKeyId && - config.secretAccessKey && + const hasR2 = R2_CONFIGS && + R2_CONFIGS.length > 0 && + R2_CONFIGS.some(config => + config.name && + config.accountId && + config.accessKeyId && + config.secretAccessKey && config.bucket ); // 检查 B2 配置 - const hasB2 = B2_CONFIGS && - B2_CONFIGS.length > 0 && - B2_CONFIGS.some(config => - config.name && - config.endPoint && - config.keyId && - config.applicationKey && + const hasB2 = B2_CONFIGS && + B2_CONFIGS.length > 0 && + B2_CONFIGS.some(config => + config.name && + config.endPoint && + config.keyId && + config.applicationKey && config.bucket ); @@ -107,31 +107,40 @@ function hasValidConfig() { // AWS SDK 签名相关函数开始 ================================= // 获取签名URL -async function getSignedUrl(config, method, path) { - const region = 'auto'; +async function getSignedUrl(config, method, path, queryParams = {}) { + const region = config.endPoint ? config.endPoint.split('.')[1] : 'auto'; const service = 's3'; - - // 根据配置类型确定 host 和认证信息 - const host = config.endPoint - ? config.endPoint // B2 配置使用 endPoint - : `${config.accountId}.r2.cloudflarestorage.com`; // R2 配置使用默认格式 - - // 根据配置类型确定认证信息 + const host = config.endPoint || `${config.accountId}.r2.cloudflarestorage.com`; const accessKeyId = config.endPoint ? config.keyId : config.accessKeyId; const secretKey = config.endPoint ? config.applicationKey : config.secretAccessKey; - const datetime = new Date().toISOString().replace(/[:-]|\.\d{3}/g, ''); const date = datetime.substr(0, 8); + // 确保路径正确编码，但保留斜杠 + const encodedPath = path.split('/') + .map(segment => encodeURIComponent(segment)) + .join('/'); + + // 构建规范请求 + const canonicalHeaders = `host:${host}\nx-amz-content-sha256:UNSIGNED-PAYLOAD\nx-amz-date:${datetime}\n`; + const signedHeaders = 'host;x-amz-content-sha256;x-amz-date'; + + // 按字母顺序排序查询参数 + const sortedParams = Object.keys(queryParams).sort().reduce((acc, key) => { + acc[key] = queryParams[key]; + return acc; + }, {}); + + const canonicalQueryString = Object.entries(sortedParams) + .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`) + .join('&'); + const canonicalRequest = [ method, - '/' + path, - '', - `host:${host}`, - 'x-amz-content-sha256:UNSIGNED-PAYLOAD', - `x-amz-date:${datetime}`, - '', - 'host;x-amz-content-sha256;x-amz-date', + '/' + encodedPath, + canonicalQueryString, + canonicalHeaders, + signedHeaders, 'UNSIGNED-PAYLOAD' ].join('\n'); @@ -152,12 +161,14 @@ async function getSignedUrl(config, method, path) { const authorization = [ `AWS4-HMAC-SHA256 Credential=${accessKeyId}/${date}/${region}/${service}/aws4_request`, - `SignedHeaders=host;x-amz-content-sha256;x-amz-date`, + `SignedHeaders=${signedHeaders}`, `Signature=${signature}` ].join(', '); + const url = `https://${host}/${encodedPath}${canonicalQueryString ? '?' + canonicalQueryString : ''}`; + return { - url: `https://${host}/${path}`, + url, headers: { 'Authorization': authorization, 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', @@ -242,19 +253,17 @@ async function getGitHubUsername(pat) { function getFilePath(basePath, requestPath) { // 移除开头的斜杠 const cleanRequestPath = requestPath.replace(/^\//, ''); - + // 如果没有设置 basePath，直接返回请求路径 if (!basePath) return cleanRequestPath; - + // 组合基础路径和请求路径 return `${basePath}/${cleanRequestPath}`; } - // 检查 GitHub 仓库 async function checkGitHubRepo(owner, repo, pat) { const repoUrl = `https://api.github.com/repos/${owner}/${repo}`; - const contentsUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${DIR}`; const headers = { 'Authorization': `token ${pat}`, @@ -263,39 +272,43 @@ async function checkGitHubRepo(owner, repo, pat) { }; try { - // 并发请求获取仓库信息和目录内容 - const [repoResponse, contentsResponse] = await Promise.all([ - fetch(repoUrl, { headers }), - fetch(contentsUrl, { headers }) - ]); - + // 获取仓库信息，确定默认分支 + const repoResponse = await fetch(repoUrl, { headers }); const repoData = await repoResponse.json(); - if (repoResponse.status !== 200) { + if (repoResponse.status!== 200) { throw new Error(`Repository error: ${repoData.message}`); } - if (contentsResponse.status !== 200) { - return [`working (${repoData.private ? 'private' : 'public'})`, 0, 0]; + const defaultBranch = repoData.default_branch; + const contentsUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${defaultBranch}?recursive=1`; + + // 获取文件树信息 + const contentsResponse = await fetch(contentsUrl, { headers }); + if (contentsResponse.status!== 200) { + const contentsErrorData = await contentsResponse.json(); + throw new Error(`Contents error: ${contentsErrorData.message}`); } const contentsData = await contentsResponse.json(); - // 计算文件数量和总大小 - const fileStats = contentsData.reduce((acc, item) => { - if (item.type === 'file') { - return { - count: acc.count + 1, - size: acc.size + (item.size || 0) - }; + let fileCount = 0; + let totalSize = 0; + + if (contentsData.tree) { + for (const item of contentsData.tree) { + // 检查是否是文件 + if (item.type === 'blob' && (DIR === '' || item.path.startsWith(DIR + '/'))) { + fileCount++; + totalSize += item.size || 0; + } } - return acc; - }, { count: 0, size: 0 }); + } return [ `working (${repoData.private ? 'private' : 'public'})`, - fileStats.count, - fileStats.size + fileCount, + totalSize ]; } catch (error) { @@ -307,14 +320,14 @@ async function checkGitHubRepo(owner, repo, pat) { // 检查 GitLab 项目 async function checkGitLabProject(projectId, pat) { const projectUrl = `https://gitlab.com/api/v4/projects/${projectId}`; - const filesUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?ref=main&path=${DIR}&recursive=true&per_page=10000`; + const treeUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?recursive=true&per_page=100&path=${DIR}`; try { - const [projectResponse, filesResponse] = await Promise.all([ + const [projectResponse, treeResponse] = await Promise.all([ fetch(projectUrl, { headers: { 'PRIVATE-TOKEN': pat } }), - fetch(filesUrl, { + fetch(treeUrl, { headers: { 'PRIVATE-TOKEN': pat } }) ]); @@ -323,9 +336,10 @@ async function checkGitLabProject(projectId, pat) { const projectData = await projectResponse.json(); let fileCount = 0; - if (filesResponse.status === 200) { - const filesData = await filesResponse.json(); - fileCount = filesData.filter(item => item.type === 'blob').length; + if (treeResponse.status === 200) { + const treeData = await treeResponse.json(); + // 只计算文件，不计算目录 + fileCount = treeData.filter(item => item.type === 'blob').length; } return [ @@ -347,11 +361,17 @@ async function checkGitLabProject(projectId, pat) { // 检查 R2 存储 async function checkR2Storage(r2Config) { try { - const listPath = `${r2Config.bucket}`; - const signedRequest = await getSignedUrl(r2Config, 'GET', listPath); + // 列出所有文件 + const listRequest = await getSignedUrl(r2Config, 'GET', r2Config.bucket, { + 'list-type': '2', + 'prefix': DIR ? `${DIR}/` : '' // 添加目录前缀筛选 + }); - const response = await fetch(signedRequest.url, { - headers: signedRequest.headers + const response = await fetch(listRequest.url, { + headers: { + ...listRequest.headers, + 'Host': `${r2Config.accountId}.r2.cloudflarestorage.com` + } }); let fileCount = 0; @@ -359,23 +379,27 @@ async function checkR2Storage(r2Config) { if (response.ok) { const data = await response.text(); - const keys = data.match(/([^<]+)<\/Key>/g) || []; - const sizes = data.match(/(\d+)<\/Size>/g) || []; - keys.forEach((key, index) => { - const filePath = key.replace(/|<\/Key>/g, ''); - if (filePath.startsWith(DIR + '/')) { - fileCount++; - const size = parseInt(sizes[index]?.replace(/|<\/Size>/g, '') || String(0), 10); - totalSize += size; + // 使用正则表达式匹配所有文件信息 + const contents = data.match(/[\s\S]*?<\/Contents>/g) || []; + + for (const content of contents) { + const keyMatch = content.match(/([^<]+)<\/Key>/); + const sizeMatch = content.match(/(\d+)<\/Size>/); + + if (keyMatch && sizeMatch) { + const key = keyMatch[1]; + // 只计算文件，不计算目录 + if (!key.endsWith('/')) { + fileCount++; + totalSize += parseInt(sizeMatch[1]); + } } - }); + } } - const status = response.ok ? 'working' : 'error'; - return [ - status, + 'working', r2Config.name, r2Config.bucket, fileCount, @@ -390,11 +414,16 @@ async function checkR2Storage(r2Config) { // 检查 B2 存储 async function checkB2Storage(b2Config) { try { - const listPath = `${b2Config.bucket}`; - const signedRequest = await getSignedUrl(b2Config, 'GET', listPath); + // 构建列出文件的请求，移除 delimiter 参数以获取所有子目录 + const signedRequest = await getSignedUrl(b2Config, 'GET', b2Config.bucket, { + 'prefix': DIR ? `${DIR}/` : '' + }); const response = await fetch(signedRequest.url, { - headers: signedRequest.headers + headers: { + ...signedRequest.headers, + 'Host': b2Config.endPoint + } }); let fileCount = 0; @@ -402,31 +431,296 @@ async function checkB2Storage(b2Config) { if (response.ok) { const data = await response.text(); - const keys = data.match(/([^<]+)<\/Key>/g) || []; - const sizes = data.match(/(\d+)<\/Size>/g) || []; - - keys.forEach((key, index) => { - const filePath = key.replace(/|<\/Key>/g, ''); - if (filePath.startsWith(DIR + '/')) { + // 使用正则表达式匹配所有文件信息 + const keyRegex = /([^<]+)<\/Key>/g; + const sizeRegex = /(\d+)<\/Size>/g; + + let keyMatch; + while ((keyMatch = keyRegex.exec(data)) !== null) { + const key = keyMatch[1]; + // 只计算文件，不计算目录，并确保文件在指定目录下 + if (!key.endsWith('/') && (!DIR || key.startsWith(DIR + '/'))) { fileCount++; - const size = parseInt(sizes[index]?.replace(/|<\/Size>/g, '') || String(0), 10); - totalSize += size; + // 获取对应的文件大小 + const sizeMatch = /(\d+)<\/Size>/g.exec(data.slice(keyMatch.index)); + if (sizeMatch) { + totalSize += parseInt(sizeMatch[1]); + } } - }); - } + } - const status = (response.status === 404 || response.status === 403 || response.ok) ? 'working' : 'error'; + return [ + 'working', + b2Config.name, + b2Config.bucket, + fileCount, + formatSize(totalSize) + ]; + } else { + throw new Error(`Failed to list bucket: ${response.status} ${response.statusText}`); + } - return [ - status, - b2Config.name, - b2Config.bucket, - fileCount, - formatSize(totalSize) - ]; } catch (error) { console.error('B2 Storage error:', error); - return ['error', b2Config.name, 'connection failed', 0, '0 B']; + return ['error', b2Config.name, b2Config.bucket, 0, '0 B']; + } +} + +// 删除 GitHub 仓库中的文件 +async function deleteGitHubFile(repo, filePath, pat) { + // 构建完整的文件路径，包含 DIR + const fullPath = DIR ? `${DIR}/${filePath.replace(/^\/+/, '')}` : filePath.replace(/^\/+/, ''); + const url = `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${fullPath}`; + + try { + // 先检查文件是否存在 + const getResponse = await fetch(url, { + headers: { + 'Authorization': `token ${pat}`, + 'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'Cloudflare Worker' + } + }); + + if (getResponse.status === 404) { + return '文件不存在'; + } + + if (!getResponse.ok) { + const errorData = await getResponse.json(); + return `删除失败：(${errorData.message})`; + } + + const fileData = await getResponse.json(); + + // 执行删除操作 + const deleteResponse = await fetch(url, { + method: 'DELETE', + headers: { + 'Authorization': `token ${pat}`, + 'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'Cloudflare Worker' + }, + body: JSON.stringify({ + message: `Delete ${fullPath}`, + sha: fileData.sha + }) + }); + + if (deleteResponse.ok) { + return '删除成功'; + } else { + const errorData = await deleteResponse.json(); + return `删除失败：(${errorData.message})`; + } + } catch (error) { + console.error('GitHub delete error:', error); + return `删除失败：(${error.message})`; + } +} + +// 删除 GitLab 项目中的文件 +async function deleteGitLabFile(projectId, filePath, pat) { + // 构建完整的文件路径，包含 DIR + const fullPath = DIR ? `${DIR}/${filePath.replace(/^\/+/, '')}` : filePath.replace(/^\/+/, ''); + const encodedPath = encodeURIComponent(fullPath); + const url = `https://gitlab.com/api/v4/projects/${projectId}/repository/files/${encodedPath}`; + + try { + // 执行删除操作 + const deleteResponse = await fetch(url, { + method: 'DELETE', + headers: { + 'PRIVATE-TOKEN': pat, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + branch: 'main', + commit_message: 'Delete file: ' + fullPath + }) + }); + + // 获取响应数据 + const errorData = await deleteResponse.json().catch(() => ({})); + + // 处理文件不存在的所有可能情况 + if (deleteResponse.status === 404 || + errorData.message === 'A file with this name doesn\'t exist' || + errorData.message?.includes('file does not exist') || + errorData.message?.includes('File not found')) { + return '文件不存在'; + } + + // 处理删除成功的情况 + if (deleteResponse.ok || + errorData.message?.includes('reference update') || + errorData.message?.includes('reference does not point')) { + return '删除成功'; + } + + return `删除失败：(${errorData.message || '未知错误'})`; + } catch (error) { + console.error('GitLab delete error:', error); + if (error.message?.includes('file') && error.message?.includes('exist')) { + return '文件不存在'; + } + return `删除失败：(${error.message})`; + } +} + +// 删除 R2 存储中的文件 +async function deleteR2File(r2Config, filePath) { + // 构建完整的文件路径，包含 DIR + const fullPath = DIR ? `${DIR}/${filePath.replace(/^\/+/, '')}` : filePath.replace(/^\/+/, ''); + + try { + // 1. 首先列出所有文件 + const listRequest = await getSignedUrl(r2Config, 'GET', r2Config.bucket, { + 'list-type': '2', + 'prefix': fullPath // 使用精确的前缀匹配 + }); + + const listResponse = await fetch(listRequest.url, { + headers: { + ...listRequest.headers, + 'Host': `${r2Config.accountId}.r2.cloudflarestorage.com` + } + }); + + if (!listResponse.ok) { + throw new Error(`Failed to list objects: ${listResponse.statusText}`); + } + + // 解析响应 + const listData = await listResponse.text(); + const contents = listData.match(/[\s\S]*?<\/Contents>/g) || []; + let fileExists = false; + + // 精确匹配文件路径 + for (const content of contents) { + const keyMatch = content.match(/([^<]+)<\/Key>/); + if (keyMatch && keyMatch[1] === fullPath) { + fileExists = true; + break; + } + } + + if (!fileExists) { + return '文件不存在'; + } + + // 2. 删除文件 + const deleteRequest = await getSignedUrl(r2Config, 'DELETE', `${r2Config.bucket}/${fullPath}`); + + const deleteResponse = await fetch(deleteRequest.url, { + method: 'DELETE', + headers: { + ...deleteRequest.headers, + 'Host': `${r2Config.accountId}.r2.cloudflarestorage.com` + } + }); + + if (!deleteResponse.ok) { + const deleteResponseText = await deleteResponse.text(); + throw new Error(`Failed to delete: ${deleteResponse.status} - ${deleteResponseText}`); + } + + return '删除成功'; + } catch (error) { + console.error('R2 delete error:', error); + return `删除失败：(${error.message})`; + } +} + +// 删除 B2 存储中的文件 +async function deleteB2File(b2Config, filePath) { + // 构建完整的文件路径，包含 DIR + const fullPath = DIR ? `${DIR}/${filePath.replace(/^\/+/, '')}` : filePath.replace(/^\/+/, ''); + + try { + // 1. 首先列出所有文件 + const listObjectsRequest = await getSignedUrl(b2Config, 'GET', b2Config.bucket, { + 'list-type': '2', + 'prefix': fullPath + }); + + const listResponse = await fetch(listObjectsRequest.url, { + headers: { + ...listObjectsRequest.headers, + 'Host': b2Config.endPoint + } + }); + + if (!listResponse.ok) { + throw new Error(`Failed to list objects: ${listResponse.statusText}`); + } + + // 解析 XML 响应 + const listData = await listResponse.text(); + const keyRegex = /([^<]+)<\/Key>/g; + const fileExists = Array.from(listData.matchAll(keyRegex)) + .some(match => match[1] === fullPath); + + if (!fileExists) { + return '文件不存在'; + } + + // 2. 获取文件的版本信息 + const versionsRequest = await getSignedUrl(b2Config, 'GET', b2Config.bucket, { + 'versions': '', + 'prefix': fullPath, + 'list-type': '2' + }); + + const versionsResponse = await fetch(versionsRequest.url, { + headers: { + ...versionsRequest.headers, + 'Host': b2Config.endPoint, + 'x-amz-date': versionsRequest.headers['x-amz-date'], + 'Authorization': versionsRequest.headers['Authorization'] + } + }); + + if (!versionsResponse.ok) { + const responseText = await versionsResponse.text(); + console.error('Version listing response:', responseText); + throw new Error(`Failed to list versions: ${versionsResponse.status} - ${responseText}`); + } + + const versionsData = await versionsResponse.text(); + + // 解析版本信息 + const versionMatch = versionsData.match(/[\s\S]*?([^<]+)<\/VersionId>[\s\S]*?<\/Version>/); + if (!versionMatch) { + throw new Error('No version information found'); + } + + const versionId = versionMatch[1]; + + // 3. 删除指定版本的文件 + const deleteRequest = await getSignedUrl(b2Config, 'DELETE', `${b2Config.bucket}/${fullPath}`, { + 'versionId': versionId + }); + + const deleteResponse = await fetch(deleteRequest.url, { + method: 'DELETE', + headers: { + ...deleteRequest.headers, + 'Host': b2Config.endPoint, + 'x-amz-date': deleteRequest.headers['x-amz-date'], + 'Authorization': deleteRequest.headers['Authorization'] + } + }); + + if (!deleteResponse.ok) { + const deleteResponseText = await deleteResponse.text(); + throw new Error(`Failed to delete: ${deleteResponse.status} - ${deleteResponseText}`); + } + + return '删除成功'; + } catch (error) { + console.error('B2 delete error:', error); + return `删除失败：(${error.message})`; } } @@ -450,7 +744,9 @@ export default { // 获取完整的请求路径 const requestPath = decodeURIComponent(url.pathname); const FILE = requestPath.split('/').pop(); - const subPath = requestPath.substring(1, requestPath.lastIndexOf('/')); + // 获取子目录路径，移除开头和结尾的斜杠 + const subPath = requestPath.substring(1, requestPath.lastIndexOf('/')) + .replace(/^\/+|\/+$/g, ''); const fullPath = DIR ? `${DIR}/${subPath}` : subPath; // 直接使用 GITLAB_CONFIGS 中的 name 作为 GitHub 仓库名 @@ -536,6 +832,67 @@ export default { }); } + // 添加删除路由 + if (url.pathname === '/delete') { + const file = url.searchParams.get('file'); + if (!file) { + return new Response('Missing "file" parameter', { + status: 400, + headers: { 'Content-Type': 'text/plain; charset=UTF-8', 'Access-Control-Allow-Origin': '*' } + }); + } + + const validConfigs = hasValidConfig(); + let result = `Delete：${file}\n`; + + // GitHub 状态 + if (validConfigs.github) { + result += '\n=== GitHub Status ===\n'; + const githubRepos = GITLAB_CONFIGS.map(config => config.name); + for (const repo of githubRepos) { + const status = await deleteGitHubFile(repo, file, GITHUB_PAT); + result += `GitHub: ${repo} - working (private) ${status}\n`; + } + } + + // GitLab 状态 + if (validConfigs.gitlab) { + result += '\n=== GitLab Status ===\n'; + for (const config of GITLAB_CONFIGS) { + const status = await deleteGitLabFile(config.id, file, config.token); + const projectData = await fetch(`https://gitlab.com/api/v4/projects/${config.id}`, { + headers: { 'PRIVATE-TOKEN': config.token } + }).then(res => res.json()); + result += `GitLab: Project ID ${config.id} - working (${projectData.visibility}) ${status}\n`; + } + } + + // R2 存储状态 + if (validConfigs.r2) { + result += '\n=== R2 Storage Status ===\n'; + for (const config of R2_CONFIGS) { + const status = await deleteR2File(config, file); + result += `R2 Storage: ${config.name} - working ${status}\n`; + } + } + + // B2 存储状态 + if (validConfigs.b2) { + result += '\n=== B2 Storage Status ===\n'; + for (const config of B2_CONFIGS) { + const status = await deleteB2File(config, file); + result += `B2 Storage: ${config.name} - working ${status}\n`; + } + } + + return new Response(result, { + headers: { + 'Content-Type': 'text/plain; charset=UTF-8', + 'Access-Control-Allow-Origin': '*' + } + }); + } + const startTime = Date.now(); let requests = []; @@ -568,16 +925,27 @@ export default { } // 生成存储请求 - const generateStorageRequests = async () => { + async function generateStorageRequests() { let requests = []; + // 处理请求路径，保留子目录结构 + const getStoragePath = (filePath) => { + return filePath.replace(/^\/+/, '').replace(/\/+/g, '/'); + }; + if (validConfigs.r2) { const r2Requests = await Promise.all(R2_CONFIGS.map(async (r2Config) => { - const r2Path = `${r2Config.bucket}/${fullPath}/${FILE}`; + // 构建包含子目录的完整路径 + const storagePath = getStoragePath(`${subPath}/${FILE}`); + const r2Path = `${r2Config.bucket}/${DIR}/${storagePath}`; + const signedRequest = await getSignedUrl(r2Config, 'GET', r2Path); return { url: signedRequest.url, - headers: signedRequest.headers, + headers: { + ...signedRequest.headers, + 'Accept': '*/*' + }, source: 'r2', repo: `${r2Config.name} (${r2Config.bucket})` }; @@ -587,11 +955,24 @@ export default { if (validConfigs.b2) { const b2Requests = await Promise.all(B2_CONFIGS.map(async (b2Config) => { - const b2Path = `${b2Config.bucket}/${fullPath}/${FILE}`; - const signedRequest = await getSignedUrl(b2Config, 'GET', b2Path); + // 构建完整路径，注意 B2 需要包含 bucket 名称 + const storagePath = getStoragePath(`${subPath}/${FILE}`); + const b2Path = `${b2Config.bucket}/${DIR}/${storagePath}`; + + const signedRequest = await getSignedUrl({ + endPoint: b2Config.endPoint, + keyId: b2Config.keyId, + applicationKey: b2Config.applicationKey, + bucket: b2Config.bucket + }, 'GET', b2Path); + return { url: signedRequest.url, - headers: signedRequest.headers, + headers: { + ...signedRequest.headers, + 'Host': b2Config.endPoint, + 'Accept': '*/*' + }, source: 'b2', repo: `${b2Config.name} (${b2Config.bucket})` }; @@ -600,7 +981,7 @@ export default { } return requests; - }; + } // 处理不同类型的请求 if (from === 'where') { @@ -628,7 +1009,7 @@ export default { if (validConfigs.gitlab) { const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${fullPath}/${FILE}`)}?ref=main`, + url: `https://gitlab.com/api/v4/projects/${config.id}/repository//${encodeURIComponent(`${fullPath}/${FILE}`)}?ref=main`, headers: { 'PRIVATE-TOKEN': config.token }, @@ -675,7 +1056,7 @@ export default { })); } else if (from === 'gitlab' && validConfigs.gitlab) { requests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${fullPath}/${FILE}`)}/raw?ref=main`, + url: `https://gitlab.com/api/v4/projects/${config.id}/repository/${DIR}/${encodeURIComponent(`${fullPath}/${FILE}`)}/raw?ref=main`, headers: { 'PRIVATE-TOKEN': config.token }, @@ -701,7 +1082,7 @@ export default { if (validConfigs.gitlab) { const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${fullPath}/${FILE}`)}/raw?ref=main`, + url: `https://gitlab.com/api/v4/projects/${config.id}/repository/${DIR}/${encodeURIComponent(`${fullPath}/${FILE}`)}/raw?ref=main`, headers: { 'PRIVATE-TOKEN': config.token }, @@ -820,4 +1201,4 @@ export default { return errorResponse; } } -} +} \ No newline at end of file From 5bfe691881253289eb5a3babffd6e77bf908ac74 Mon Sep 17 00:00:00 2001 From: fscarmen2 Date: Sun, 2 Feb 2025 14:10:46 +0000 Subject: [PATCH 06/13] The deletion operation now requires the correct ${CHECK_PASSWORD}. --- README.md | 8 ++-- cloudflare_worker/github_gitlab_r2_b2.js | 58 +++++++++++++++++++++--- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b20df29..d0f0408 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ - **同时使用 GitHub, GitLab 和 R2**: `github_gitlab_r2_b2.js` - **同时使用 GitHub, GitLab，R2 和 B2**: `github_gitlab_r2_b2.js` -## 检测节点状态 `https://<自定义域名>/` +## 检测节点状态 `https://<自定义域名>/<自定义密码>` image @@ -66,14 +66,14 @@ ## 从所有的平台删除指定文件 -- **支持同时在 GitHub / GitLab / R2 / B2 多级子目录下的文件** ``https://<自定义域名>/delete?file=<文件名>` +- **支持同时在 GitHub / GitLab / R2 / B2 多级子目录下的文件** ``https://<自定义域名>/<自定义密码>/del?file=<文件名>` - **举例** 定义的节点目录为 files，而需要删除 `<节点>/files/a/b/test.jpg` ``` # 以下两个路径都可以 -https://<自定义域名>/delete?file=a/b/test.jpg -https://<自定义域名>/delete?file=/a/b/test.jpg +https://<自定义域名>/<自定义密码>/delete?file=a/b/test.jpg +https://<自定义域名>/<自定义密码>/del?file=/a/b/test.jpg ``` ![image](https://github.com/user-attachments/assets/ccbd96df-f930-490b-a947-8df9dd9b8459) \ No newline at end of file diff --git a/cloudflare_worker/github_gitlab_r2_b2.js b/cloudflare_worker/github_gitlab_r2_b2.js index 8fa1902..180aaa6 100644 --- a/cloudflare_worker/github_gitlab_r2_b2.js +++ b/cloudflare_worker/github_gitlab_r2_b2.js @@ -737,19 +737,64 @@ function formatSize(sizeInBytes) { export default { async fetch(request, env, ctx) { + // 获取请求 URL 对象 const url = new URL(request.url); + + // 从 URL 的查询参数中获取 'from' 参数并转换为小写 const from = url.searchParams.get('from')?.toLowerCase(); + + // 检查是否有有效的配置，调用 `hasValidConfig()` 函数 const validConfigs = hasValidConfig(); - // 获取完整的请求路径 + // 获取请求路径并解码（对 URL 编码进行解码） const requestPath = decodeURIComponent(url.pathname); + + // 从路径中提取文件名（即路径的最后一部分） const FILE = requestPath.split('/').pop(); + // 获取子目录路径，移除开头和结尾的斜杠 - const subPath = requestPath.substring(1, requestPath.lastIndexOf('/')) - .replace(/^\/+|\/+$/g, ''); + const subPath = requestPath.substring(1, requestPath.lastIndexOf('/')).replace(/^\/+|\/+$/g, ''); + + // 如果 DIR 存在，拼接 DIR 和子目录路径；否则仅使用子目录路径 const fullPath = DIR ? `${DIR}/${subPath}` : subPath; - // 直接使用 GITLAB_CONFIGS 中的 name 作为 GitHub 仓库名 + // 检查请求路径是否匹配删除请求（支持 'delete' 或 'del'） + const isDeleteRequest = requestPath.match(new RegExp(`^/${CHECK_PASSWORD}/(delete|del)$`)); + + // 检查是否是未授权的删除请求 + const isUnauthorizedDelete = requestPath.match(/^\/(delete|del)$/); + if (isUnauthorizedDelete) { + const file = url.searchParams.get('file'); + if (!file) { + return new Response( + '需要指定要删除的文件。\n' + + '正确的删除格式为: /<自定义密码>/del?file=文件路径\n' + + '例如: /<自定义密码>/del?file=example.png', + { + status: 403, + headers: { + 'Content-Type': 'text/plain; charset=UTF-8', + 'Access-Control-Allow-Origin': '*' + } + } + ); + } + + return new Response( + '需要密码验证才能删除文件。\n' + + '要删除文件 ' + file + ' 的正确格式为:\n' + + '/<自定义密码>/del?file=' + file, + { + status: 403, + headers: { + 'Content-Type': 'text/plain; charset=UTF-8', + 'Access-Control-Allow-Origin': '*' + } + } + ); + } + + // 从 GITLAB_CONFIGS 中获取每个配置的 name 作为 GitHub 仓库名 const githubRepos = GITLAB_CONFIGS.map(config => config.name); // 只在没有 from 参数时才检查和使用缓存 @@ -826,14 +871,14 @@ export default { return new Response(result, { headers: { - 'Content-Type': 'text/plain', + 'Content-Type': 'text/plain; charset=UTF-8', 'Access-Control-Allow-Origin': '*' } }); } // 添加删除路由 - if (url.pathname === '/delete') { + if (isDeleteRequest) { const file = url.searchParams.get('file'); if (!file) { return new Response('Missing "file" parameter', { @@ -842,7 +887,6 @@ export default { }); } - const validConfigs = hasValidConfig(); let result = `Delete：${file}\n`; // GitHub 状态 From 83adbc857b4eea22a08f761245cf0a2fe4a829a8 Mon Sep 17 00:00:00 2001 From: fscarmen2 Date: Thu, 6 Feb 2025 01:33:47 +0000 Subject: [PATCH 07/13] Fix bug where requesting data from Gitlab fails. Add project description. --- cloudflare_worker/github_gitlab_r2_b2.js | 39 ++++++++++++++++++++---- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/cloudflare_worker/github_gitlab_r2_b2.js b/cloudflare_worker/github_gitlab_r2_b2.js index 180aaa6..0807f1c 100644 --- a/cloudflare_worker/github_gitlab_r2_b2.js +++ b/cloudflare_worker/github_gitlab_r2_b2.js @@ -749,6 +749,31 @@ export default { // 获取请求路径并解码（对 URL 编码进行解码） const requestPath = decodeURIComponent(url.pathname); + // 添加根路径项目介绍 + if (requestPath === '/') { + return new Response( + '欢迎来到文件托管集群！(File Hosting Cluster)\n' + + '这是一个分布式存储集群项目，旨在提供高效的文件存储和管理服务。\n\n' + + '项目链接： https://github.com/fscarmen2/pic-hosting-cluster\n' + + '视频介绍： https://youtu.be/5i-86oBLWP8\n\n' + + '您可以使用以下操作：\n' + + '1. 从集群所有节点获取文件： /<文件名>\n' + + '2. 指定从 Github 获取文件： /<文件名>?from=github\n' + + '3. 指定从 Gitlab 获取文件： /<文件名>?from=gitlab\n' + + '4. 指定从 Cloudflare R2 获取文件： /<文件名>?from=r2\n' + + '5. 指定从 Backblaze B2 获取文件： /<文件名>?from=b2\n' + + '6. 查找文件信息： /<文件名>?from=where\n' + + '7. 查各节点状态： /<自定义密码>\n' + + '8. 删除文件： /<自定义密码>/del?file=<文件名>', + { + headers: { + 'Content-Type': 'text/plain; charset=UTF-8', + 'Access-Control-Allow-Origin': '*' + } + } + ); + } + // 从路径中提取文件名（即路径的最后一部分） const FILE = requestPath.split('/').pop(); @@ -1031,7 +1056,7 @@ export default { if (from === 'where') { if (validConfigs.github) { const githubRequests = githubRepos.map(repo => ({ - url: `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${fullPath}/${FILE}`, + url: `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${getFilePath(DIR, `${subPath}/${FILE}`)}`, headers: { 'Authorization': `token ${GITHUB_PAT}`, 'Accept': 'application/vnd.github.v3+json', @@ -1053,7 +1078,8 @@ export default { if (validConfigs.gitlab) { const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository//${encodeURIComponent(`${fullPath}/${FILE}`)}?ref=main`, + // GitLab where 查询 URL + url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(getFilePath(DIR, `${subPath}/${FILE}`))}?ref=main`, headers: { 'PRIVATE-TOKEN': config.token }, @@ -1062,9 +1088,8 @@ export default { processResponse: async (response) => { if (!response.ok) throw new Error('Not found'); const data = await response.json(); - const size = atob(data.content).length; return { - size: size, + size: data.size, exists: true }; } @@ -1100,7 +1125,8 @@ export default { })); } else if (from === 'gitlab' && validConfigs.gitlab) { requests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/${DIR}/${encodeURIComponent(`${fullPath}/${FILE}`)}/raw?ref=main`, + // GitLab 文件获取 URL + url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(getFilePath(DIR, `${subPath}/${FILE}`))}/raw?ref=main`, headers: { 'PRIVATE-TOKEN': config.token }, @@ -1126,7 +1152,8 @@ export default { if (validConfigs.gitlab) { const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/${DIR}/${encodeURIComponent(`${fullPath}/${FILE}`)}/raw?ref=main`, + // GitLab URL 构建方式 + url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(getFilePath(DIR, `${subPath}/${FILE}`))}/raw?ref=main`, headers: { 'PRIVATE-TOKEN': config.token }, From 59dc3ccbe36ad4234056a94887df65c733e5f6b6 Mon Sep 17 00:00:00 2001 From: notagshen <106027330+notagshen@users.noreply.github.com> Date: Mon, 21 Apr 2025 21:50:53 +0800 Subject: [PATCH 08/13] Fix: Double slash in R2 path construction causing 404 errors (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: const r2Path = `${r2Config.bucket}/${DIR}/${storagePath}`; 当 DIR 为空时，生成的路径变成 bucket//subfolder/file.ext，这在 S3 兼容的 API 中被视为不同于 bucket/subfolder/file.ext 的路径，因此找不到文件。 Solution: const r2Path = DIR ? `${r2Config.bucket}/${DIR}/${storagePath}` : `${r2Config.bucket}/${storagePath}`; 修改路径构建逻辑，增加处理dir为空的逻辑，如果为空则直接拼接 bucket 和 storagePath。 --- cloudflare_worker/github_gitlab_r2_b2.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudflare_worker/github_gitlab_r2_b2.js b/cloudflare_worker/github_gitlab_r2_b2.js index 0807f1c..06d0fea 100644 --- a/cloudflare_worker/github_gitlab_r2_b2.js +++ b/cloudflare_worker/github_gitlab_r2_b2.js @@ -1006,8 +1006,8 @@ export default { const r2Requests = await Promise.all(R2_CONFIGS.map(async (r2Config) => { // 构建包含子目录的完整路径 const storagePath = getStoragePath(`${subPath}/${FILE}`); - const r2Path = `${r2Config.bucket}/${DIR}/${storagePath}`; - + // 检查 DIR 是否为空，如果为空则直接拼接 bucket 和 storagePath。 + const r2Path = DIR ? `${r2Config.bucket}/${DIR}/${storagePath}` : `${r2Config.bucket}/${storagePath}`; const signedRequest = await getSignedUrl(r2Config, 'GET', r2Path); return { url: signedRequest.url, @@ -1272,4 +1272,4 @@ export default { return errorResponse; } } -} \ No newline at end of file +} From 1f8faab48c8b7949db1b41f48622dfdfc62d2352 Mon Sep 17 00:00:00 2001 From: fscarmen2 Date: Tue, 22 Apr 2025 21:04:09 +0800 Subject: [PATCH 09/13] Update README.md --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d0f0408..feda493 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # 分布式存储集群 -## 更新日期 2025-01-31 +## 更新日期 2025-04-21 ## 各方案的独立分仓库 | 方案 | worker 文件 | 同步仓库模版 | 视频教程 | @@ -76,4 +76,8 @@ https://<自定义域名>/<自定义密码>/delete?file=a/b/test.jpg https://<自定义域名>/<自定义密码>/del?file=/a/b/test.jpg ``` -![image](https://github.com/user-attachments/assets/ccbd96df-f930-490b-a947-8df9dd9b8459) \ No newline at end of file +![image](https://github.com/user-attachments/assets/ccbd96df-f930-490b-a947-8df9dd9b8459) + +## Sponsors + +This project is generously sponsored by VTEXS. From f8579552d939d93fe66871e35090ff15bcb8aa29 Mon Sep 17 00:00:00 2001 From: fscarmen2 Date: Tue, 3 Jun 2025 00:06:44 +0800 Subject: [PATCH 10/13] Thanks to [Digitalvirt](https://digitalvirt.com/) --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index feda493..7e56625 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ | B2 + R2 + GitHub + GitLab | [worker.js](https://raw.githubusercontent.com/fscarmen2/pic-hosting-cluster/refs/heads/main/cloudflare_worker/github_gitlab_r2_b2.js) | [点击使用模板库，注意需要手动改为私有仓库](https://github.com/new?template_name=files-hosting-template-2&template_owner=fscarmen2) | https://youtu.be/4X1FjLCAckI | ## GitHub --→ GitLab 目录下1个文件，放到 GitHub 库: -- **AC 脚本**: `./github/workflows/cluster_sync.yml` +- **AC 脚本**: `.github/workflows/cluster_sync.yml` ## GitLab --→ GitHub 目录下3个文件，放到 GitLab 库: - **配置文件**: `config.yml` @@ -20,11 +20,11 @@ - **CI/CD 脚本**: `.gitlab-ci.yml` ## R2 --→ GitHub 目录下1个文件，放到 GitHub 库: -- **AC 脚本**: `./github/workflows/r2_to_github.yml` +- **AC 脚本**: `.github/workflows/r2_to_github.yml` - **设置3个secrets**: `ACCOUNT_ID`, `WORKER_NAME` 和 `API_TOKEN` ## S3 (R2+B2) --→ GitHub 目录下1个文件，放到 GitHub 库: -- **AC 脚本**: `./github/workflows/s3_to_github.yml` +- **AC 脚本**: `.github/workflows/s3_to_github.yml` - **设置3个secrets**: `ACCOUNT_ID`, `WORKER_NAME` 和 `API_TOKEN` ## ACCOUNT_ID,WORKER_NAME,API_TOKEN 获取方式 @@ -80,4 +80,8 @@ https://<自定义域名>/<自定义密码>/del?file=/a/b/test.jpg ## Sponsors -This project is generously sponsored by VTEXS. +This project is generously sponsored by [VTEXS](https://zmto.com/). + +This project is generously sponsored by [Digitalvirt](https://digitalvirt.com/). + +![image](https://digitalvirt.com/templates/BlueWhite/img/logo-dark.svg) From 0eccb818878fd81c7cf9290e66bdb5b446264310 Mon Sep 17 00:00:00 2001 From: fscarmen2 Date: Wed, 30 Jul 2025 13:26:37 +0800 Subject: [PATCH 11/13] Thanks to [Dartnode](https://dartnode.com) --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 7e56625..2473277 100644 --- a/README.md +++ b/README.md @@ -85,3 +85,15 @@ This project is generously sponsored by [VTEXS](https://zmto.com/). This project is generously sponsored by [Digitalvirt](https://digitalvirt.com/). ![image](https://digitalvirt.com/templates/BlueWhite/img/logo-dark.svg) + + + + +
+ + + + + +I am honored that DARTNODE is offering a free server to sponsor my project.
+DARTNODE's official Web Site : [https://dartnode.com](https://dartnode.com?aff=CraftyMouse750) From e4dd10af9eb27e97598fc3366c8f4e82e431b3ea Mon Sep 17 00:00:00 2001 From: fscarmen2 Date: Sun, 31 Aug 2025 21:21:31 +0800 Subject: [PATCH 12/13] Thanks to [vps.town](https://vps.town/) --- README.md | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2473277..c34e45f 100644 --- a/README.md +++ b/README.md @@ -80,18 +80,27 @@ https://<自定义域名>/<自定义密码>/del?file=/a/b/test.jpg ## Sponsors -This project is generously sponsored by [VTEXS](https://zmto.com/). +- This project is generously sponsored by [VTEXS](https://zmto.com/). -This project is generously sponsored by [Digitalvirt](https://digitalvirt.com/). +- 感谢 vps.town 对本项目的支持和赞助 -![image](https://digitalvirt.com/templates/BlueWhite/img/logo-dark.svg) + + Sponsor + + +- 感谢 digitalvirt.com 对本项目的支持和赞助 + + + Sponsor + +- 感谢 dartnode.com 对本项目的支持和赞助 - +
- + From fe9ea7e8100b1b7b379a4182a82098fcc97d8016 Mon Sep 17 00:00:00 2001 From: fscarmen2 Date: Thu, 26 Mar 2026 05:32:35 +0000 Subject: [PATCH 13/13] fix: support CJK filenames and fix URL encoding across all three Cloudflare Worker scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes in github_gitlab_r2_b2.js: - Add encodePathSegments() and buildEncodedPath() to encode path segments individually - Fix double-slash bug when DIR is empty (main//filename → main/filename) - Wrap decodeURIComponent in try/catch to prevent crashes on malformed URLs - Apply correct encoding to all GitHub raw URLs and GitHub API (where mode) URLs - Split CHECK_PASSWORD into CUSTOM_PASSWORD + CHECK_PASSWORD to avoid "always truthy" lint warning - Add @type {string} annotation to DIR to suppress "types have no overlap" lint warning Changes in github_only.js: - Decode requestPath with try/catch before extracting FILE, enabling CJK filename support - Add encodePathSegments() helper and rewrite URL builder to filter empty DIR, avoiding double slashes Changes in gitlab_only.js: - Encode each path segment individually in filePath construction instead of joining then encoding, preserving multi-level directory slashes - Fix getFileSizeFromGitLab to split file.path by "/" and encode each segment, preventing "/" from being encoded as "%2F" --- 修复：在三个 Cloudflare Worker 脚本中支持中日韩文件名及 URL 编码问题 github_gitlab_r2_b2.js 的改动： - 新增 encodePathSegments() 和 buildEncodedPath()，对路径逐段编码，支持中文/日文/韩文文件名 - 修复双斜杠问题：DIR 为空时不再产生 main//文件名 - decodeURIComponent 加 try/catch 保护，防止畸形 URL 导致 Worker 崩溃 - 所有 GitHub raw URL 及 GitHub API（where 模式）均应用了新编码逻辑 - 将 CHECK_PASSWORD 拆分为 CUSTOM_PASSWORD + CHECK_PASSWORD，消除"always truthy"静态检查警告 - 为 DIR 添加 @type {string} 注解，消除"types have no overlap"警告 github_only.js 的改动： - 对 url.pathname 做带 try/catch 保护的解码后再提取 FILE，支持中文文件名 - 新增 encodePathSegments() 并重写 URL 构建逻辑，过滤空 DIR，避免 raw URL 出现双斜杠 gitlab_only.js 的改动： - filePath 构建改为对 pathParts 逐段编码再拼接，修复整体编码时 / 被错误编码为 %2F 的问题 - 修复 getFileSizeFromGitLab：file.path 按 / 分割后逐段编码，正确保留多级目录的斜杠结构 --- cloudflare_worker/github_gitlab_r2_b2.js | 43 ++++++++++++++++++------ cloudflare_worker/github_only.js | 23 +++++++++---- cloudflare_worker/gitlab_only.js | 6 ++-- 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/cloudflare_worker/github_gitlab_r2_b2.js b/cloudflare_worker/github_gitlab_r2_b2.js index 06d0fea..34ac276 100644 --- a/cloudflare_worker/github_gitlab_r2_b2.js +++ b/cloudflare_worker/github_gitlab_r2_b2.js @@ -51,10 +51,11 @@ const B2_CONFIGS = [ ]; // 定义集群访问目录 -const DIR = ''; +const DIR = /** @type {string} */ (''); // 定义集群里全部节点连接状态的密码验证，区分大小写（优先使用自定义密码，若为空则使用 GITHUB_PAT） -const CHECK_PASSWORD = '' || GITHUB_PAT; +const CUSTOM_PASSWORD = ''; +const CHECK_PASSWORD = CUSTOM_PASSWORD || GITHUB_PAT; // GitHub 备份策略 const STRATEGY = 'size' // 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 @@ -746,8 +747,27 @@ export default { // 检查是否有有效的配置，调用 `hasValidConfig()` 函数 const validConfigs = hasValidConfig(); - // 获取请求路径并解码（对 URL 编码进行解码） - const requestPath = decodeURIComponent(url.pathname); + // 获取请求路径： + // url.pathname 在 Cloudflare Worker 中已是 Unicode 字符串，直接使用即可。 + // 使用 decodeURIComponent 兜底处理仍为百分号编码的情况，同时捕获异常防止恶意编码导致崩溃。 + let requestPath; + try { + requestPath = decodeURIComponent(url.pathname); + } catch (e) { + // 解码失败时直接使用原始 pathname + requestPath = url.pathname; + } + + // 对路径各段重新编码，用于拼接到 URL 中（支持中文/日文/韩文等非 ASCII 文件名） + // 过滤空段，避免 DIR 或 subPath 为空时产生多余斜杠（如 main//文件名） + function encodePathSegments(path) { + return path.split('/').filter(seg => seg !== '').map(seg => encodeURIComponent(seg)).join('/'); + } + + // 构建用于拼接 URL 的编码路径，过滤空值，避免双斜杠 + function buildEncodedPath(...parts) { + return parts.filter(p => p && p !== '').map(p => encodePathSegments(p)).join('/'); + } // 添加根路径项目介绍 if (requestPath === '/') { @@ -1056,7 +1076,8 @@ export default { if (from === 'where') { if (validConfigs.github) { const githubRequests = githubRepos.map(repo => ({ - url: `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${getFilePath(DIR, `${subPath}/${FILE}`)}`, + // buildEncodedPath 过滤空段，避免双斜杠；encodePathSegments 处理中文/日文/韩文 + url: `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${encodePathSegments(getFilePath(DIR, `${subPath}/${FILE}`))}`, headers: { 'Authorization': `token ${GITHUB_PAT}`, 'Accept': 'application/vnd.github.v3+json', @@ -1078,7 +1099,7 @@ export default { if (validConfigs.gitlab) { const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - // GitLab where 查询 URL + // GitLab where 查询 URL（encodeURIComponent 对整个路径编码，GitLab API 要求） url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(getFilePath(DIR, `${subPath}/${FILE}`))}?ref=main`, headers: { 'PRIVATE-TOKEN': config.token @@ -1115,7 +1136,8 @@ export default { // 获取文件内容模式 if (from === 'github' && validConfigs.github) { requests = githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${fullPath}/${FILE}`, + // buildEncodedPath 过滤空段，避免双斜杠，支持中文/日文/韩文文件名 + url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${buildEncodedPath(fullPath, FILE)}`, headers: { 'Authorization': `token ${GITHUB_PAT}`, 'User-Agent': 'Cloudflare Worker' @@ -1125,7 +1147,7 @@ export default { })); } else if (from === 'gitlab' && validConfigs.gitlab) { requests = GITLAB_CONFIGS.map(config => ({ - // GitLab 文件获取 URL + // GitLab 文件获取 URL（encodeURIComponent 对整个路径编码） url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(getFilePath(DIR, `${subPath}/${FILE}`))}/raw?ref=main`, headers: { 'PRIVATE-TOKEN': config.token @@ -1139,7 +1161,8 @@ export default { } else if (!from) { if (validConfigs.github) { const githubRequests = githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${fullPath}/${FILE}`, + // buildEncodedPath 过滤空段，避免双斜杠，支持中文/日文/韩文文件名 + url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${buildEncodedPath(fullPath, FILE)}`, headers: { 'Authorization': `token ${GITHUB_PAT}`, 'User-Agent': 'Cloudflare Worker' @@ -1152,7 +1175,7 @@ export default { if (validConfigs.gitlab) { const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - // GitLab URL 构建方式 + // GitLab URL 构建方式（encodeURIComponent 对整个路径编码） url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(getFilePath(DIR, `${subPath}/${FILE}`))}/raw?ref=main`, headers: { 'PRIVATE-TOKEN': config.token diff --git a/cloudflare_worker/github_only.js b/cloudflare_worker/github_only.js index 02d3538..d9264c4 100644 --- a/cloudflare_worker/github_only.js +++ b/cloudflare_worker/github_only.js @@ -154,8 +154,17 @@ export default { ); } - // 原有的文件获取逻辑保持不变 - const FILE = url.pathname.split('/').pop(); + // 获取请求路径并解码，支持中文/日文/韩文等非 ASCII 文件名 + let requestPath; + try { + requestPath = decodeURIComponent(url.pathname); + } catch (e) { + requestPath = url.pathname; + } + const FILE = requestPath.split('/').pop(); + // 对路径各段重新编码，过滤空段避免双斜杠（如 DIR 为空时 main//文件名） + const encodePathSegments = (path) => + path.split('/').filter(seg => seg !== '').map(seg => encodeURIComponent(seg)).join('/'); // 缓存检查 const cacheUrl = new URL(request.url); @@ -167,10 +176,12 @@ export default { return cacheResponse; } - // 构建 GitHub raw 文件的 URL 列表 - const urls = REPOS.map(repoNumber => - `https://raw.githubusercontent.com/${CONFIG.GITHUB_USERNAME}/${CONFIG.GITHUB_REPO_PREFIX}${repoNumber}/main/${CONFIG.DIR}/${FILE}` - ); + // 构建 GitHub raw 文件的 URL 列表，使用 encodePathSegments 支持中文/日文/韩文文件名，过滤空段避免双斜杠 + const urls = REPOS.map(repoNumber => { + const parts = [CONFIG.DIR, FILE].filter(p => p && p !== ''); + const encodedPath = parts.map(p => encodePathSegments(p)).join('/'); + return `https://raw.githubusercontent.com/${CONFIG.GITHUB_USERNAME}/${CONFIG.GITHUB_REPO_PREFIX}${repoNumber}/main/${encodedPath}`; + }); // 创建并发请求 const requests = urls.map(githubUrl => { diff --git a/cloudflare_worker/gitlab_only.js b/cloudflare_worker/gitlab_only.js index 7e2c08c..3e551bb 100644 --- a/cloudflare_worker/gitlab_only.js +++ b/cloudflare_worker/gitlab_only.js @@ -111,7 +111,8 @@ async function checkGitLabProject(projectId, pat) { // CONCURRENT_REQUESTS: 控制同时发起的并发请求数，防止对 GitLab API 施加过大压力 const CONCURRENT_REQUESTS = 100; const sizes = await asyncPool(CONCURRENT_REQUESTS, filesData, async (file) => { - const encodedPath = `/${encodeURIComponent(file.path)}`; + // 逐段编码路径，支持中文/日文/韩文文件名，同时保留多级目录的斜杠结构 + const encodedPath = '/' + file.path.split('/').map(seg => encodeURIComponent(seg)).join('/'); const size = await getFileSizeFromGitLab(projectId, encodedPath, pat); return size; }); @@ -204,7 +205,8 @@ export default { } // 构建文件路径 - const filePath = pathParts.slice(1).join('/'); + // 对路径各段逐段编码，支持中文/日文/韩文等非 ASCII 文件名，保留多级目录结构 + const filePath = pathParts.slice(1).map(seg => encodeURIComponent(seg)).join('/'); console.log('Accessing file:', filePath); // 检查文件是否存在