diff --git a/README.md b/README.md index fbe3bb2..c34e45f 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,54 @@ -# R2，GitHub 和 GitLab 分布式存储集群 +# 分布式存储集群 -- **方案的配置文件都是 `config.yml`，只需要根据实际修改** +## 更新日期 2025-04-21 -## GitHub --→ GitLab 目录下2个文件，放到 GitHub 库: -- **配置文件**: `config.yml` -- **AC 脚本**: `./github/workflows/cluster_sync.yml` +## 各方案的独立分仓库 +| 方案 | worker 文件 | 同步仓库模版 | 视频教程 | +| --- |--- |--- |--- | +| GitHub only | [worker.js](https://raw.githubusercontent.com/fscarmen2/pic-hosting-cluster/refs/heads/main/cloudflare_worker/github_only.js) | [博文 7.2 Github 设置](https://www.fscarmen.com/2024/10/blog-post.html) | https://youtu.be/eRqIpeeo9SA | +| GitLab only |[worker.js](https://raw.githubusercontent.com/fscarmen2/pic-hosting-cluster/refs/heads/main/cloudflare_worker/gitlab_only.js) | 使用 GitLab 平台自带的镜像功能 | https://youtu.be/tjiI3I3MkaQ | +| GitHub + GitLab | [worker.js](https://raw.githubusercontent.com/fscarmen2/pic-hosting-cluster/refs/heads/main/cloudflare_worker/github_gitlab_r2_b2.js) | [点击使用模板库，注意需要手动改为私有仓库](https://github.com/new?template_name=files-hosting-template-1&template_owner=fscarmen2) | https://youtu.be/SGex7xJ9YdQ | +| R2 + GitHub + GitLab | [worker.js](https://raw.githubusercontent.com/fscarmen2/pic-hosting-cluster/refs/heads/main/cloudflare_worker/github_gitlab_r2_b2.js) | [点击使用模板库，注意需要手动改为私有仓库](https://github.com/new?template_name=files-hosting-template-2&template_owner=fscarmen2) | https://youtu.be/5i-86oBLWP8 | +| B2 + R2 + GitHub + GitLab | [worker.js](https://raw.githubusercontent.com/fscarmen2/pic-hosting-cluster/refs/heads/main/cloudflare_worker/github_gitlab_r2_b2.js) | [点击使用模板库，注意需要手动改为私有仓库](https://github.com/new?template_name=files-hosting-template-2&template_owner=fscarmen2) | https://youtu.be/4X1FjLCAckI | + +## GitHub --→ GitLab 目录下1个文件，放到 GitHub 库: +- **AC 脚本**: `.github/workflows/cluster_sync.yml` ## GitLab --→ GitHub 目录下3个文件，放到 GitLab 库: - **配置文件**: `config.yml` - **同步脚本**: `sync_to_github.sh` - **CI/CD 脚本**: `.gitlab-ci.yml` -## R2 --→ GitHub 目录下2个文件，放到 GitHub 库: -- **配置文件**: `config.yml` -- **AC 脚本**: `./github/workflows/r2_to_github.yml` +## R2 --→ GitHub 目录下1个文件，放到 GitHub 库: +- **AC 脚本**: `.github/workflows/r2_to_github.yml` +- **设置3个secrets**: `ACCOUNT_ID`, `WORKER_NAME` 和 `API_TOKEN` -## S2 (R2+B2) --→ GitHub 目录下2个文件，放到 GitHub 库: -- **配置文件**: `config.yml` -- **AC 脚本**: `./github/workflows/s3_to_github.yml` +## S3 (R2+B2) --→ GitHub 目录下1个文件，放到 GitHub 库: +- **AC 脚本**: `.github/workflows/s3_to_github.yml` +- **设置3个secrets**: `ACCOUNT_ID`, `WORKER_NAME` 和 `API_TOKEN` + +## ACCOUNT_ID,WORKER_NAME,API_TOKEN 获取方式 + +### 在 Cloudflare 面板创建可以读取 Worker 项目的 API, https://dash.cloudflare.com/profile/api-tokens + +![image](https://github.com/user-attachments/assets/9e49b29a-54ae-46f0-aeda-28d95f4a9041) +![image](https://github.com/user-attachments/assets/11dceb4b-ab2e-41a8-b8e4-7317bcf4b50f) +![image](https://github.com/user-attachments/assets/b1e6f1c3-3d8d-4ba3-8d98-35ab4f061b14) +![image](https://github.com/user-attachments/assets/81e66642-cd5c-43d3-bb72-7fecf24e16a3) +![image](https://github.com/user-attachments/assets/3c832e81-bfc6-480d-939c-1d0731a07c17) -## Cloudflare worker 目录下5个文件，复制代码到 worker 处： +### 在 Action 处设置 3 个 secret 变量 + +![image](https://github.com/user-attachments/assets/25b8d0fa-8302-4cb9-a6db-83e449e9664c) + +## Cloudflare worker 目录下3个文件，复制代码到 worker 处： - **只使用 GitHub**: `github_only.js` - **只使用 GitLab**: `gitlab_only.js` -- **同时使用 GitHub 和 GitLab**: `github_gitlab.js` -- **同时使用 GitHub, GitLab 和 R2**: `github_gitlab_r2.js` -- **同时使用 GitHub, GitLab，R2 和 B2**: `github_gitlab_s3.js` - +- **同时使用 GitHub 和 GitLab**: `github_gitlab_r2_b2.js` +- **同时使用 GitHub, GitLab 和 R2**: `github_gitlab_r2_b2.js` +- **同时使用 GitHub, GitLab，R2 和 B2**: `github_gitlab_r2_b2.js` -## 检测节点状态 `https://<自定义域名>/` +## 检测节点状态 `https://<自定义域名>/<自定义密码>` image @@ -41,4 +62,47 @@ - **从 Cloudflare R2 获取** `https://<自定义域名>/<文件名>?from=r2` -- **从 Backblaze B2 获取** `https://<自定义域名>/<文件名>?from=b2` \ No newline at end of file +- **从 Backblaze B2 获取** `https://<自定义域名>/<文件名>?from=b2` + +## 从所有的平台删除指定文件 + +- **支持同时在 GitHub / GitLab / R2 / B2 多级子目录下的文件** ``https://<自定义域名>/<自定义密码>/del?file=<文件名>` + +- **举例** 定义的节点目录为 files，而需要删除 `<节点>/files/a/b/test.jpg` + +``` +# 以下两个路径都可以 +https://<自定义域名>/<自定义密码>/delete?file=a/b/test.jpg +https://<自定义域名>/<自定义密码>/del?file=/a/b/test.jpg +``` + +![image](https://github.com/user-attachments/assets/ccbd96df-f930-490b-a947-8df9dd9b8459) + +## Sponsors + +- This project is generously sponsored by [VTEXS](https://zmto.com/). + +- 感谢 vps.town 对本项目的支持和赞助 + + + Sponsor + + +- 感谢 digitalvirt.com 对本项目的支持和赞助 + + + Sponsor + + +- 感谢 dartnode.com 对本项目的支持和赞助 + + + +
+ + + + + +I am honored that DARTNODE is offering a free server to sponsor my project.
+DARTNODE's official Web Site : [https://dartnode.com](https://dartnode.com?aff=CraftyMouse750) diff --git a/cloudflare_worker/github_gitlab.js b/cloudflare_worker/github_gitlab.js deleted file mode 100644 index f63738e..0000000 --- a/cloudflare_worker/github_gitlab.js +++ /dev/null @@ -1,537 +0,0 @@ -// 用户配置区域开始 ================================= - -// GitLab 仓库配置列表，包含仓库名称、ID 和访问 Token -const GITLAB_CONFIGS = [ - { name: 'repo1', id: 'repoID1', token: 'repoToken1' }, // 节点1配置 - { name: 'repo2', id: 'repoID2', token: 'repoToken2' }, // 节点2配置 - { name: 'repo3', id: 'repoID3', token: 'repoToken3' }, // 节点3配置 - { name: 'repo4', id: 'repoID4', token: 'repoToken4' }, // 节点4配置 -]; - -// GitHub 仓库列表，留空表示与 GitLab 的仓库名称一致 -const GITHUB_REPOS = ['']; // 如果需要指定 GitHub 仓库名，可以在此处填写。如果 GitHub 节点与 GitLab 不一样，可以用 ['ghRepo1','ghRepo2','ghRepo3','ghRepo4'] - -// GitHub 用户名和个人访问令牌（PAT） -const GITHUB_USERNAME = ''; // 填写 GitHub 用户名 -const GITHUB_PAT = ''; // 填写 GitHub 访问令牌 - -// 定义集群访问目录 -const DIR = ''; // 存储的文件目录，可以为空，表示根目录 - -// 定义集群里全部节点连接状态的密码验证，区分大小写（优先使用自定义密码，若为空则使用 GITHUB_PAT） -const CHECK_PASSWORD = '' || GITHUB_PAT; - -// 定义全局缓存时间，单位为秒，默认值为一年（31556952秒） -const CACHE_MAX_AGE = 31556952; // 一年 - -// 用户配置区域结束 ================================= - -// Cloudflare Worker 主函数，处理 HTTP 请求 -export default { - async fetch(request, env, ctx) { - const url = new URL(request.url); // 获取请求的 URL - const from = url.searchParams.get('from')?.toLowerCase(); // 获取来源参数 - - // 只在没有 from 参数时才检查和使用缓存 - let cacheResponse; - if (!from) { - const cacheUrl = new URL(request.url); // 获取请求 URL 的对象表示 - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - cacheResponse = await cache.match(cacheKey); - - if (cacheResponse) { - // 如果有缓存，直接返回缓存内容 - return cacheResponse; - } - } - - // 验证 GitHub 仓库列表是否有效 - const isValidGithubRepos = Array.isArray(GITHUB_REPOS) && - GITHUB_REPOS.length > 0 && - GITHUB_REPOS.some(repo => repo.trim() !== ''); - - // 如果 GitHub 仓库列表无效，使用 GitLab 的仓库名称 - const githubRepos = isValidGithubRepos - ? GITHUB_REPOS.filter(repo => repo.trim() !== '') - : GITLAB_CONFIGS.map(config => config.name); - - const FILE = url.pathname.split('/').pop(); // 获取请求文件名 - - if (url.pathname === `/${CHECK_PASSWORD}`) { - const result = await listProjects(GITLAB_CONFIGS, githubRepos, GITHUB_USERNAME, GITHUB_PAT); - // 如果有 from 参数，添加禁止缓存的头部 - if (from) { - result.headers.append('Cache-Control', 'no-store, no-cache, must-revalidate, proxy-revalidate'); - result.headers.append('Pragma', 'no-cache'); - result.headers.append('Expires', '0'); - } else { - result.headers.append('Cache-Control', `public, max-age=${CACHE_MAX_AGE}`); - const cacheUrl = new URL(request.url); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - ctx.waitUntil(cache.put(cacheKey, result.clone())); // 异步存入缓存 - } - return result; - } - - const startTime = Date.now(); // 记录请求开始时间 - let requests = []; // 初始化请求数组 - - // 构建 API 请求数组，根据请求来源（GitHub 或 GitLab 或混合） - if (from === 'where') { - // 获取文件信息模式 - const githubRequests = githubRepos.map(repo => ({ - url: `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found') - const data = await response.json() - return { - size: data.size, - exists: true - } - } - })) - - const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found') - const data = await response.json() - const size = atob(data.content).length - return { - size: size, - exists: true - } - } - })) - - requests = [...githubRequests, ...gitlabRequests] - } else { - // 获取文件内容模式 - if (from === 'github') { - requests = githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo - })) - } else if (from === 'gitlab') { - requests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name - })) - } else { - requests = [ - ...githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo - })), - ...GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name - })) - ] - } - } - - const fetchPromises = requests.map(({ url, headers, source, repo, processResponse }) => { - return fetch(new Request(url, { - method: 'GET', - headers: headers - })).then(async response => { - if (from === 'where') { - // 查询文件所在位置 - try { - const result = await processResponse(response); - const endTime = Date.now(); // 记录请求结束时间 - const duration = endTime - startTime; // 计算请求耗时 - - // 格式化文件大小 - const formattedSize = result.size > 1024 * 1024 - ? `${(result.size / (1024 * 1024)).toFixed(2)} MB` - : `${(result.size / 1024).toFixed(2)} kB`; - - // 返回文件信息 - return { - fileName: FILE, - size: formattedSize, - source: `${source} (${repo})`, - duration: `${duration}ms` - }; - } catch (error) { - throw new Error(`Not found in ${source} (${repo})`); - } - } else { - // 如果不是 "where" 查询，直接返回响应内容 - if (!response.ok) { - throw new Error(`Not found in ${source} (${repo})`); - } - return response; - } - }).catch(error => { - throw new Error(`Error in ${source} (${repo}): ${error.message}`); - }); - }); - - try { - if (requests.length === 0) { - throw new Error('No valid source specified'); - } - - const result = await Promise.any(fetchPromises); - - let response; - if (from === 'where') { - // 如果是 where 查询，返回 JSON 格式响应 - response = new Response(JSON.stringify(result, null, 2), { - headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-store, no-cache, must-revalidate, proxy-revalidate', - 'Pragma': 'no-cache', - 'Expires': '0' - } - }); - } else if (result instanceof Response) { - // 先读取响应体 - const blob = await result.blob(); - const headers = { - 'Content-Type': result.headers.get('Content-Type') || 'application/octet-stream', - 'Access-Control-Allow-Origin': '*' - }; - - // 如果有 from 参数，添加禁止缓存的头部 - if (from) { - headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, proxy-revalidate'; - headers['Pragma'] = 'no-cache'; - headers['Expires'] = '0'; - } else { - // 如果没有 from 参数，使用配置的缓存时间 - headers['Cache-Control'] = `public, max-age=${CACHE_MAX_AGE}`; - } - - // 创建新的响应，只使用最基本的必要头部 - response = new Response(blob, { - status: 200, - headers: headers - }); - } else { - throw new Error("Unexpected result type"); - } - - // 只在没有 from 参数且不是 where 查询时才缓存响应 - if (!from && from !== 'where') { - const cacheUrl = new URL(request.url); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - ctx.waitUntil(cache.put(cacheKey, response.clone())); - } - - return response; - - } catch (error) { - const sourceText = from === 'where' - ? 'in any repository' - : from - ? `from ${from}` - : 'in the GitHub and GitLab picture cluster'; - - const errorResponse = new Response( - `404: Cannot find the ${FILE} ${sourceText}.`, - { - status: 404, - headers: { - 'Content-Type': 'text/plain', - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-store, no-cache, must-revalidate, proxy-revalidate', - 'Pragma': 'no-cache', - 'Expires': '0' - } - } - ); - - return errorResponse; - } - } -} - -// 列出所有节点仓库状态 -async function listProjects(gitlabConfigs, githubRepos, githubUsername, githubPat) { - let result = 'GitHub and GitLab Nodes status:\n\n'; - - try { - // 并发执行所有检查 - const [username, ...allChecks] = await Promise.all([ - getGitHubUsername(githubPat), - ...githubRepos.map(repo => - checkGitHubRepo(githubUsername, repo, githubPat) - ), - ...gitlabConfigs.map(config => - checkGitLabProject(config.id, config.token) - ), - ]); - - // 计算各类检查结果的数量 - const githubCount = githubRepos.length; - const gitlabCount = gitlabConfigs.length; - - // 分割检查结果 - const githubResults = allChecks.slice(0, githubCount); - const gitlabResults = allChecks.slice(githubCount, githubCount + gitlabCount); - - // 添加 GitHub 结果 - githubRepos.forEach((repo, index) => { - const [status, fileCount, totalSize] = githubResults[index]; - const formattedSize = formatSize(totalSize); - result += `GitHub: ${repo} - ${status} (Username: ${username}, Files: ${fileCount}, Size: ${formattedSize})\n`; - }); - - // 添加 GitLab 结果 - gitlabConfigs.forEach((config, index) => { - const [status, username, fileCount, totalSize] = gitlabResults[index]; - result += `GitLab: Project ID ${config.id} - ${status} (Username: ${username}, Files: ${fileCount}, Size: ${totalSize})\n`; - }); - - } catch (error) { - result += `Error during status check: ${error.message}\n`; - } - - return new Response(result, { - headers: { 'Content-Type': 'text/plain' } - }); -} - -// 文件大小格式化函数 -function formatSize(sizeInBytes) { - if (sizeInBytes >= 1024 * 1024 * 1024) { - return `${(sizeInBytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; - } else if (sizeInBytes >= 1024 * 1024) { - return `${(sizeInBytes / (1024 * 1024)).toFixed(2)} MB`; - } else { - return `${(sizeInBytes / 1024).toFixed(2)} kB`; - } -} - -// 获取 GitHub 用户名的异步函数 -async function getGitHubUsername(pat) { - const url = 'https://api.github.com/user'; // GitHub 用户信息 API 地址 - try { - const response = await fetch(url, { - headers: { - 'Authorization': `token ${pat}`, // 使用个人访问令牌进行授权 - 'Accept': 'application/vnd.github.v3+json', // 指定接受的响应格式 - 'User-Agent': 'Cloudflare Worker' // 用户代理 - } - }); - - // 如果响应状态为 200，表示成功 - if (response.status === 200) { - const data = await response.json(); // 解析 JSON 数据 - return data.login; // 返回用户登录名 - } else { - console.error('GitHub API Error:', response.status); // 记录错误状态 - return 'Unknown'; // 返回未知状态 - } - } catch (error) { - console.error('GitHub request error:', error); // 记录请求错误 - return 'Error'; // 返回错误状态 - } -} - -// 检查 GitHub 仓库的异步函数 -async function checkGitHubRepo(owner, repo, pat) { - const repoUrl = `https://api.github.com/repos/${owner}/${repo}`; - const contentsUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${DIR}`; // 直接检查指定目录 - - const headers = { - 'Authorization': `token ${pat}`, - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'Cloudflare Worker' - }; - - try { - // 并发请求获取仓库信息和目录内容 - const [repoResponse, contentsResponse] = await Promise.all([ - fetch(repoUrl, { headers }), - fetch(contentsUrl, { headers }) - ]); - - const repoData = await repoResponse.json(); - - if (repoResponse.status !== 200) { - throw new Error(`Repository error: ${repoData.message}`); - } - - if (contentsResponse.status !== 200) { - return [`working (${repoData.private ? 'private' : 'public'})`, 0, 0]; - } - - const contentsData = await contentsResponse.json(); - - // 计算文件数量和总大小 - const fileStats = contentsData.reduce((acc, item) => { - if (item.type === 'file') { - return { - count: acc.count + 1, - size: acc.size + (item.size || 0) - }; - } - return acc; - }, { count: 0, size: 0 }); - - return [ - `working (${repoData.private ? 'private' : 'public'})`, - fileStats.count, - fileStats.size - ]; - - } catch (error) { - console.error(`Error checking GitHub repo ${repo}:`, error); - return [`error: ${error.message}`, 0, 0]; - } -} - -// 获取单个文件大小的辅助函数 -async function getFileSizeFromGitLab(projectId, filePath, pat, retries = 3) { - for (let i = 0; i < retries; i++) { - try { - // 使用 raw 端点和 HEAD 请求获取文件大小 - const fileUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/files${filePath}/raw?ref=main`; - const response = await fetch(fileUrl, { - method: 'HEAD', - headers: { 'PRIVATE-TOKEN': pat } - }); - - if (response.status === 200) { - const contentLength = response.headers.get('content-length'); - return contentLength ? parseInt(contentLength, 10) : 0; - } else if (response.status === 429) { - // 遇到限流时等待后重试 - await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1))); - continue; - } - console.error(`Failed to get file size: ${response.status}`); - return 0; - } catch (error) { - if (i === retries - 1) { - console.error(`Error fetching file size:`, error); - } - if (i < retries - 1) { - await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1))); - continue; - } - } - return 0; - } - return 0; -} - -// 添加并发控制的辅助函数 -async function asyncPool(concurrency, iterable, iteratorFn) { - const ret = []; - const executing = new Set(); - - for (const item of iterable) { - const p = Promise.resolve().then(() => iteratorFn(item)); - ret.push(p); - executing.add(p); - - const clean = () => executing.delete(p); - p.then(clean).catch(clean); - - if (executing.size >= concurrency) { - await Promise.race(executing); - } - } - - return Promise.all(ret); -} - -// 检查 GitLab 项目的异步函数 -async function checkGitLabProject(projectId, pat) { - const projectUrl = `https://gitlab.com/api/v4/projects/${projectId}`; - // 步骤1: 获取文件列表 - const filesUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?ref=main&path=${DIR}`; - - try { - const [projectResponse, filesResponse] = await Promise.all([ - fetch(projectUrl, { - headers: { 'PRIVATE-TOKEN': pat } - }), - fetch(filesUrl, { - headers: { 'PRIVATE-TOKEN': pat } - }) - ]); - - if (projectResponse.status === 200) { - const projectData = await projectResponse.json(); - let totalSize = 0; - let fileCount = 0; - - if (filesResponse.status === 200) { - const filesData = await filesResponse.json(); - fileCount = filesData.length; - - if (fileCount > 0) { - console.log(`Found ${fileCount} files in ${DIR} directory`); - - // 步骤2: 并发获取每个文件的大小 - const CONCURRENT_REQUESTS = 100; - const sizes = await asyncPool(CONCURRENT_REQUESTS, filesData, async (file) => { - // 构造文件路径，格式为 /files%2Ffilename - const encodedPath = `/${encodeURIComponent(file.path)}`; - const size = await getFileSizeFromGitLab(projectId, encodedPath, pat); - console.log(`File: ${file.path}, Size: ${formatSize(size)}`); - return size; - }); - - totalSize = sizes.reduce((acc, size) => acc + size, 0); - console.log(`Total size: ${formatSize(totalSize)}`); - } - } - - return [ - `working (${projectData.visibility})`, - projectData.owner.username, - fileCount, - formatSize(totalSize) - ]; - } else if (projectResponse.status === 404) { - return ['not found', 'Unknown', 0, '0 B']; - } else { - return ['disconnect', 'Unknown', 0, '0 B']; - } - } catch (error) { - console.error('GitLab project check error:', error); - return ['disconnect', 'Error', 0, '0 B']; - } -} \ No newline at end of file diff --git a/cloudflare_worker/github_gitlab_r2.js b/cloudflare_worker/github_gitlab_r2.js deleted file mode 100644 index bb3b56f..0000000 --- a/cloudflare_worker/github_gitlab_r2.js +++ /dev/null @@ -1,729 +0,0 @@ -// 用户配置区域开始 ================================= - -// GitLab 节点配置 -const GITLAB_CONFIGS = [ - { name: '', id: '', token: '' }, // GitLab 账户1 - { name: '', id: '', token: '' }, // GitLab 账户2 - { name: '', id: '', token: '' }, // GitLab 账户3 - { name: '', id: '', token: '' }, // GitLab 账户4 -]; - -// GitHub 配置 -const GITHUB_REPOS = ['']; // GitHub 仓库名列表 -const GITHUB_USERNAME = ''; // GitHub 用户名 -const GITHUB_PAT = ''; // GitHub 个人访问令牌 - -// R2 存储配置 -const R2_CONFIGS = [ - { - name: '', // 帐户1 ID - accountId: '', // 帐户1 访问密钥 ID - accessKeyId: '', // 帐户1 机密访问密钥 - secretAccessKey: '', // 帐户1 机密访问密钥 - bucket: '' // 帐户1 R2 存储桶名称 - }, - { - name: '', // 帐户2 ID - accountId: '', // 帐户2 访问密钥 ID - accessKeyId: '', // 帐户2 机密访问密钥 - secretAccessKey: '', // 帐户2 机密访问密钥 - bucket: '' // 帐户2 R2 存储桶名称 - }, - // 可以添加更多 R2 配置 -]; - -// 定义集群访问目录 -const DIR = ''; - -// 定义集群里全部节点连接状态的密码验证，区分大小写（优先使用自定义密码，若为空则使用 GITHUB_PAT） -const CHECK_PASSWORD = '' || GITHUB_PAT; - -// 用户配置区域结束 ================================= - -// AWS SDK 签名相关函数开始 ================================= - -// 获取签名URL -async function getSignedUrl(r2Config, method, path) { - const region = 'auto'; - const service = 's3'; - const host = `${r2Config.accountId}.r2.cloudflarestorage.com`; - const datetime = new Date().toISOString().replace(/[:-]|\.\d{3}/g, ''); - const date = datetime.substr(0, 8); - - const canonicalRequest = [ - method, - '/' + path, - '', - `host:${host}`, - 'x-amz-content-sha256:UNSIGNED-PAYLOAD', - `x-amz-date:${datetime}`, - '', - 'host;x-amz-content-sha256;x-amz-date', - 'UNSIGNED-PAYLOAD' - ].join('\n'); - - const stringToSign = [ - 'AWS4-HMAC-SHA256', - datetime, - `${date}/${region}/${service}/aws4_request`, - await sha256(canonicalRequest) - ].join('\n'); - - const signature = await getSignature( - r2Config.secretAccessKey, - date, - region, - service, - stringToSign - ); - - const authorization = [ - `AWS4-HMAC-SHA256 Credential=${r2Config.accessKeyId}/${date}/${region}/${service}/aws4_request`, - `SignedHeaders=host;x-amz-content-sha256;x-amz-date`, - `Signature=${signature}` - ].join(', '); - - return { - url: `https://${host}/${path}`, - headers: { - 'Authorization': authorization, - 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - 'x-amz-date': datetime, - 'Host': host - } - }; -} - -// SHA256 哈希函数 -async function sha256(message) { - const msgBuffer = new TextEncoder().encode(message); - const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer); - return Array.from(new Uint8Array(hashBuffer)) - .map(b => b.toString(16).padStart(2, '0')) - .join(''); -} - -// HMAC-SHA256 函数 -async function hmacSha256(key, message) { - const keyBuffer = key instanceof ArrayBuffer ? key : new TextEncoder().encode(key); - const messageBuffer = new TextEncoder().encode(message); - - const cryptoKey = await crypto.subtle.importKey( - 'raw', - keyBuffer, - { name: 'HMAC', hash: 'SHA-256' }, - false, - ['sign'] - ); - - const signature = await crypto.subtle.sign( - 'HMAC', - cryptoKey, - messageBuffer - ); - - return signature; -} - -// 获取签名 -async function getSignature(secret, date, region, service, stringToSign) { - const kDate = await hmacSha256('AWS4' + secret, date); - const kRegion = await hmacSha256(kDate, region); - const kService = await hmacSha256(kRegion, service); - const kSigning = await hmacSha256(kService, 'aws4_request'); - const signature = await hmacSha256(kSigning, stringToSign); - - return Array.from(new Uint8Array(signature)) - .map(b => b.toString(16).padStart(2, '0')) - .join(''); -} - -export default { - async fetch(request, env, ctx) { - const url = new URL(request.url); - const from = url.searchParams.get('from')?.toLowerCase(); - - // 只在没有 from 参数时才检查和使用缓存 - let cacheResponse; - if (!from) { - const cacheUrl = new URL(request.url); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - cacheResponse = await cache.match(cacheKey); - - if (cacheResponse) { - return cacheResponse; - } - } - - const isValidGithubRepos = Array.isArray(GITHUB_REPOS) && - GITHUB_REPOS.length > 0 && - GITHUB_REPOS.some(repo => repo.trim() !== ''); - - const githubRepos = isValidGithubRepos - ? GITHUB_REPOS.filter(repo => repo.trim() !== '') - : GITLAB_CONFIGS.map(config => config.name); - - const FILE = url.pathname.split('/').pop(); - - if (url.pathname === `/${CHECK_PASSWORD}`) { - const response = await listProjects(GITLAB_CONFIGS, githubRepos, GITHUB_USERNAME, GITHUB_PAT); - // 如果有 from 参数，添加禁止缓存的头部 - if (from) { - response.headers.append('Cache-Control', 'no-store, no-cache, must-revalidate, proxy-revalidate'); - response.headers.append('Pragma', 'no-cache'); - response.headers.append('Expires', '0'); - } else { - const cacheUrl = new URL(request.url); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - ctx.waitUntil(cache.put(cacheKey, response.clone())); - } - return response; - } - - const startTime = Date.now(); - // 根据不同的访问方式构建请求 - let requests = []; - - // R2 请求生成函数 - const generateR2Requests = async () => { - return Promise.all(R2_CONFIGS.map(async (r2Config) => { - const r2Path = `${r2Config.bucket}/${DIR}/${FILE}`; - const signedRequest = await getSignedUrl(r2Config, 'GET', r2Path); - return { - url: signedRequest.url, - headers: signedRequest.headers, - source: 'r2', - repo: `${r2Config.name} (${r2Config.bucket})` - }; - })); - }; - - if (from === 'where') { - // 获取文件信息模式 - const githubRequests = githubRepos.map(repo => ({ - url: `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found'); - const data = await response.json(); - return { - size: data.size, - exists: true - }; - } - })); - - const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found'); - const data = await response.json(); - const size = atob(data.content).length; - return { - size: size, - exists: true - }; - } - })); - - const r2Requests = await generateR2Requests(); - const r2WhereRequests = r2Requests.map(request => ({ - ...request, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found'); - const size = response.headers.get('content-length'); - return { - size: parseInt(size), - exists: true - }; - } - })); - - requests = [...githubRequests, ...gitlabRequests, ...r2WhereRequests]; - - } else { - // 获取文件内容模式 - if (from === 'github') { - requests = githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo - })); - } else if (from === 'gitlab') { - requests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name - })); - } else if (from === 'r2') { - requests = await generateR2Requests(); - } else { - // 如果没有指定来源，则从所有源获取 - const githubRequests = githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo - })); - - const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name - })); - - const r2Requests = await generateR2Requests(); - - requests = [...githubRequests, ...gitlabRequests, ...r2Requests]; - } - } - - // 发送请求并处理响应 - const fetchPromises = requests.map(request => { - const { url, headers, source, repo, processResponse } = request; - - return fetch(new Request(url, { - method: 'GET', - headers: headers - })).then(async response => { - if (from === 'where' && typeof processResponse === 'function') { - // 使用 `processResponse` 处理 where 查询逻辑 - try { - const result = await processResponse(response); - const endTime = Date.now(); - const duration = endTime - startTime; - - const formattedSize = result.size > 1024 * 1024 - ? `${(result.size / (1024 * 1024)).toFixed(2)} MB` - : `${(result.size / 1024).toFixed(2)} kB`; - - return { - fileName: FILE, - size: formattedSize, - source: `${source} (${repo})`, - duration: `${duration}ms` - }; - } catch (error) { - throw new Error(`Not found in ${source} (${repo})`); - } - } else { - // 对于内容获取，直接返回响应 - if (!response.ok) { - throw new Error(`Not found in ${source} (${repo})`); - } - return response; - } - }).catch(error => { - throw new Error(`Error in ${source} (${repo}): ${error.message}`); - }); - }); - - try { - if (requests.length === 0) { - throw new Error('No valid source specified'); - } - - const result = await Promise.any(fetchPromises); - - let response; - if (from === 'where') { - response = new Response(JSON.stringify(result, null, 2), { - headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-store, no-cache, must-revalidate, proxy-revalidate', - 'Pragma': 'no-cache', - 'Expires': '0' - } - }); - } else if (result instanceof Response) { - const blob = await result.blob(); - const headers = { - 'Content-Type': result.headers.get('Content-Type') || 'application/octet-stream', - 'Access-Control-Allow-Origin': '*' - }; - - // 如果有 from 参数，添加禁止缓存的头部 - if (from) { - headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, proxy-revalidate'; - headers['Pragma'] = 'no-cache'; - headers['Expires'] = '0'; - } - - response = new Response(blob, { - status: 200, - headers: headers - }); - } else { - throw new Error("Unexpected result type"); - } - - // 只在没有 from 参数时才缓存响应 - if (!from) { - const cacheUrl = new URL(request.url); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - ctx.waitUntil(cache.put(cacheKey, response.clone())); - } - - return response; - - } catch (error) { - const sourceText = from === 'where' - ? 'in any repository' - : from - ? `from ${from}` - : 'in the GitHub, GitLab and R2 storage'; - - const errorResponse = new Response( - `404: Cannot find the ${FILE} ${sourceText}.`, - { - status: 404, - headers: { - 'Content-Type': 'text/plain', - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-store, no-cache, must-revalidate, proxy-revalidate', - 'Pragma': 'no-cache', - 'Expires': '0' - } - } - ); - - return errorResponse; - } - } -}; - -// 列出所有节点仓库状态 -async function listProjects(gitlabConfigs, githubRepos, githubUsername, githubPat) { - let result = 'GitHub, GitLab and R2 Storage status:\n\n'; - - try { - // 并发执行所有检查 - const [username, ...allChecks] = await Promise.all([ - getGitHubUsername(githubPat), - ...githubRepos.map(repo => - checkGitHubRepo(githubUsername, repo, githubPat) - ), - ...gitlabConfigs.map(config => - checkGitLabProject(config.id, config.token) - ), - ...R2_CONFIGS.map(config => - checkR2Storage(config) - ) - ]); - - // 计算各类检查结果的数量 - const githubCount = githubRepos.length; - const gitlabCount = gitlabConfigs.length; - - // 分割检查结果 - const githubResults = allChecks.slice(0, githubCount); - const gitlabResults = allChecks.slice(githubCount, githubCount + gitlabCount); - const r2Results = allChecks.slice(githubCount + gitlabCount); - - // 添加 GitHub 结果 - githubRepos.forEach((repo, index) => { - const [status, fileCount, totalSize] = githubResults[index]; - const formattedSize = formatSize(totalSize); - result += `GitHub: ${repo} - ${status} (Username: ${username}, Files: ${fileCount}, Size: ${formattedSize})\n`; - }); - - // 添加 GitLab 结果 - gitlabConfigs.forEach((config, index) => { - const [status, username, fileCount, totalSize] = gitlabResults[index]; - result += `GitLab: Project ID ${config.id} - ${status} (Username: ${username}, Files: ${fileCount}, Size: ${totalSize})\n`; - }); - - // 添加 R2 结果 - r2Results.forEach(([status, name, bucket, fileCount, totalSize]) => { - result += `R2 Storage: ${name} - ${status} (Bucket: ${bucket}, Files: ${fileCount}, Size: ${totalSize})\n`; - }); - - } catch (error) { - result += `Error during status check: ${error.message}\n`; - } - - return new Response(result, { - headers: { 'Content-Type': 'text/plain' } - }); -} - -// 文件大小格式化函数 -function formatSize(sizeInBytes) { - if (sizeInBytes >= 1024 * 1024 * 1024) { - return `${(sizeInBytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; - } else if (sizeInBytes >= 1024 * 1024) { - return `${(sizeInBytes / (1024 * 1024)).toFixed(2)} MB`; - } else { - return `${(sizeInBytes / 1024).toFixed(2)} kB`; - } -} - -// 获取 GitHub 用户名的异步函数 -async function getGitHubUsername(pat) { - const url = 'https://api.github.com/user'; // GitHub 用户信息 API 地址 - try { - const response = await fetch(url, { - headers: { - 'Authorization': `token ${pat}`, // 使用个人访问令牌进行授权 - 'Accept': 'application/vnd.github.v3+json', // 指定接受的响应格式 - 'User-Agent': 'Cloudflare Worker' // 用户代理 - } - }); - - // 如果响应状态为 200，表示成功 - if (response.status === 200) { - const data = await response.json(); // 解析 JSON 数据 - return data.login; // 返回用户登录名 - } else { - console.error('GitHub API Error:', response.status); // 记录错误状态 - return 'Unknown'; // 返回未知状态 - } - } catch (error) { - console.error('GitHub request error:', error); // 记录请求错误 - return 'Error'; // 返回错误状态 - } -} - -// 检查 GitHub 仓库的异步函数 -async function checkGitHubRepo(owner, repo, pat) { - const repoUrl = `https://api.github.com/repos/${owner}/${repo}`; - const contentsUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${DIR}`; // 直接检查指定目录 - - const headers = { - 'Authorization': `token ${pat}`, - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'Cloudflare Worker' - }; - - try { - // 并发请求获取仓库信息和目录内容 - const [repoResponse, contentsResponse] = await Promise.all([ - fetch(repoUrl, { headers }), - fetch(contentsUrl, { headers }) - ]); - - const repoData = await repoResponse.json(); - - if (repoResponse.status !== 200) { - throw new Error(`Repository error: ${repoData.message}`); - } - - if (contentsResponse.status !== 200) { - return [`working (${repoData.private ? 'private' : 'public'})`, 0, 0]; - } - - const contentsData = await contentsResponse.json(); - - // 计算文件数量和总大小 - const fileStats = contentsData.reduce((acc, item) => { - if (item.type === 'file') { - return { - count: acc.count + 1, - size: acc.size + (item.size || 0) - }; - } - return acc; - }, { count: 0, size: 0 }); - - return [ - `working (${repoData.private ? 'private' : 'public'})`, - fileStats.count, - fileStats.size - ]; - - } catch (error) { - console.error(`Error checking GitHub repo ${repo}:`, error); - return [`error: ${error.message}`, 0, 0]; - } -} - -// 获取单个文件大小的辅助函数 -async function getFileSizeFromGitLab(projectId, filePath, pat, retries = 3) { - for (let i = 0; i < retries; i++) { - try { - // 使用 raw 端点和 HEAD 请求获取文件大小 - const fileUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/files${filePath}/raw?ref=main`; - const response = await fetch(fileUrl, { - method: 'HEAD', - headers: { 'PRIVATE-TOKEN': pat } - }); - - if (response.status === 200) { - const contentLength = response.headers.get('content-length'); - return contentLength ? parseInt(contentLength, 10) : 0; - } else if (response.status === 429) { - // 遇到限流时等待后重试 - await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1))); - continue; - } - console.error(`Failed to get file size: ${response.status}`); - return 0; - } catch (error) { - if (i === retries - 1) { - console.error(`Error fetching file size:`, error); - } - if (i < retries - 1) { - await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1))); - continue; - } - } - return 0; - } - return 0; -} - -// 添加并发控制的辅助函数 -async function asyncPool(concurrency, iterable, iteratorFn) { - const ret = []; - const executing = new Set(); - - for (const item of iterable) { - const p = Promise.resolve().then(() => iteratorFn(item)); - ret.push(p); - executing.add(p); - - const clean = () => executing.delete(p); - p.then(clean).catch(clean); - - if (executing.size >= concurrency) { - await Promise.race(executing); - } - } - - return Promise.all(ret); -} - -// 检查 GitLab 项目的异步函数 -async function checkGitLabProject(projectId, pat) { - const projectUrl = `https://gitlab.com/api/v4/projects/${projectId}`; - // 步骤1: 获取文件列表 - const filesUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?ref=main&path=${DIR}`; - - try { - const [projectResponse, filesResponse] = await Promise.all([ - fetch(projectUrl, { - headers: { 'PRIVATE-TOKEN': pat } - }), - fetch(filesUrl, { - headers: { 'PRIVATE-TOKEN': pat } - }) - ]); - - if (projectResponse.status === 200) { - const projectData = await projectResponse.json(); - let totalSize = 0; - let fileCount = 0; - - if (filesResponse.status === 200) { - const filesData = await filesResponse.json(); - fileCount = filesData.length; - - if (fileCount > 0) { - console.log(`Found ${fileCount} files in ${DIR} directory`); - - // 步骤2: 并发获取每个文件的大小 - const CONCURRENT_REQUESTS = 100; - const sizes = await asyncPool(CONCURRENT_REQUESTS, filesData, async (file) => { - // 构造文件路径，格式为 /files%2Ffilename - const encodedPath = `/${encodeURIComponent(file.path)}`; - const size = await getFileSizeFromGitLab(projectId, encodedPath, pat); - console.log(`File: ${file.path}, Size: ${formatSize(size)}`); - return size; - }); - - totalSize = sizes.reduce((acc, size) => acc + size, 0); - console.log(`Total size: ${formatSize(totalSize)}`); - } - } - - return [ - `working (${projectData.visibility})`, - projectData.owner.username, - fileCount, - formatSize(totalSize) - ]; - } else if (projectResponse.status === 404) { - return ['not found', 'Unknown', 0, '0 B']; - } else { - return ['disconnect', 'Unknown', 0, '0 B']; - } - } catch (error) { - console.error('GitLab project check error:', error); - return ['disconnect', 'Error', 0, '0 B']; - } -} - -// 检查 R2 存储状态 -async function checkR2Storage(r2Config) { - try { - // 1. 列出目录下所有文件 - const listPath = `${r2Config.bucket}`; // 列出根目录 - const signedRequest = await getSignedUrl(r2Config, 'GET', listPath); - - const response = await fetch(signedRequest.url, { - headers: signedRequest.headers - }); - - let fileCount = 0; - let totalSize = 0; - - if (response.ok) { - const data = await response.text(); - // 解析 XML 响应 - const keys = data.match(/([^<]+)<\/Key>/g) || []; - const sizes = data.match(/(\d+)<\/Size>/g) || []; - - // 只计算指定目录下的文件 - keys.forEach((key, index) => { - const filePath = key.replace(/|<\/Key>/g, ''); - if (filePath.startsWith(DIR + '/')) { - fileCount++; - const size = parseInt(sizes[index]?.replace(/|<\/Size>/g, '') || String(0), 10); - totalSize += size; - } - }); - } - - // 即使文件不存在，只要能访问到存储桶就认为是正常的 - const status = response.ok ? 'working' : 'error'; - - return [ - status, - r2Config.name, - r2Config.bucket, - fileCount, - formatSize(totalSize) - ]; - } catch (error) { - console.error('R2 Storage error:', error); - return ['error', r2Config.name, 'connection failed', 0, '0 B']; - } -} \ No newline at end of file diff --git a/cloudflare_worker/github_gitlab_r2_b2.js b/cloudflare_worker/github_gitlab_r2_b2.js new file mode 100644 index 0000000..34ac276 --- /dev/null +++ b/cloudflare_worker/github_gitlab_r2_b2.js @@ -0,0 +1,1298 @@ +// 用户配置区域开始 ================================= + +// GitLab 节点配置 +const GITLAB_CONFIGS = [ + { name: '', id: '', token: '' }, // GitLab 账户1 + { name: '', id: '', token: '' }, // GitLab 账户2 + { name: '', id: '', token: '' }, // GitLab 账户3 + { name: '', id: '', token: '' }, // GitLab 账户4 +]; + +// GitHub 配置，仓库名与 GitLab 的相同，故创建仓库时需要注意一定要对称 +const GITHUB_USERNAME = ''; // GitHub 用户名 +const GITHUB_PAT = ''; // GitHub 个人访问令牌 + +// R2 存储配置，没有可以留空不填，但不要删除 +const R2_CONFIGS = [ + { + name: '', // 帐户1 ID + accountId: '', // 帐户1 访问密钥 ID + accessKeyId: '', // 帐户1 机密访问密钥 + secretAccessKey: '', // 帐户1 机密访问密钥 + bucket: '' // 帐户1 R2 存储桶名称 + }, + { + name: '', // 帐户2 ID + accountId: '', // 帐户2 访问密钥 ID + accessKeyId: '', // 帐户2 机密访问密钥 + secretAccessKey: '', // 帐户2 机密访问密钥 + bucket: '' // 帐户2 R2 存储桶名称 + }, + // 可以添加更多 R2 配置 +]; + +// B2 存储配置，没有可以留空不填，但不要删除 +const B2_CONFIGS = [ + { + name: '', // 帐户1 名 + endPoint: '', // 账户1 Endpoint + keyId: '', // 账户1 keyID + applicationKey: '', // 账户1 applicationKey + bucket: '' // 账户1桶名 bucketName + }, + { + name: '', // 帐户2 名 + endPoint: '', // 账户2 Endpoint + keyId: '', // 账户2 keyID + applicationKey: '', // 账户2 applicationKey + bucket: '' // 账户2桶名 bucketName + }, + // 可以添加更多 B2 配置 +]; + +// 定义集群访问目录 +const DIR = /** @type {string} */ (''); + +// 定义集群里全部节点连接状态的密码验证，区分大小写（优先使用自定义密码，若为空则使用 GITHUB_PAT） +const CUSTOM_PASSWORD = ''; +const CHECK_PASSWORD = CUSTOM_PASSWORD || GITHUB_PAT; + +// GitHub 备份策略 +const STRATEGY = 'size' // 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 +const DELETE = 'true' // 可选 [true (默认) | false]，已复制到 GitHub 的文件，是否从 R2 删除 + +// 用户配置区域结束 ================================= + +// 检查配置是否有效 +function hasValidConfig() { + // 检查 GitHub 配置 + const hasGithub = GITHUB_PAT && GITHUB_USERNAME && GITLAB_CONFIGS && + GITLAB_CONFIGS.length > 0 && + GITLAB_CONFIGS.some(config => config.name && config.id && config.token); + + // 检查 GitLab 配置 + const hasGitlab = GITLAB_CONFIGS && + GITLAB_CONFIGS.length > 0 && + GITLAB_CONFIGS.some(config => config.name && config.id && config.token); + + // 检查 R2 配置 + const hasR2 = R2_CONFIGS && + R2_CONFIGS.length > 0 && + R2_CONFIGS.some(config => + config.name && + config.accountId && + config.accessKeyId && + config.secretAccessKey && + config.bucket + ); + + // 检查 B2 配置 + const hasB2 = B2_CONFIGS && + B2_CONFIGS.length > 0 && + B2_CONFIGS.some(config => + config.name && + config.endPoint && + config.keyId && + config.applicationKey && + config.bucket + ); + + return { + github: hasGithub, + gitlab: hasGitlab, + r2: hasR2, + b2: hasB2 + }; +} + +// AWS SDK 签名相关函数开始 ================================= + +// 获取签名URL +async function getSignedUrl(config, method, path, queryParams = {}) { + const region = config.endPoint ? config.endPoint.split('.')[1] : 'auto'; + const service = 's3'; + const host = config.endPoint || `${config.accountId}.r2.cloudflarestorage.com`; + const accessKeyId = config.endPoint ? config.keyId : config.accessKeyId; + const secretKey = config.endPoint ? config.applicationKey : config.secretAccessKey; + const datetime = new Date().toISOString().replace(/[:-]|\.\d{3}/g, ''); + const date = datetime.substr(0, 8); + + // 确保路径正确编码，但保留斜杠 + const encodedPath = path.split('/') + .map(segment => encodeURIComponent(segment)) + .join('/'); + + // 构建规范请求 + const canonicalHeaders = `host:${host}\nx-amz-content-sha256:UNSIGNED-PAYLOAD\nx-amz-date:${datetime}\n`; + const signedHeaders = 'host;x-amz-content-sha256;x-amz-date'; + + // 按字母顺序排序查询参数 + const sortedParams = Object.keys(queryParams).sort().reduce((acc, key) => { + acc[key] = queryParams[key]; + return acc; + }, {}); + + const canonicalQueryString = Object.entries(sortedParams) + .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`) + .join('&'); + + const canonicalRequest = [ + method, + '/' + encodedPath, + canonicalQueryString, + canonicalHeaders, + signedHeaders, + 'UNSIGNED-PAYLOAD' + ].join('\n'); + + const stringToSign = [ + 'AWS4-HMAC-SHA256', + datetime, + `${date}/${region}/${service}/aws4_request`, + await sha256(canonicalRequest) + ].join('\n'); + + const signature = await getSignature( + secretKey, + date, + region, + service, + stringToSign + ); + + const authorization = [ + `AWS4-HMAC-SHA256 Credential=${accessKeyId}/${date}/${region}/${service}/aws4_request`, + `SignedHeaders=${signedHeaders}`, + `Signature=${signature}` + ].join(', '); + + const url = `https://${host}/${encodedPath}${canonicalQueryString ? '?' + canonicalQueryString : ''}`; + + return { + url, + headers: { + 'Authorization': authorization, + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-date': datetime, + 'Host': host + } + }; +} + +// SHA256 哈希函数 +async function sha256(message) { + const msgBuffer = new TextEncoder().encode(message); + const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer); + return Array.from(new Uint8Array(hashBuffer)) + .map(b => b.toString(16).padStart(2, '0')) + .join(''); +} + +// HMAC-SHA256 函数 +async function hmacSha256(key, message) { + const keyBuffer = key instanceof ArrayBuffer ? key : new TextEncoder().encode(key); + const messageBuffer = new TextEncoder().encode(message); + + const cryptoKey = await crypto.subtle.importKey( + 'raw', + keyBuffer, + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign'] + ); + + const signature = await crypto.subtle.sign( + 'HMAC', + cryptoKey, + messageBuffer + ); + + return signature; +} + +// 获取签名 +async function getSignature(secret, date, region, service, stringToSign) { + const kDate = await hmacSha256('AWS4' + secret, date); + const kRegion = await hmacSha256(kDate, region); + const kService = await hmacSha256(kRegion, service); + const kSigning = await hmacSha256(kService, 'aws4_request'); + const signature = await hmacSha256(kSigning, stringToSign); + + return Array.from(new Uint8Array(signature)) + .map(b => b.toString(16).padStart(2, '0')) + .join(''); +} + +// AWS SDK 签名相关函数结束 ================================= + +// 检查服务函数 +async function getGitHubUsername(pat) { + const url = 'https://api.github.com/user'; + try { + const response = await fetch(url, { + headers: { + 'Authorization': `token ${pat}`, + 'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'Cloudflare Worker' + } + }); + + if (response.status === 200) { + const data = await response.json(); + return data.login; + } else { + console.error('GitHub API Error:', response.status); + return 'Unknown'; + } + } catch (error) { + console.error('GitHub request error:', error); + return 'Error'; + } +} + +// 修改文件路径处理函数 +function getFilePath(basePath, requestPath) { + // 移除开头的斜杠 + const cleanRequestPath = requestPath.replace(/^\//, ''); + + // 如果没有设置 basePath，直接返回请求路径 + if (!basePath) return cleanRequestPath; + + // 组合基础路径和请求路径 + return `${basePath}/${cleanRequestPath}`; +} + +// 检查 GitHub 仓库 +async function checkGitHubRepo(owner, repo, pat) { + const repoUrl = `https://api.github.com/repos/${owner}/${repo}`; + + const headers = { + 'Authorization': `token ${pat}`, + 'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'Cloudflare Worker' + }; + + try { + // 获取仓库信息，确定默认分支 + const repoResponse = await fetch(repoUrl, { headers }); + const repoData = await repoResponse.json(); + + if (repoResponse.status!== 200) { + throw new Error(`Repository error: ${repoData.message}`); + } + + const defaultBranch = repoData.default_branch; + const contentsUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${defaultBranch}?recursive=1`; + + // 获取文件树信息 + const contentsResponse = await fetch(contentsUrl, { headers }); + if (contentsResponse.status!== 200) { + const contentsErrorData = await contentsResponse.json(); + throw new Error(`Contents error: ${contentsErrorData.message}`); + } + + const contentsData = await contentsResponse.json(); + + let fileCount = 0; + let totalSize = 0; + + if (contentsData.tree) { + for (const item of contentsData.tree) { + // 检查是否是文件 + if (item.type === 'blob' && (DIR === '' || item.path.startsWith(DIR + '/'))) { + fileCount++; + totalSize += item.size || 0; + } + } + } + + return [ + `working (${repoData.private ? 'private' : 'public'})`, + fileCount, + totalSize + ]; + + } catch (error) { + console.error(`Error checking GitHub repo ${repo}:`, error); + return [`error: ${error.message}`, 0, 0]; + } +} + +// 检查 GitLab 项目 +async function checkGitLabProject(projectId, pat) { + const projectUrl = `https://gitlab.com/api/v4/projects/${projectId}`; + const treeUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?recursive=true&per_page=100&path=${DIR}`; + + try { + const [projectResponse, treeResponse] = await Promise.all([ + fetch(projectUrl, { + headers: { 'PRIVATE-TOKEN': pat } + }), + fetch(treeUrl, { + headers: { 'PRIVATE-TOKEN': pat } + }) + ]); + + if (projectResponse.status === 200) { + const projectData = await projectResponse.json(); + let fileCount = 0; + + if (treeResponse.status === 200) { + const treeData = await treeResponse.json(); + // 只计算文件，不计算目录 + fileCount = treeData.filter(item => item.type === 'blob').length; + } + + return [ + `working (${projectData.visibility})`, + projectData.owner.username, + fileCount + ]; + } else if (projectResponse.status === 404) { + return ['not found', 'Unknown', 0]; + } else { + return ['disconnect', 'Unknown', 0]; + } + } catch (error) { + console.error('GitLab project check error:', error); + return ['disconnect', 'Error', 0]; + } +} + +// 检查 R2 存储 +async function checkR2Storage(r2Config) { + try { + // 列出所有文件 + const listRequest = await getSignedUrl(r2Config, 'GET', r2Config.bucket, { + 'list-type': '2', + 'prefix': DIR ? `${DIR}/` : '' // 添加目录前缀筛选 + }); + + const response = await fetch(listRequest.url, { + headers: { + ...listRequest.headers, + 'Host': `${r2Config.accountId}.r2.cloudflarestorage.com` + } + }); + + let fileCount = 0; + let totalSize = 0; + + if (response.ok) { + const data = await response.text(); + + // 使用正则表达式匹配所有文件信息 + const contents = data.match(/[\s\S]*?<\/Contents>/g) || []; + + for (const content of contents) { + const keyMatch = content.match(/([^<]+)<\/Key>/); + const sizeMatch = content.match(/(\d+)<\/Size>/); + + if (keyMatch && sizeMatch) { + const key = keyMatch[1]; + // 只计算文件，不计算目录 + if (!key.endsWith('/')) { + fileCount++; + totalSize += parseInt(sizeMatch[1]); + } + } + } + } + + return [ + 'working', + r2Config.name, + r2Config.bucket, + fileCount, + formatSize(totalSize) + ]; + } catch (error) { + console.error('R2 Storage error:', error); + return ['error', r2Config.name, 'connection failed', 0, '0 B']; + } +} + +// 检查 B2 存储 +async function checkB2Storage(b2Config) { + try { + // 构建列出文件的请求，移除 delimiter 参数以获取所有子目录 + const signedRequest = await getSignedUrl(b2Config, 'GET', b2Config.bucket, { + 'prefix': DIR ? `${DIR}/` : '' + }); + + const response = await fetch(signedRequest.url, { + headers: { + ...signedRequest.headers, + 'Host': b2Config.endPoint + } + }); + + let fileCount = 0; + let totalSize = 0; + + if (response.ok) { + const data = await response.text(); + // 使用正则表达式匹配所有文件信息 + const keyRegex = /([^<]+)<\/Key>/g; + const sizeRegex = /(\d+)<\/Size>/g; + + let keyMatch; + while ((keyMatch = keyRegex.exec(data)) !== null) { + const key = keyMatch[1]; + // 只计算文件，不计算目录，并确保文件在指定目录下 + if (!key.endsWith('/') && (!DIR || key.startsWith(DIR + '/'))) { + fileCount++; + // 获取对应的文件大小 + const sizeMatch = /(\d+)<\/Size>/g.exec(data.slice(keyMatch.index)); + if (sizeMatch) { + totalSize += parseInt(sizeMatch[1]); + } + } + } + + return [ + 'working', + b2Config.name, + b2Config.bucket, + fileCount, + formatSize(totalSize) + ]; + } else { + throw new Error(`Failed to list bucket: ${response.status} ${response.statusText}`); + } + + } catch (error) { + console.error('B2 Storage error:', error); + return ['error', b2Config.name, b2Config.bucket, 0, '0 B']; + } +} + +// 删除 GitHub 仓库中的文件 +async function deleteGitHubFile(repo, filePath, pat) { + // 构建完整的文件路径，包含 DIR + const fullPath = DIR ? `${DIR}/${filePath.replace(/^\/+/, '')}` : filePath.replace(/^\/+/, ''); + const url = `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${fullPath}`; + + try { + // 先检查文件是否存在 + const getResponse = await fetch(url, { + headers: { + 'Authorization': `token ${pat}`, + 'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'Cloudflare Worker' + } + }); + + if (getResponse.status === 404) { + return '文件不存在'; + } + + if (!getResponse.ok) { + const errorData = await getResponse.json(); + return `删除失败：(${errorData.message})`; + } + + const fileData = await getResponse.json(); + + // 执行删除操作 + const deleteResponse = await fetch(url, { + method: 'DELETE', + headers: { + 'Authorization': `token ${pat}`, + 'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'Cloudflare Worker' + }, + body: JSON.stringify({ + message: `Delete ${fullPath}`, + sha: fileData.sha + }) + }); + + if (deleteResponse.ok) { + return '删除成功'; + } else { + const errorData = await deleteResponse.json(); + return `删除失败：(${errorData.message})`; + } + } catch (error) { + console.error('GitHub delete error:', error); + return `删除失败：(${error.message})`; + } +} + +// 删除 GitLab 项目中的文件 +async function deleteGitLabFile(projectId, filePath, pat) { + // 构建完整的文件路径，包含 DIR + const fullPath = DIR ? `${DIR}/${filePath.replace(/^\/+/, '')}` : filePath.replace(/^\/+/, ''); + const encodedPath = encodeURIComponent(fullPath); + const url = `https://gitlab.com/api/v4/projects/${projectId}/repository/files/${encodedPath}`; + + try { + // 执行删除操作 + const deleteResponse = await fetch(url, { + method: 'DELETE', + headers: { + 'PRIVATE-TOKEN': pat, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + branch: 'main', + commit_message: 'Delete file: ' + fullPath + }) + }); + + // 获取响应数据 + const errorData = await deleteResponse.json().catch(() => ({})); + + // 处理文件不存在的所有可能情况 + if (deleteResponse.status === 404 || + errorData.message === 'A file with this name doesn\'t exist' || + errorData.message?.includes('file does not exist') || + errorData.message?.includes('File not found')) { + return '文件不存在'; + } + + // 处理删除成功的情况 + if (deleteResponse.ok || + errorData.message?.includes('reference update') || + errorData.message?.includes('reference does not point')) { + return '删除成功'; + } + + return `删除失败：(${errorData.message || '未知错误'})`; + } catch (error) { + console.error('GitLab delete error:', error); + if (error.message?.includes('file') && error.message?.includes('exist')) { + return '文件不存在'; + } + return `删除失败：(${error.message})`; + } +} + +// 删除 R2 存储中的文件 +async function deleteR2File(r2Config, filePath) { + // 构建完整的文件路径，包含 DIR + const fullPath = DIR ? `${DIR}/${filePath.replace(/^\/+/, '')}` : filePath.replace(/^\/+/, ''); + + try { + // 1. 首先列出所有文件 + const listRequest = await getSignedUrl(r2Config, 'GET', r2Config.bucket, { + 'list-type': '2', + 'prefix': fullPath // 使用精确的前缀匹配 + }); + + const listResponse = await fetch(listRequest.url, { + headers: { + ...listRequest.headers, + 'Host': `${r2Config.accountId}.r2.cloudflarestorage.com` + } + }); + + if (!listResponse.ok) { + throw new Error(`Failed to list objects: ${listResponse.statusText}`); + } + + // 解析响应 + const listData = await listResponse.text(); + const contents = listData.match(/[\s\S]*?<\/Contents>/g) || []; + let fileExists = false; + + // 精确匹配文件路径 + for (const content of contents) { + const keyMatch = content.match(/([^<]+)<\/Key>/); + if (keyMatch && keyMatch[1] === fullPath) { + fileExists = true; + break; + } + } + + if (!fileExists) { + return '文件不存在'; + } + + // 2. 删除文件 + const deleteRequest = await getSignedUrl(r2Config, 'DELETE', `${r2Config.bucket}/${fullPath}`); + + const deleteResponse = await fetch(deleteRequest.url, { + method: 'DELETE', + headers: { + ...deleteRequest.headers, + 'Host': `${r2Config.accountId}.r2.cloudflarestorage.com` + } + }); + + if (!deleteResponse.ok) { + const deleteResponseText = await deleteResponse.text(); + throw new Error(`Failed to delete: ${deleteResponse.status} - ${deleteResponseText}`); + } + + return '删除成功'; + } catch (error) { + console.error('R2 delete error:', error); + return `删除失败：(${error.message})`; + } +} + +// 删除 B2 存储中的文件 +async function deleteB2File(b2Config, filePath) { + // 构建完整的文件路径，包含 DIR + const fullPath = DIR ? `${DIR}/${filePath.replace(/^\/+/, '')}` : filePath.replace(/^\/+/, ''); + + try { + // 1. 首先列出所有文件 + const listObjectsRequest = await getSignedUrl(b2Config, 'GET', b2Config.bucket, { + 'list-type': '2', + 'prefix': fullPath + }); + + const listResponse = await fetch(listObjectsRequest.url, { + headers: { + ...listObjectsRequest.headers, + 'Host': b2Config.endPoint + } + }); + + if (!listResponse.ok) { + throw new Error(`Failed to list objects: ${listResponse.statusText}`); + } + + // 解析 XML 响应 + const listData = await listResponse.text(); + const keyRegex = /([^<]+)<\/Key>/g; + const fileExists = Array.from(listData.matchAll(keyRegex)) + .some(match => match[1] === fullPath); + + if (!fileExists) { + return '文件不存在'; + } + + // 2. 获取文件的版本信息 + const versionsRequest = await getSignedUrl(b2Config, 'GET', b2Config.bucket, { + 'versions': '', + 'prefix': fullPath, + 'list-type': '2' + }); + + const versionsResponse = await fetch(versionsRequest.url, { + headers: { + ...versionsRequest.headers, + 'Host': b2Config.endPoint, + 'x-amz-date': versionsRequest.headers['x-amz-date'], + 'Authorization': versionsRequest.headers['Authorization'] + } + }); + + if (!versionsResponse.ok) { + const responseText = await versionsResponse.text(); + console.error('Version listing response:', responseText); + throw new Error(`Failed to list versions: ${versionsResponse.status} - ${responseText}`); + } + + const versionsData = await versionsResponse.text(); + + // 解析版本信息 + const versionMatch = versionsData.match(/[\s\S]*?([^<]+)<\/VersionId>[\s\S]*?<\/Version>/); + if (!versionMatch) { + throw new Error('No version information found'); + } + + const versionId = versionMatch[1]; + + // 3. 删除指定版本的文件 + const deleteRequest = await getSignedUrl(b2Config, 'DELETE', `${b2Config.bucket}/${fullPath}`, { + 'versionId': versionId + }); + + const deleteResponse = await fetch(deleteRequest.url, { + method: 'DELETE', + headers: { + ...deleteRequest.headers, + 'Host': b2Config.endPoint, + 'x-amz-date': deleteRequest.headers['x-amz-date'], + 'Authorization': deleteRequest.headers['Authorization'] + } + }); + + if (!deleteResponse.ok) { + const deleteResponseText = await deleteResponse.text(); + throw new Error(`Failed to delete: ${deleteResponse.status} - ${deleteResponseText}`); + } + + return '删除成功'; + } catch (error) { + console.error('B2 delete error:', error); + return `删除失败：(${error.message})`; + } +} + +// 文件大小格式化函数 +function formatSize(sizeInBytes) { + if (sizeInBytes >= 1024 * 1024 * 1024) { + return `${(sizeInBytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; + } else if (sizeInBytes >= 1024 * 1024) { + return `${(sizeInBytes / (1024 * 1024)).toFixed(2)} MB`; + } else { + return `${(sizeInBytes / 1024).toFixed(2)} kB`; + } +} + +export default { + async fetch(request, env, ctx) { + // 获取请求 URL 对象 + const url = new URL(request.url); + + // 从 URL 的查询参数中获取 'from' 参数并转换为小写 + const from = url.searchParams.get('from')?.toLowerCase(); + + // 检查是否有有效的配置，调用 `hasValidConfig()` 函数 + const validConfigs = hasValidConfig(); + + // 获取请求路径： + // url.pathname 在 Cloudflare Worker 中已是 Unicode 字符串，直接使用即可。 + // 使用 decodeURIComponent 兜底处理仍为百分号编码的情况，同时捕获异常防止恶意编码导致崩溃。 + let requestPath; + try { + requestPath = decodeURIComponent(url.pathname); + } catch (e) { + // 解码失败时直接使用原始 pathname + requestPath = url.pathname; + } + + // 对路径各段重新编码，用于拼接到 URL 中（支持中文/日文/韩文等非 ASCII 文件名） + // 过滤空段，避免 DIR 或 subPath 为空时产生多余斜杠（如 main//文件名） + function encodePathSegments(path) { + return path.split('/').filter(seg => seg !== '').map(seg => encodeURIComponent(seg)).join('/'); + } + + // 构建用于拼接 URL 的编码路径，过滤空值，避免双斜杠 + function buildEncodedPath(...parts) { + return parts.filter(p => p && p !== '').map(p => encodePathSegments(p)).join('/'); + } + + // 添加根路径项目介绍 + if (requestPath === '/') { + return new Response( + '欢迎来到文件托管集群！(File Hosting Cluster)\n' + + '这是一个分布式存储集群项目，旨在提供高效的文件存储和管理服务。\n\n' + + '项目链接： https://github.com/fscarmen2/pic-hosting-cluster\n' + + '视频介绍： https://youtu.be/5i-86oBLWP8\n\n' + + '您可以使用以下操作：\n' + + '1. 从集群所有节点获取文件： /<文件名>\n' + + '2. 指定从 Github 获取文件： /<文件名>?from=github\n' + + '3. 指定从 Gitlab 获取文件： /<文件名>?from=gitlab\n' + + '4. 指定从 Cloudflare R2 获取文件： /<文件名>?from=r2\n' + + '5. 指定从 Backblaze B2 获取文件： /<文件名>?from=b2\n' + + '6. 查找文件信息： /<文件名>?from=where\n' + + '7. 查各节点状态： /<自定义密码>\n' + + '8. 删除文件： /<自定义密码>/del?file=<文件名>', + { + headers: { + 'Content-Type': 'text/plain; charset=UTF-8', + 'Access-Control-Allow-Origin': '*' + } + } + ); + } + + // 从路径中提取文件名（即路径的最后一部分） + const FILE = requestPath.split('/').pop(); + + // 获取子目录路径，移除开头和结尾的斜杠 + const subPath = requestPath.substring(1, requestPath.lastIndexOf('/')).replace(/^\/+|\/+$/g, ''); + + // 如果 DIR 存在，拼接 DIR 和子目录路径；否则仅使用子目录路径 + const fullPath = DIR ? `${DIR}/${subPath}` : subPath; + + // 检查请求路径是否匹配删除请求（支持 'delete' 或 'del'） + const isDeleteRequest = requestPath.match(new RegExp(`^/${CHECK_PASSWORD}/(delete|del)$`)); + + // 检查是否是未授权的删除请求 + const isUnauthorizedDelete = requestPath.match(/^\/(delete|del)$/); + if (isUnauthorizedDelete) { + const file = url.searchParams.get('file'); + if (!file) { + return new Response( + '需要指定要删除的文件。\n' + + '正确的删除格式为: /<自定义密码>/del?file=文件路径\n' + + '例如: /<自定义密码>/del?file=example.png', + { + status: 403, + headers: { + 'Content-Type': 'text/plain; charset=UTF-8', + 'Access-Control-Allow-Origin': '*' + } + } + ); + } + + return new Response( + '需要密码验证才能删除文件。\n' + + '要删除文件 ' + file + ' 的正确格式为:\n' + + '/<自定义密码>/del?file=' + file, + { + status: 403, + headers: { + 'Content-Type': 'text/plain; charset=UTF-8', + 'Access-Control-Allow-Origin': '*' + } + } + ); + } + + // 从 GITLAB_CONFIGS 中获取每个配置的 name 作为 GitHub 仓库名 + const githubRepos = GITLAB_CONFIGS.map(config => config.name); + + // 只在没有 from 参数时才检查和使用缓存 + let cacheResponse; + if (!from) { + const cacheUrl = new URL(request.url); + const cacheKey = new Request(cacheUrl.toString(), request); + const cache = caches.default; + cacheResponse = await cache.match(cacheKey); + + if (cacheResponse) { + return cacheResponse; + } + } + + // 检查状态页面 + if (url.pathname === `/${CHECK_PASSWORD}`) { + let result = ''; + let hasAnyValidConfig = false; + + try { + // GitHub 状态检查 + if (validConfigs.github) { + hasAnyValidConfig = true; + result += '=== GitHub Status ===\n'; + const username = await getGitHubUsername(GITHUB_PAT); + for (const repo of githubRepos) { + const [status, fileCount, totalSize] = await checkGitHubRepo(GITHUB_USERNAME, repo, GITHUB_PAT); + const formattedSize = formatSize(totalSize); + result += `GitHub: ${repo} - ${status} (Username: ${username}, Files: ${fileCount}, Size: ${formattedSize})\n`; + } + } + + // GitLab 状态检查 + if (validConfigs.gitlab) { + hasAnyValidConfig = true; + result += result ? '\n=== GitLab Status ===\n' : '=== GitLab Status ===\n'; + for (const config of GITLAB_CONFIGS) { + const [status, username, fileCount] = await checkGitLabProject(config.id, config.token); + result += `GitLab: Project ID ${config.id} - ${status} (Username: ${username}, Files: ${fileCount})\n`; + } + } + + // R2 状态检查 + if (validConfigs.r2) { + hasAnyValidConfig = true; + result += result ? '\n=== R2 Storage Status ===\n' : '=== R2 Storage Status ===\n'; + for (const config of R2_CONFIGS) { + const [status, name, bucket, fileCount, totalSize] = await checkR2Storage(config); + result += `R2 Storage: ${name} - ${status} (Bucket: ${bucket}, Files: ${fileCount}, Size: ${totalSize})\n`; + } + } + + // B2 状态检查 + if (validConfigs.b2) { + hasAnyValidConfig = true; + result += result ? '\n=== B2 Storage Status ===\n' : '=== B2 Storage Status ===\n'; + for (const config of B2_CONFIGS) { + const [status, name, bucket, fileCount, totalSize] = await checkB2Storage(config); + result += `B2 Storage: ${name} - ${status} (Bucket: ${bucket}, Files: ${fileCount}, Size: ${totalSize})\n`; + } + } + + // 如果没有任何有效配置 + if (!hasAnyValidConfig) { + result = 'No storage services configured.\n'; + } else { + result = 'Storage status:\n\n' + result; + } + + } catch (error) { + result += `Error during status check: ${error.message}\n`; + } + + return new Response(result, { + headers: { + 'Content-Type': 'text/plain; charset=UTF-8', + 'Access-Control-Allow-Origin': '*' + } + }); + } + + // 添加删除路由 + if (isDeleteRequest) { + const file = url.searchParams.get('file'); + if (!file) { + return new Response('Missing "file" parameter', { + status: 400, + headers: { 'Content-Type': 'text/plain; charset=UTF-8', 'Access-Control-Allow-Origin': '*' } + }); + } + + let result = `Delete：${file}\n`; + + // GitHub 状态 + if (validConfigs.github) { + result += '\n=== GitHub Status ===\n'; + const githubRepos = GITLAB_CONFIGS.map(config => config.name); + for (const repo of githubRepos) { + const status = await deleteGitHubFile(repo, file, GITHUB_PAT); + result += `GitHub: ${repo} - working (private) ${status}\n`; + } + } + + // GitLab 状态 + if (validConfigs.gitlab) { + result += '\n=== GitLab Status ===\n'; + for (const config of GITLAB_CONFIGS) { + const status = await deleteGitLabFile(config.id, file, config.token); + const projectData = await fetch(`https://gitlab.com/api/v4/projects/${config.id}`, { + headers: { 'PRIVATE-TOKEN': config.token } + }).then(res => res.json()); + result += `GitLab: Project ID ${config.id} - working (${projectData.visibility}) ${status}\n`; + } + } + + // R2 存储状态 + if (validConfigs.r2) { + result += '\n=== R2 Storage Status ===\n'; + for (const config of R2_CONFIGS) { + const status = await deleteR2File(config, file); + result += `R2 Storage: ${config.name} - working ${status}\n`; + } + } + + // B2 存储状态 + if (validConfigs.b2) { + result += '\n=== B2 Storage Status ===\n'; + for (const config of B2_CONFIGS) { + const status = await deleteB2File(config, file); + result += `B2 Storage: ${config.name} - working ${status}\n`; + } + } + + return new Response(result, { + headers: { + 'Content-Type': 'text/plain; charset=UTF-8', + 'Access-Control-Allow-Origin': '*' + } + }); + } + + const startTime = Date.now(); + let requests = []; + + // 检查特定服务的请求是否有效 + if (from) { + if (from === 'github' && !validConfigs.github) { + return new Response('GitHub service is not configured.', { + status: 400, + headers: { 'Content-Type': 'text/plain', 'Access-Control-Allow-Origin': '*' } + }); + } + if (from === 'gitlab' && !validConfigs.gitlab) { + return new Response('GitLab service is not configured.', { + status: 400, + headers: { 'Content-Type': 'text/plain', 'Access-Control-Allow-Origin': '*' } + }); + } + if (from === 'r2' && !validConfigs.r2) { + return new Response('R2 storage service is not configured.', { + status: 400, + headers: { 'Content-Type': 'text/plain', 'Access-Control-Allow-Origin': '*' } + }); + } + if (from === 'b2' && !validConfigs.b2) { + return new Response('B2 storage service is not configured.', { + status: 400, + headers: { 'Content-Type': 'text/plain', 'Access-Control-Allow-Origin': '*' } + }); + } + } + + // 生成存储请求 + async function generateStorageRequests() { + let requests = []; + + // 处理请求路径，保留子目录结构 + const getStoragePath = (filePath) => { + return filePath.replace(/^\/+/, '').replace(/\/+/g, '/'); + }; + + if (validConfigs.r2) { + const r2Requests = await Promise.all(R2_CONFIGS.map(async (r2Config) => { + // 构建包含子目录的完整路径 + const storagePath = getStoragePath(`${subPath}/${FILE}`); + // 检查 DIR 是否为空，如果为空则直接拼接 bucket 和 storagePath。 + const r2Path = DIR ? `${r2Config.bucket}/${DIR}/${storagePath}` : `${r2Config.bucket}/${storagePath}`; + const signedRequest = await getSignedUrl(r2Config, 'GET', r2Path); + return { + url: signedRequest.url, + headers: { + ...signedRequest.headers, + 'Accept': '*/*' + }, + source: 'r2', + repo: `${r2Config.name} (${r2Config.bucket})` + }; + })); + requests = [...requests, ...r2Requests]; + } + + if (validConfigs.b2) { + const b2Requests = await Promise.all(B2_CONFIGS.map(async (b2Config) => { + // 构建完整路径，注意 B2 需要包含 bucket 名称 + const storagePath = getStoragePath(`${subPath}/${FILE}`); + const b2Path = `${b2Config.bucket}/${DIR}/${storagePath}`; + + const signedRequest = await getSignedUrl({ + endPoint: b2Config.endPoint, + keyId: b2Config.keyId, + applicationKey: b2Config.applicationKey, + bucket: b2Config.bucket + }, 'GET', b2Path); + + return { + url: signedRequest.url, + headers: { + ...signedRequest.headers, + 'Host': b2Config.endPoint, + 'Accept': '*/*' + }, + source: 'b2', + repo: `${b2Config.name} (${b2Config.bucket})` + }; + })); + requests = [...requests, ...b2Requests]; + } + + return requests; + } + + // 处理不同类型的请求 + if (from === 'where') { + if (validConfigs.github) { + const githubRequests = githubRepos.map(repo => ({ + // buildEncodedPath 过滤空段，避免双斜杠；encodePathSegments 处理中文/日文/韩文 + url: `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${encodePathSegments(getFilePath(DIR, `${subPath}/${FILE}`))}`, + headers: { + 'Authorization': `token ${GITHUB_PAT}`, + 'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'Cloudflare Worker' + }, + source: 'github', + repo: repo, + processResponse: async (response) => { + if (!response.ok) throw new Error('Not found'); + const data = await response.json(); + return { + size: data.size, + exists: true + }; + } + })); + requests = [...requests, ...githubRequests]; + } + + if (validConfigs.gitlab) { + const gitlabRequests = GITLAB_CONFIGS.map(config => ({ + // GitLab where 查询 URL（encodeURIComponent 对整个路径编码，GitLab API 要求） + url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(getFilePath(DIR, `${subPath}/${FILE}`))}?ref=main`, + headers: { + 'PRIVATE-TOKEN': config.token + }, + source: 'gitlab', + repo: config.name, + processResponse: async (response) => { + if (!response.ok) throw new Error('Not found'); + const data = await response.json(); + return { + size: data.size, + exists: true + }; + } + })); + requests = [...requests, ...gitlabRequests]; + } + + const storageRequests = await generateStorageRequests(); + const storageWhereRequests = storageRequests.map(request => ({ + ...request, + processResponse: async (response) => { + if (!response.ok) throw new Error('Not found'); + const size = response.headers.get('content-length'); + return { + size: parseInt(size), + exists: true + }; + } + })); + requests = [...requests, ...storageWhereRequests]; + + } else { + // 获取文件内容模式 + if (from === 'github' && validConfigs.github) { + requests = githubRepos.map(repo => ({ + // buildEncodedPath 过滤空段，避免双斜杠，支持中文/日文/韩文文件名 + url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${buildEncodedPath(fullPath, FILE)}`, + headers: { + 'Authorization': `token ${GITHUB_PAT}`, + 'User-Agent': 'Cloudflare Worker' + }, + source: 'github', + repo: repo + })); + } else if (from === 'gitlab' && validConfigs.gitlab) { + requests = GITLAB_CONFIGS.map(config => ({ + // GitLab 文件获取 URL（encodeURIComponent 对整个路径编码） + url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(getFilePath(DIR, `${subPath}/${FILE}`))}/raw?ref=main`, + headers: { + 'PRIVATE-TOKEN': config.token + }, + source: 'gitlab', + repo: config.name + })); + } else if ((from === 'r2' && validConfigs.r2) || (from === 'b2' && validConfigs.b2)) { + requests = await generateStorageRequests(); + requests = requests.filter(req => req.source === from); + } else if (!from) { + if (validConfigs.github) { + const githubRequests = githubRepos.map(repo => ({ + // buildEncodedPath 过滤空段，避免双斜杠，支持中文/日文/韩文文件名 + url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${buildEncodedPath(fullPath, FILE)}`, + headers: { + 'Authorization': `token ${GITHUB_PAT}`, + 'User-Agent': 'Cloudflare Worker' + }, + source: 'github', + repo: repo + })); + requests = [...requests, ...githubRequests]; + } + + if (validConfigs.gitlab) { + const gitlabRequests = GITLAB_CONFIGS.map(config => ({ + // GitLab URL 构建方式（encodeURIComponent 对整个路径编码） + url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(getFilePath(DIR, `${subPath}/${FILE}`))}/raw?ref=main`, + headers: { + 'PRIVATE-TOKEN': config.token + }, + source: 'gitlab', + repo: config.name + })); + requests = [...requests, ...gitlabRequests]; + } + + const storageRequests = await generateStorageRequests(); + requests = [...requests, ...storageRequests]; + } + } + + // 处理请求和响应 + try { + if (requests.length === 0) { + throw new Error('No valid source specified or no valid configurations found'); + } + + const fetchPromises = requests.map(request => { + const { url, headers, source, repo, processResponse } = request; + + return fetch(new Request(url, { + method: 'GET', + headers: headers + })).then(async response => { + if (from === 'where' && typeof processResponse === 'function') { + try { + const result = await processResponse(response); + const endTime = Date.now(); + const duration = endTime - startTime; + + const formattedSize = result.size > 1024 * 1024 + ? `${(result.size / (1024 * 1024)).toFixed(2)} MB` + : `${(result.size / 1024).toFixed(2)} kB`; + + return { + fileName: FILE, + size: formattedSize, + source: `${source} (${repo})`, + duration: `${duration}ms` + }; + } catch (error) { + throw new Error(`Not found in ${source} (${repo})`); + } + } else { + if (!response.ok) { + throw new Error(`Not found in ${source} (${repo})`); + } + return response; + } + }).catch(error => { + throw new Error(`Error in ${source} (${repo}): ${error.message}`); + }); + }); + + const result = await Promise.any(fetchPromises); + + let response; + if (from === 'where') { + response = new Response(JSON.stringify(result, null, 2), { + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*' + } + }); + } else if (result instanceof Response) { + const blob = await result.blob(); + const headers = { + 'Content-Type': result.headers.get('Content-Type') || 'application/octet-stream', + 'Access-Control-Allow-Origin': '*' + }; + + if (from) { + headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, proxy-revalidate'; + headers['Pragma'] = 'no-cache'; + headers['Expires'] = '0'; + } + + response = new Response(blob, { + status: 200, + headers: headers + }); + } else { + throw new Error("Unexpected result type"); + } + + if (!from && from !== 'where') { + const cacheUrl = new URL(request.url); + const cacheKey = new Request(cacheUrl.toString(), request); + const cache = caches.default; + ctx.waitUntil(cache.put(cacheKey, response.clone())); + } + + return response; + + } catch (error) { + const sourceText = from === 'where' + ? 'in any repository' + : from + ? `from ${from}` + : 'in any configured storage'; + + const errorResponse = new Response( + `404: Cannot find ${FILE} ${sourceText}. ${error.message}`, + { + status: 404, + headers: { + 'Content-Type': 'text/plain', + 'Access-Control-Allow-Origin': '*' + } + } + ); + + return errorResponse; + } + } +} diff --git a/cloudflare_worker/github_gitlab_s3.js b/cloudflare_worker/github_gitlab_s3.js deleted file mode 100644 index ddbc39e..0000000 --- a/cloudflare_worker/github_gitlab_s3.js +++ /dev/null @@ -1,843 +0,0 @@ -// 用户配置区域开始 ================================= - -// GitLab 节点配置 -const GITLAB_CONFIGS = [ - { name: '', id: '', token: '' }, // GitLab 账户1 - { name: '', id: '', token: '' }, // GitLab 账户2 - { name: '', id: '', token: '' }, // GitLab 账户3 - { name: '', id: '', token: '' }, // GitLab 账户4 -]; - -// GitHub 配置 -const GITHUB_REPOS = ['']; // GitHub 仓库名列表 -const GITHUB_USERNAME = ''; // GitHub 用户名 -const GITHUB_PAT = ''; // GitHub 个人访问令牌 - -// R2 存储配置 -const R2_CONFIGS = [ - { - name: '', // 帐户1 ID - accountId: '', // 帐户1 访问密钥 ID - accessKeyId: '', // 帐户1 机密访问密钥 - secretAccessKey: '', // 帐户1 机密访问密钥 - bucket: '' // 帐户1 R2 存储桶名称 - }, - { - name: '', // 帐户2 ID - accountId: '', // 帐户2 访问密钥 ID - accessKeyId: '', // 帐户2 机密访问密钥 - secretAccessKey: '', // 帐户2 机密访问密钥 - bucket: '' // 帐户2 R2 存储桶名称 - }, - // 可以添加更多 R2 配置 -]; - -// B2 存储配置 -const B2_CONFIGS = [ - { - name: '', // 帐户1 名 - endPoint: '', // 账户1 Endpoint - keyId: '', // 账户1 keyID - applicationKey: '', // 账户1 applicationKey - bucket: '' // 账户1桶名 bucketName - }, - { - name: '', // 帐户2 名 - endPoint: '', // 账户2 Endpoint - keyId: '', // 账户2 keyID - applicationKey: '', // 账户2 applicationKey - bucket: '' // 账户2桶名 bucketName - }, - // 可以添加更多 B2 配置 -]; - -// 定义集群访问目录 -const DIR = ''; - -// 定义集群里全部节点连接状态的密码验证，区分大小写（优先使用自定义密码，若为空则使用 GITHUB_PAT） -const CHECK_PASSWORD = '' || GITHUB_PAT; - -// 用户配置区域结束 ================================= - -// AWS SDK 签名相关函数开始 ================================= - -// 获取签名URL -async function getSignedUrl(config, method, path) { - const region = 'auto'; - const service = 's3'; - - // 根据配置类型确定 host 和认证信息 - const host = config.endPoint - ? config.endPoint // B2 配置使用 endPoint - : `${config.accountId}.r2.cloudflarestorage.com`; // R2 配置使用默认格式 - - // 根据配置类型确定认证信息 - const accessKeyId = config.endPoint ? config.keyId : config.accessKeyId; - const secretKey = config.endPoint ? config.applicationKey : config.secretAccessKey; - - const datetime = new Date().toISOString().replace(/[:-]|\.\d{3}/g, ''); - const date = datetime.substr(0, 8); - - const canonicalRequest = [ - method, - '/' + path, - '', - `host:${host}`, - 'x-amz-content-sha256:UNSIGNED-PAYLOAD', - `x-amz-date:${datetime}`, - '', - 'host;x-amz-content-sha256;x-amz-date', - 'UNSIGNED-PAYLOAD' - ].join('\n'); - - const stringToSign = [ - 'AWS4-HMAC-SHA256', - datetime, - `${date}/${region}/${service}/aws4_request`, - await sha256(canonicalRequest) - ].join('\n'); - - const signature = await getSignature( - secretKey, // 使用映射后的密钥 - date, - region, - service, - stringToSign - ); - - const authorization = [ - `AWS4-HMAC-SHA256 Credential=${accessKeyId}/${date}/${region}/${service}/aws4_request`, // 使用映射后的密钥ID - `SignedHeaders=host;x-amz-content-sha256;x-amz-date`, - `Signature=${signature}` - ].join(', '); - - return { - url: `https://${host}/${path}`, - headers: { - 'Authorization': authorization, - 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - 'x-amz-date': datetime, - 'Host': host - } - }; -} - -// SHA256 哈希函数 -async function sha256(message) { - const msgBuffer = new TextEncoder().encode(message); - const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer); - return Array.from(new Uint8Array(hashBuffer)) - .map(b => b.toString(16).padStart(2, '0')) - .join(''); - } - - // HMAC-SHA256 函数 - async function hmacSha256(key, message) { - const keyBuffer = key instanceof ArrayBuffer ? key : new TextEncoder().encode(key); - const messageBuffer = new TextEncoder().encode(message); - - const cryptoKey = await crypto.subtle.importKey( - 'raw', - keyBuffer, - { name: 'HMAC', hash: 'SHA-256' }, - false, - ['sign'] - ); - - const signature = await crypto.subtle.sign( - 'HMAC', - cryptoKey, - messageBuffer - ); - - return signature; - } - - // 获取签名 - async function getSignature(secret, date, region, service, stringToSign) { - const kDate = await hmacSha256('AWS4' + secret, date); - const kRegion = await hmacSha256(kDate, region); - const kService = await hmacSha256(kRegion, service); - const kSigning = await hmacSha256(kService, 'aws4_request'); - const signature = await hmacSha256(kSigning, stringToSign); - - return Array.from(new Uint8Array(signature)) - .map(b => b.toString(16).padStart(2, '0')) - .join(''); - } - -export default { - async fetch(request, env, ctx) { - const url = new URL(request.url); - const from = url.searchParams.get('from')?.toLowerCase(); - - // 只在没有 from 参数时才检查和使用缓存 - let cacheResponse; - if (!from) { - // 检查缓存 - const cacheUrl = new URL(request.url); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - cacheResponse = await cache.match(cacheKey); - - if (cacheResponse) { - return cacheResponse; - } - } - - const isValidGithubRepos = Array.isArray(GITHUB_REPOS) && - GITHUB_REPOS.length > 0 && - GITHUB_REPOS.some(repo => repo.trim() !== ''); - - const githubRepos = isValidGithubRepos - ? GITHUB_REPOS.filter(repo => repo.trim() !== '') - : GITLAB_CONFIGS.map(config => config.name); - - const FILE = url.pathname.split('/').pop(); - - if (url.pathname === `/${CHECK_PASSWORD}`) { - const response = await listProjects(GITLAB_CONFIGS, githubRepos, GITHUB_USERNAME, GITHUB_PAT); - // 不缓存状态检查页面 - return response; - } - - const startTime = Date.now(); - - // 根据不同的访问方式构建请求 - let requests = []; - - // 在生成请求时需要合并 R2 和 B2 的请求 - const generateStorageRequests = async () => { - const r2Requests = await Promise.all(R2_CONFIGS.map(async (r2Config) => { - const r2Path = `${r2Config.bucket}/${DIR}/${FILE}`; - const signedRequest = await getSignedUrl(r2Config, 'GET', r2Path); - return { - url: signedRequest.url, - headers: signedRequest.headers, - source: 'r2', - repo: `${r2Config.name} (${r2Config.bucket})` - }; - })); - - const b2Requests = await Promise.all(B2_CONFIGS.map(async (b2Config) => { - const b2Path = `${b2Config.bucket}/${DIR}/${FILE}`; - const signedRequest = await getSignedUrl(b2Config, 'GET', b2Path); - return { - url: signedRequest.url, - headers: signedRequest.headers, - source: 'b2', - repo: `${b2Config.name} (${b2Config.bucket})` - }; - })); - - return [...r2Requests, ...b2Requests]; - }; - - if (from === 'where') { - // 获取文件信息模式 - const githubRequests = githubRepos.map(repo => ({ - url: `https://api.github.com/repos/${GITHUB_USERNAME}/${repo}/contents/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found'); - const data = await response.json(); - return { - size: data.size, - exists: true - }; - } - })); - - const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found'); - const data = await response.json(); - const size = atob(data.content).length; - return { - size: size, - exists: true - }; - } - })); - - const r2Requests = await generateStorageRequests(); - const r2WhereRequests = r2Requests.map(request => ({ - ...request, - processResponse: async (response) => { - if (!response.ok) throw new Error('Not found'); - const size = response.headers.get('content-length'); - return { - size: parseInt(size), - exists: true - }; - } - })); - - requests = [...githubRequests, ...gitlabRequests, ...r2WhereRequests]; - - } else { - // 获取文件内容模式 - if (from === 'github') { - requests = githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo - })); - } else if (from === 'gitlab') { - requests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name - })); - } else if (from === 'r2') { - // 只从 R2 存储获取 - requests = await Promise.all(R2_CONFIGS.map(async (r2Config) => { - const r2Path = `${r2Config.bucket}/${DIR}/${FILE}`; - const signedRequest = await getSignedUrl(r2Config, 'GET', r2Path); - return { - url: signedRequest.url, - headers: signedRequest.headers, - source: 'r2', - repo: `${r2Config.name} (${r2Config.bucket})` - }; - })); - } else if (from === 'b2') { - // 只从 B2 存储获取 - requests = await Promise.all(B2_CONFIGS.map(async (b2Config) => { - const b2Path = `${b2Config.bucket}/${DIR}/${FILE}`; - const signedRequest = await getSignedUrl(b2Config, 'GET', b2Path); - return { - url: signedRequest.url, - headers: signedRequest.headers, - source: 'b2', - repo: `${b2Config.name} (${b2Config.bucket})` - }; - })); - } else { - // 如果没有指定来源，则从所有源获取 - const githubRequests = githubRepos.map(repo => ({ - url: `https://raw.githubusercontent.com/${GITHUB_USERNAME}/${repo}/main/${DIR}/${FILE}`, - headers: { - 'Authorization': `token ${GITHUB_PAT}`, - 'User-Agent': 'Cloudflare Worker' - }, - source: 'github', - repo: repo - })); - - const gitlabRequests = GITLAB_CONFIGS.map(config => ({ - url: `https://gitlab.com/api/v4/projects/${config.id}/repository/files/${encodeURIComponent(`${DIR}/${FILE}`)}/raw?ref=main`, - headers: { - 'PRIVATE-TOKEN': config.token - }, - source: 'gitlab', - repo: config.name - })); - - const r2Requests = await generateStorageRequests(); - - requests = [...githubRequests, ...gitlabRequests, ...r2Requests]; - } - } - - // 发送请求并处理响应 - const fetchPromises = requests.map(request => { - const { url, headers, source, repo, processResponse } = request; - - return fetch(new Request(url, { - method: 'GET', - headers: headers - })).then(async response => { - if (from === 'where' && typeof processResponse === 'function') { - // 使用 `processResponse` 处理 where 查询逻辑 - try { - const result = await processResponse(response); - const endTime = Date.now(); - const duration = endTime - startTime; - - const formattedSize = result.size > 1024 * 1024 - ? `${(result.size / (1024 * 1024)).toFixed(2)} MB` - : `${(result.size / 1024).toFixed(2)} kB`; - - return { - fileName: FILE, - size: formattedSize, - source: `${source} (${repo})`, - duration: `${duration}ms` - }; - } catch (error) { - throw new Error(`Not found in ${source} (${repo})`); - } - } else { - // 对于内容获取，直接返回响应 - if (!response.ok) { - throw new Error(`Not found in ${source} (${repo})`); - } - return response; - } - }).catch(error => { - throw new Error(`Error in ${source} (${repo}): ${error.message}`); - }); - }); - - try { - if (requests.length === 0) { - throw new Error('No valid source specified'); - } - - const result = await Promise.any(fetchPromises); - - let response; - if (from === 'where') { - // 如果是 where 查询，返回 JSON 格式响应 - response = new Response(JSON.stringify(result, null, 2), { - headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*' - } - }); - } else if (result instanceof Response) { - // 先读取响应体 - const blob = await result.blob(); - - // 创建新的响应，设置适当的头部 - const headers = { - 'Content-Type': result.headers.get('Content-Type') || 'application/octet-stream', - 'Access-Control-Allow-Origin': '*' - }; - - // 如果有 from 参数，添加禁止缓存的头部 - if (from) { - headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, proxy-revalidate'; - headers['Pragma'] = 'no-cache'; - headers['Expires'] = '0'; - } - - response = new Response(blob, { - status: 200, - headers: headers - }); - } else { - throw new Error("Unexpected result type"); - } - - // 只在没有 from 参数且不是 where 查询时才缓存响应 - if (!from && from !== 'where') { - const cacheUrl = new URL(request.url); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - ctx.waitUntil(cache.put(cacheKey, response.clone())); - } - - return response; - - } catch (error) { - const sourceText = from === 'where' - ? 'in any repository' - : from - ? `from ${from}` - : 'in the GitHub, GitLab, R2, and B2 storage'; - - const errorResponse = new Response( - `404: Cannot find the ${FILE} ${sourceText}.`, - { - status: 404, - headers: { - 'Content-Type': 'text/plain', - 'Access-Control-Allow-Origin': '*' - } - } - ); - - return errorResponse; - } - } -}; - -// 列出所有节点仓库状态 -async function listProjects(gitlabConfigs, githubRepos, githubUsername, githubPat) { - let result = 'GitHub, GitLab, R2 and B2 Storage status:\n\n'; - - try { - // 并发执行所有检查 - const [username, ...allChecks] = await Promise.all([ - getGitHubUsername(githubPat), - ...githubRepos.map(repo => - checkGitHubRepo(githubUsername, repo, githubPat) - ), - ...gitlabConfigs.map(config => - checkGitLabProject(config.id, config.token) - ), - ...R2_CONFIGS.map(config => - checkR2Storage(config) - ), - ...B2_CONFIGS.map(config => - checkB2Storage(config) - ) - ]); - - // 计算各类检查结果的数量和起始位置 - const githubCount = githubRepos.length; - const gitlabCount = gitlabConfigs.length; - const r2Count = R2_CONFIGS.length; - const b2Count = B2_CONFIGS.length; - - // 分割检查结果 - const githubResults = allChecks.slice(0, githubCount); - const gitlabResults = allChecks.slice(githubCount, githubCount + gitlabCount); - const r2Results = allChecks.slice(githubCount + gitlabCount, githubCount + gitlabCount + r2Count); - const b2Results = allChecks.slice(githubCount + gitlabCount + r2Count); - - // 添加 GitHub 结果 - githubRepos.forEach((repo, index) => { - const [status, fileCount, totalSize] = githubResults[index]; - const formattedSize = formatSize(totalSize); - result += `GitHub: ${repo} - ${status} (Username: ${username}, Files: ${fileCount}, Size: ${formattedSize})\n`; - }); - - // 添加 GitLab 结果 - gitlabConfigs.forEach((config, index) => { - const [status, username, fileCount, totalSize] = gitlabResults[index]; - result += `GitLab: Project ID ${config.id} - ${status} (Username: ${username}, Files: ${fileCount}, Size: ${totalSize})\n`; - }); - - // 添加 R2 结果 - r2Results.forEach(([status, name, bucket, fileCount, totalSize]) => { - result += `R2 Storage: ${name} - ${status} (Bucket: ${bucket}, Files: ${fileCount}, Size: ${totalSize})\n`; - }); - - // 添加 B2 结果 - b2Results.forEach(([status, name, bucket, fileCount, totalSize]) => { - result += `B2 Storage: ${name} - ${status} (Bucket: ${bucket}, Files: ${fileCount}, Size: ${totalSize})\n`; - }); - - } catch (error) { - result += `Error during status check: ${error.message}\n`; - } - - return new Response(result, { - headers: { - 'Content-Type': 'text/plain', - 'Access-Control-Allow-Origin': '*' - } - }); -} - -// 文件大小格式化函数 -function formatSize(sizeInBytes) { - if (sizeInBytes >= 1024 * 1024 * 1024) { - return `${(sizeInBytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; - } else if (sizeInBytes >= 1024 * 1024) { - return `${(sizeInBytes / (1024 * 1024)).toFixed(2)} MB`; - } else { - return `${(sizeInBytes / 1024).toFixed(2)} kB`; - } -} - -// 获取 GitHub 用户名的异步函数 -async function getGitHubUsername(pat) { - const url = 'https://api.github.com/user'; // GitHub 用户信息 API 地址 - try { - const response = await fetch(url, { - headers: { - 'Authorization': `token ${pat}`, // 使用个人访问令牌进行授权 - 'Accept': 'application/vnd.github.v3+json', // 指定接受的响应格式 - 'User-Agent': 'Cloudflare Worker' // 用户代理 - } - }); - - // 如果响应状态为 200，表示成功 - if (response.status === 200) { - const data = await response.json(); // 解析 JSON 数据 - return data.login; // 返回用户登录名 - } else { - console.error('GitHub API Error:', response.status); // 记录错误状态 - return 'Unknown'; // 返回未知状态 - } - } catch (error) { - console.error('GitHub request error:', error); // 记录请求错误 - return 'Error'; // 返回错误状态 - } -} - -// 检查 GitHub 仓库的异步函数 -async function checkGitHubRepo(owner, repo, pat) { - const repoUrl = `https://api.github.com/repos/${owner}/${repo}`; - const contentsUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${DIR}`; // 直接检查指定目录 - - const headers = { - 'Authorization': `token ${pat}`, - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'Cloudflare Worker' - }; - - try { - // 并发请求获取仓库信息和目录内容 - const [repoResponse, contentsResponse] = await Promise.all([ - fetch(repoUrl, { headers }), - fetch(contentsUrl, { headers }) - ]); - - const repoData = await repoResponse.json(); - - if (repoResponse.status !== 200) { - throw new Error(`Repository error: ${repoData.message}`); - } - - if (contentsResponse.status !== 200) { - return [`working (${repoData.private ? 'private' : 'public'})`, 0, 0]; - } - - const contentsData = await contentsResponse.json(); - - // 计算文件数量和总大小 - const fileStats = contentsData.reduce((acc, item) => { - if (item.type === 'file') { - return { - count: acc.count + 1, - size: acc.size + (item.size || 0) - }; - } - return acc; - }, { count: 0, size: 0 }); - - return [ - `working (${repoData.private ? 'private' : 'public'})`, - fileStats.count, - fileStats.size - ]; - - } catch (error) { - console.error(`Error checking GitHub repo ${repo}:`, error); - return [`error: ${error.message}`, 0, 0]; - } -} - -// 获取单个文件大小的辅助函数 -async function getFileSizeFromGitLab(projectId, filePath, pat, retries = 3) { - for (let i = 0; i < retries; i++) { - try { - // 使用 raw 端点和 HEAD 请求获取文件大小 - const fileUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/files${filePath}/raw?ref=main`; - const response = await fetch(fileUrl, { - method: 'HEAD', - headers: { 'PRIVATE-TOKEN': pat } - }); - - if (response.status === 200) { - const contentLength = response.headers.get('content-length'); - return contentLength ? parseInt(contentLength, 10) : 0; - } else if (response.status === 429) { - // 遇到限流时等待后重试 - await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1))); - continue; - } - console.error(`Failed to get file size: ${response.status}`); - return 0; - } catch (error) { - if (i === retries - 1) { - console.error(`Error fetching file size:`, error); - } - if (i < retries - 1) { - await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1))); - continue; - } - } - return 0; - } - return 0; -} - -// 添加并发控制的辅助函数 -async function asyncPool(concurrency, iterable, iteratorFn) { - const ret = []; - const executing = new Set(); - - for (const item of iterable) { - const p = Promise.resolve().then(() => iteratorFn(item)); - ret.push(p); - executing.add(p); - - const clean = () => executing.delete(p); - p.then(clean).catch(clean); - - if (executing.size >= concurrency) { - await Promise.race(executing); - } - } - - return Promise.all(ret); -} - -// 检查 GitLab 项目的异步函数 -async function checkGitLabProject(projectId, pat) { - const projectUrl = `https://gitlab.com/api/v4/projects/${projectId}`; - // 步骤1: 获取文件列表 - const filesUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/tree?ref=main&path=${DIR}`; - - try { - const [projectResponse, filesResponse] = await Promise.all([ - fetch(projectUrl, { - headers: { 'PRIVATE-TOKEN': pat } - }), - fetch(filesUrl, { - headers: { 'PRIVATE-TOKEN': pat } - }) - ]); - - if (projectResponse.status === 200) { - const projectData = await projectResponse.json(); - let totalSize = 0; - let fileCount = 0; - - if (filesResponse.status === 200) { - const filesData = await filesResponse.json(); - fileCount = filesData.length; - - if (fileCount > 0) { - console.log(`Found ${fileCount} files in ${DIR} directory`); - - // 步骤2: 并发获取每个文件的大小 - const CONCURRENT_REQUESTS = 100; - const sizes = await asyncPool(CONCURRENT_REQUESTS, filesData, async (file) => { - // 构造文件路径，格式为 /files%2Ffilename - const encodedPath = `/${encodeURIComponent(file.path)}`; - const size = await getFileSizeFromGitLab(projectId, encodedPath, pat); - console.log(`File: ${file.path}, Size: ${formatSize(size)}`); - return size; - }); - - totalSize = sizes.reduce((acc, size) => acc + size, 0); - console.log(`Total size: ${formatSize(totalSize)}`); - } - } - - return [ - `working (${projectData.visibility})`, - projectData.owner.username, - fileCount, - formatSize(totalSize) - ]; - } else if (projectResponse.status === 404) { - return ['not found', 'Unknown', 0, '0 B']; - } else { - return ['disconnect', 'Unknown', 0, '0 B']; - } - } catch (error) { - console.error('GitLab project check error:', error); - return ['disconnect', 'Error', 0, '0 B']; - } -} - -// 检查 R2 存储状态 -async function checkR2Storage(r2Config) { - try { - // 1. 列出目录下所有文件 - const listPath = `${r2Config.bucket}`; // 列出根目录 - const signedRequest = await getSignedUrl(r2Config, 'GET', listPath); - - const response = await fetch(signedRequest.url, { - headers: signedRequest.headers - }); - - let fileCount = 0; - let totalSize = 0; - - if (response.ok) { - const data = await response.text(); - // 解析 XML 响应 - const keys = data.match(/([^<]+)<\/Key>/g) || []; - const sizes = data.match(/(\d+)<\/Size>/g) || []; - - // 只计算指定目录下的文件 - keys.forEach((key, index) => { - const filePath = key.replace(/|<\/Key>/g, ''); - if (filePath.startsWith(DIR + '/')) { - fileCount++; - const size = parseInt(sizes[index]?.replace(/|<\/Size>/g, '') || String(0), 10); - totalSize += size; - } - }); - } - - // 即使文件不存在，只要能访问到存储桶就认为是正常的 - const status = response.ok ? 'working' : 'error'; - - return [ - status, - r2Config.name, - r2Config.bucket, - fileCount, - formatSize(totalSize) - ]; - } catch (error) { - console.error('R2 Storage error:', error); - return ['error', r2Config.name, 'connection failed', 0, '0 B']; - } -} - -// 检查 B2 存储状态 -async function checkB2Storage(b2Config) { - try { - // 1. 列出目录下所有文件 - const listPath = `${b2Config.bucket}`; // 列出根目录 - const signedRequest = await getSignedUrl(b2Config, 'GET', listPath); - - const response = await fetch(signedRequest.url, { - headers: signedRequest.headers - }); - - let fileCount = 0; - let totalSize = 0; - - if (response.ok) { - const data = await response.text(); - // 解析 XML 响应 - const keys = data.match(/([^<]+)<\/Key>/g) || []; - const sizes = data.match(/(\d+)<\/Size>/g) || []; - - // 只计算指定目录下的文件 - keys.forEach((key, index) => { - const filePath = key.replace(/|<\/Key>/g, ''); - if (filePath.startsWith(DIR + '/')) { - fileCount++; - const size = parseInt(sizes[index]?.replace(/|<\/Size>/g, '') || String(0), 10); - totalSize += size; - } - }); - } - - // 即使文件不存在，只要能访问到存储桶就认为是正常的 - const status = (response.status === 404 || response.status === 403 || response.ok) ? 'working' : 'error'; - - return [ - status, - b2Config.name, - b2Config.bucket, - fileCount, - formatSize(totalSize) - ]; - } catch (error) { - console.error('B2 Storage error:', error); - return ['error', b2Config.name, 'connection failed', 0, '0 B']; - } -} \ No newline at end of file diff --git a/cloudflare_worker/github_only.js b/cloudflare_worker/github_only.js index 02d3538..d9264c4 100644 --- a/cloudflare_worker/github_only.js +++ b/cloudflare_worker/github_only.js @@ -154,8 +154,17 @@ export default { ); } - // 原有的文件获取逻辑保持不变 - const FILE = url.pathname.split('/').pop(); + // 获取请求路径并解码，支持中文/日文/韩文等非 ASCII 文件名 + let requestPath; + try { + requestPath = decodeURIComponent(url.pathname); + } catch (e) { + requestPath = url.pathname; + } + const FILE = requestPath.split('/').pop(); + // 对路径各段重新编码，过滤空段避免双斜杠（如 DIR 为空时 main//文件名） + const encodePathSegments = (path) => + path.split('/').filter(seg => seg !== '').map(seg => encodeURIComponent(seg)).join('/'); // 缓存检查 const cacheUrl = new URL(request.url); @@ -167,10 +176,12 @@ export default { return cacheResponse; } - // 构建 GitHub raw 文件的 URL 列表 - const urls = REPOS.map(repoNumber => - `https://raw.githubusercontent.com/${CONFIG.GITHUB_USERNAME}/${CONFIG.GITHUB_REPO_PREFIX}${repoNumber}/main/${CONFIG.DIR}/${FILE}` - ); + // 构建 GitHub raw 文件的 URL 列表，使用 encodePathSegments 支持中文/日文/韩文文件名，过滤空段避免双斜杠 + const urls = REPOS.map(repoNumber => { + const parts = [CONFIG.DIR, FILE].filter(p => p && p !== ''); + const encodedPath = parts.map(p => encodePathSegments(p)).join('/'); + return `https://raw.githubusercontent.com/${CONFIG.GITHUB_USERNAME}/${CONFIG.GITHUB_REPO_PREFIX}${repoNumber}/main/${encodedPath}`; + }); // 创建并发请求 const requests = urls.map(githubUrl => { diff --git a/cloudflare_worker/gitlab_only.js b/cloudflare_worker/gitlab_only.js index 7e2c08c..3e551bb 100644 --- a/cloudflare_worker/gitlab_only.js +++ b/cloudflare_worker/gitlab_only.js @@ -111,7 +111,8 @@ async function checkGitLabProject(projectId, pat) { // CONCURRENT_REQUESTS: 控制同时发起的并发请求数，防止对 GitLab API 施加过大压力 const CONCURRENT_REQUESTS = 100; const sizes = await asyncPool(CONCURRENT_REQUESTS, filesData, async (file) => { - const encodedPath = `/${encodeURIComponent(file.path)}`; + // 逐段编码路径，支持中文/日文/韩文文件名，同时保留多级目录的斜杠结构 + const encodedPath = '/' + file.path.split('/').map(seg => encodeURIComponent(seg)).join('/'); const size = await getFileSizeFromGitLab(projectId, encodedPath, pat); return size; }); @@ -204,7 +205,8 @@ export default { } // 构建文件路径 - const filePath = pathParts.slice(1).join('/'); + // 对路径各段逐段编码，支持中文/日文/韩文等非 ASCII 文件名，保留多级目录结构 + const filePath = pathParts.slice(1).map(seg => encodeURIComponent(seg)).join('/'); console.log('Accessing file:', filePath); // 检查文件是否存在 diff --git a/github_to_gitlab/.github/workflows/cluster_sync.yml b/github_to_gitlab/.github/workflows/cluster_sync.yml index 6cd25ed..c372440 100644 --- a/github_to_gitlab/.github/workflows/cluster_sync.yml +++ b/github_to_gitlab/.github/workflows/cluster_sync.yml @@ -8,9 +8,15 @@ on: jobs: GitHub_To_GitLab : runs-on: ubuntu-latest + + env: + ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} + WORKER_NAME: ${{ secrets.WORKER_NAME }} + API_TOKEN: ${{ secrets.API_TOKEN }} + steps: - name: Checkout code uses: actions/checkout@v4.2.2 - name: Clone Repo to GitLab - uses: fscarmen2/github_to_gitlab@v1.0.0 \ No newline at end of file + uses: fscarmen2/github_to_gitlab@v1.0.1 \ No newline at end of file diff --git a/github_to_gitlab/config.yml b/github_to_gitlab/config.yml deleted file mode 100644 index 6c00a46..0000000 --- a/github_to_gitlab/config.yml +++ /dev/null @@ -1,33 +0,0 @@ -# 以下所有的 KV 填写格式要求: 1. 修改的地方需要把尖括号一并去掉；2. : 冒号后面有空格 -## GitHub 部分 -github_pat: <改为 GitHub PAT获取到的 Token> -github_repo_prefix: <改为 GitHub 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> - -## GitLab 部分 -gitlab_repo_prefix: <改为 GitLab 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> -gitlab_username: <改为 GitLab 用户名> -gitlab_pats: - # 注意，下面只填仓库名数字序号，比如 repo1,repo2,repo3,repo4，就只填 1,2,3,4 即可 - 1: <改为 GitLab 仓库名1 api> - 2: <改为 GitLab 仓库名2 api> - 3: <改为 GitLab 仓库名3 api> - 4: <改为 GitLab 仓库名4 api> - -## CloudFlare R2 部分 -r2_accounts: - - name: <改为 R2 帐户1 名，建议用 email 方便管理> - account_id: <改为 R2 账户1 ID> - access_key_id: <改为 R2 账户1 访问密钥 ID> - secret_access_key: <改为 R2 账户1 机密访问密钥> - bucket: <改为 R2 账户1 存储桶名称> - dir: <改为 R2 账户1 文件存放目录> - - name: <改为 R2 帐户2 名，建议用 email 方便管理> - account_id: <改为 R2 账户2 ID> - access_key_id: <改为 R2 账户2 访问密钥 ID> - secret_access_key: <改为 R2 账户2 机密访问密钥> - bucket: <改为 R2 账户2 存储桶名称> - dir: <改为 R2 账户2 文件存放目录> - -## 备份策略 -strategy: size # 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 -delete: true # 可选 [true (默认) | false]，已复制到 GitHub 的文件，是否从 R2 删除 \ No newline at end of file diff --git a/gitlab_to_github/config.yml b/gitlab_to_github/config.yml index 6c00a46..2be4c75 100644 --- a/gitlab_to_github/config.yml +++ b/gitlab_to_github/config.yml @@ -5,29 +5,4 @@ github_repo_prefix: <改为 GitHub 仓库名前缀，比如 repo1,repo2,repo3,re ## GitLab 部分 gitlab_repo_prefix: <改为 GitLab 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> -gitlab_username: <改为 GitLab 用户名> -gitlab_pats: - # 注意，下面只填仓库名数字序号，比如 repo1,repo2,repo3,repo4，就只填 1,2,3,4 即可 - 1: <改为 GitLab 仓库名1 api> - 2: <改为 GitLab 仓库名2 api> - 3: <改为 GitLab 仓库名3 api> - 4: <改为 GitLab 仓库名4 api> - -## CloudFlare R2 部分 -r2_accounts: - - name: <改为 R2 帐户1 名，建议用 email 方便管理> - account_id: <改为 R2 账户1 ID> - access_key_id: <改为 R2 账户1 访问密钥 ID> - secret_access_key: <改为 R2 账户1 机密访问密钥> - bucket: <改为 R2 账户1 存储桶名称> - dir: <改为 R2 账户1 文件存放目录> - - name: <改为 R2 帐户2 名，建议用 email 方便管理> - account_id: <改为 R2 账户2 ID> - access_key_id: <改为 R2 账户2 访问密钥 ID> - secret_access_key: <改为 R2 账户2 机密访问密钥> - bucket: <改为 R2 账户2 存储桶名称> - dir: <改为 R2 账户2 文件存放目录> - -## 备份策略 -strategy: size # 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 -delete: true # 可选 [true (默认) | false]，已复制到 GitHub 的文件，是否从 R2 删除 \ No newline at end of file +gitlab_username: <改为 GitLab 用户名> \ No newline at end of file diff --git a/gitlab_to_github/sync_to_github.sh b/gitlab_to_github/sync_to_github.sh index 6bbbfa6..8e565c8 100644 --- a/gitlab_to_github/sync_to_github.sh +++ b/gitlab_to_github/sync_to_github.sh @@ -3,7 +3,7 @@ set -e # 从文件读取配置，为防止用户没有去掉尖括号，故用 sed 处理强制去掉所有的 < > 符号 -CONFIG=$(sed "s/[<>]//g" config.yml) +CONFIG=$(sed 's/[<>]//g; s/:/: /g; s/:[[:space:]]\+/: /g' config.yml) GITHUB_PAT=$(awk '$1 ~ "^github_pat" {sub(/[^:]*:[[:space:]]*/, ""); print}' <<< "$CONFIG") GITLAB_USERNAME=$(awk '$1 ~ "^gitlab_username" {sub(/[^:]*:[[:space:]]*/, ""); print}' <<< "$CONFIG") GITHUB_REPO_PREFIX=$(awk '$1 ~ "^github_repo_prefix" {sub(/[^:]*:[[:space:]]*/, ""); print}' <<< "$CONFIG") diff --git a/r2_to_github/.github/workflows/r2_to_github.yml b/r2_to_github/.github/workflows/r2_to_github.yml index 63480a4..7d6e8b3 100644 --- a/r2_to_github/.github/workflows/r2_to_github.yml +++ b/r2_to_github/.github/workflows/r2_to_github.yml @@ -8,9 +8,15 @@ on: jobs: R2_To_GitHub: runs-on: ubuntu-latest + + env: + ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} + WORKER_NAME: ${{ secrets.WORKER_NAME }} + API_TOKEN: ${{ secrets.API_TOKEN }} + steps: - name: Checkout code uses: actions/checkout@v4.2.2 - name: Sync files to GitHub - uses: fscarmen2/r2_to_github@v1.0.0 \ No newline at end of file + uses: fscarmen2/r2_to_github@v1.0.2 \ No newline at end of file diff --git a/r2_to_github/config.yml b/r2_to_github/config.yml deleted file mode 100644 index 6c00a46..0000000 --- a/r2_to_github/config.yml +++ /dev/null @@ -1,33 +0,0 @@ -# 以下所有的 KV 填写格式要求: 1. 修改的地方需要把尖括号一并去掉；2. : 冒号后面有空格 -## GitHub 部分 -github_pat: <改为 GitHub PAT获取到的 Token> -github_repo_prefix: <改为 GitHub 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> - -## GitLab 部分 -gitlab_repo_prefix: <改为 GitLab 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> -gitlab_username: <改为 GitLab 用户名> -gitlab_pats: - # 注意，下面只填仓库名数字序号，比如 repo1,repo2,repo3,repo4，就只填 1,2,3,4 即可 - 1: <改为 GitLab 仓库名1 api> - 2: <改为 GitLab 仓库名2 api> - 3: <改为 GitLab 仓库名3 api> - 4: <改为 GitLab 仓库名4 api> - -## CloudFlare R2 部分 -r2_accounts: - - name: <改为 R2 帐户1 名，建议用 email 方便管理> - account_id: <改为 R2 账户1 ID> - access_key_id: <改为 R2 账户1 访问密钥 ID> - secret_access_key: <改为 R2 账户1 机密访问密钥> - bucket: <改为 R2 账户1 存储桶名称> - dir: <改为 R2 账户1 文件存放目录> - - name: <改为 R2 帐户2 名，建议用 email 方便管理> - account_id: <改为 R2 账户2 ID> - access_key_id: <改为 R2 账户2 访问密钥 ID> - secret_access_key: <改为 R2 账户2 机密访问密钥> - bucket: <改为 R2 账户2 存储桶名称> - dir: <改为 R2 账户2 文件存放目录> - -## 备份策略 -strategy: size # 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 -delete: true # 可选 [true (默认) | false]，已复制到 GitHub 的文件，是否从 R2 删除 \ No newline at end of file diff --git a/s3_to_github/.github/workflows/s3_to_github.yml b/s3_to_github/.github/workflows/s3_to_github.yml index 4bdcf05..c26b050 100644 --- a/s3_to_github/.github/workflows/s3_to_github.yml +++ b/s3_to_github/.github/workflows/s3_to_github.yml @@ -8,9 +8,15 @@ on: jobs: S3_To_GitHub: runs-on: ubuntu-latest + + env: + ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} + WORKER_NAME: ${{ secrets.WORKER_NAME }} + API_TOKEN: ${{ secrets.API_TOKEN }} + steps: - name: Checkout code uses: actions/checkout@v4.2.2 - name: Sync files to GitHub - uses: fscarmen2/r2_to_github@v1.0.1 \ No newline at end of file + uses: fscarmen2/r2_to_github@v1.0.2 \ No newline at end of file diff --git a/s3_to_github/config.yml b/s3_to_github/config.yml deleted file mode 100644 index b1b4825..0000000 --- a/s3_to_github/config.yml +++ /dev/null @@ -1,48 +0,0 @@ -# 以下所有的 KV 填写格式要求: 1. 修改的地方需要把尖括号一并去掉；2. : 冒号后面有空格 -## GitHub 部分 -github_pat: <改为 GitHub PAT获取到的 Token> -github_repo_prefix: <改为 GitHub 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> - -## GitLab 部分 -gitlab_repo_prefix: <改为 GitLab 仓库名前缀，比如 repo1,repo2,repo3,repo4, 就只填 repo> -gitlab_username: <改为 GitLab 用户名> -gitlab_pats: - # 注意，下面只填仓库名数字序号，比如 repo1,repo2,repo3,repo4，就只填 1,2,3,4 即可 - 1: <改为 GitLab 仓库名1 api> - 2: <改为 GitLab 仓库名2 api> - 3: <改为 GitLab 仓库名3 api> - 4: <改为 GitLab 仓库名4 api> - -## CloudFlare R2 部分 -r2_accounts: - - name: <改为 R2 帐户1 名，建议用 email 方便管理> - account_id: <改为 R2 账户1 ID> - access_key_id: <改为 R2 账户1 访问密钥 ID> - secret_access_key: <改为 R2 账户1 机密访问密钥> - bucket: <改为 R2 账户1 存储桶名称> - dir: <改为 R2 账户1 文件存放目录> - - name: <改为 R2 帐户2 名，建议用 email 方便管理> - account_id: <改为 R2 账户2 ID> - access_key_id: <改为 R2 账户2 访问密钥 ID> - secret_access_key: <改为 R2 账户2 机密访问密钥> - bucket: <改为 R2 账户2 存储桶名称> - dir: <改为 R2 账户2 文件存放目录> - -## Backblaze B2 部分 -b2_accounts: - - name: <改为 B2 帐户1 名，建议用 email 方便管理> - endpoint: <改为 B2 账户1 Endpoint，以 s3 开头的，不要以 https:// 开头> - key_id: <改为 B2 账户1 keyID> - application_key: <改为 B2 账户1 applicationKey> - bucket: <改为 B2 账户1 bucketName，这个是全 Backblaze 唯一的> - dir: <改为 B2 账户1 文件存放目录> - - name: <改为 B2 帐户2 名，建议用 email 方便管理> - endpoint: <改为 B2 账户2 Endpoint，以 s3 开头的，不要以 https:// 开头> - key_id: <改为 B2 账户2 keyID> - application_key: <改为 B2 账户2 applicationKey> - bucket: <改为 B2 账户2 bucketName，这个是全 Backblaze 唯一的> - dir: <改为 B2 账户2 文件存放目录> - -## 备份策略 -strategy: size # 可选 [size (默认) | quantity | 指定节点]; size: 选择容量最少的仓库来存储文件; quantity: 选择文件最少的仓库来存储文件; 指定节点: 比如 pic1 或者 pic2 -delete: true # 可选 [true (默认) | false]，已复制到 GitHub 的文件，是否从 R2 删除 \ No newline at end of file