import { createElement, createFragment, createRender } from "../../lib/skeleton/index.js"; import { toHref } from "../../lib/skeleton/router.js"; import rxjs, { effect, onClick } from "../../lib/rx.js"; import { forwardURLParams } from "../../lib/path.js"; import { animate, slideYOut } from "../../lib/animate.js"; import { qs, qsa } from "../../lib/dom.js"; import { AjaxError } from "../../lib/error.js"; import assert from "../../lib/assert.js"; import { get as getConfig } from "../../model/config.js"; import { loadCSS } from "../../helpers/loader.js"; import { currentPath, isNativeFileUpload } from "./helper.js"; import { getPermission, calculatePermission } from "./model_acl.js"; import { mkdir, save } from "./model_virtual_layer.js"; import t from "../../locales/index.js"; const workers$ = new rxjs.BehaviorSubject({ tasks: [], size: null }); const ABORT_ERROR = new AjaxError("aborted", null, "ABORTED"); export default async function(render) { if (!document.querySelector(`[is="component_upload_queue"]`)) { const $queue = createElement(`
`); document.body.appendChild($queue); componentUploadQueue(createRender($queue), { workers$ }); } effect(getPermission().pipe( rxjs.filter(() => calculatePermission(currentPath(), "new-file")), rxjs.tap(() => { const $page = createFragment(`
`); componentFilezone(createRender(assert.type($page.children[0], HTMLElement)), { workers$ }); componentUploadFAB(createRender(assert.type($page.children[1], HTMLElement)), { workers$ }); render($page); }), )); } export function init() { return loadCSS(import.meta.url, "./ctrl_upload.css"); } function componentUploadFAB(render, { workers$ }) { const $page = createElement(`
`); effect(rxjs.fromEvent(qs($page, `input[type="file"]`), "change").pipe( rxjs.tap(async(e) => { workers$.next({ loading: true }); workers$.next(await processFiles(e.target.files)); }), )); render($page); } function componentFilezone(render, { workers$ }) { const selector = `[data-bind="filemanager-children"]`; const $target = assert.type(qs(document.body, selector), HTMLElement); $target.ondragenter = (e) => { if (!isNativeFileUpload(e)) return; $target.classList.add("dropzone"); }; $target.ondrop = async(e) => { if (!isNativeFileUpload(e)) return; $target.classList.remove("dropzone"); e.preventDefault(); workers$.next({ loading: true }); if (e.dataTransfer.items instanceof window.DataTransferItemList) { workers$.next(await processItems(e.dataTransfer.items)); } else if (e.dataTransfer.files instanceof window.FileList) { workers$.next(await processFiles(e.dataTransfer.files)); } else { assert.fail("NOT_IMPLEMENTED - unknown entry type in ctrl_upload.js"); } }; $target.ondragleave = (e) => { if (!isNativeFileUpload(e)) return; if (!(e.relatedTarget === null || // eg: drag outside the window !e.relatedTarget.closest(selector) // eg: drag on the breadcrumb, ... )) return; $target.classList.remove("dropzone"); }; $target.ondragover = (e) => e.preventDefault(); } const MAX_WORKERS = 4; function componentUploadQueue(render, { workers$ }) { const $page = createElement(` `); render($page); const $content = qs($page, ".stats_content"); const $file = createElement(`
`); const updateTotal = { reset: () => { qs($page, ".grandTotal").innerText = 0; qs($page, ".completed").innerText = 0; }, addToTotal: (n) => { const $total = qs($page, ".grandTotal"); $total.innerText = parseInt($total.innerText) + n; }, incrementCompleted: () => { const $completed = qs($page, ".completed"); $completed.innerText = parseInt($completed.innerText) + 1; }, }; // feature1: close the queue onClick(qs($page, `img[alt="close"]`)).pipe(rxjs.tap(async() => { const cleanup = await animate($page, { time: 200, keyframes: slideYOut(50) }); $content.innerHTML = ""; $page.classList.add("hidden"); updateTotal.reset(); cleanup(); })).subscribe(); // feature2: setup the task queue in the dom workers$.subscribe(({ tasks, loading = false, size }) => { if (loading) { $page.classList.remove("hidden"); updateDOMGlobalTitle($page, t("Loading")+ "..."); return; } if (tasks.length === 0) return; updateTotal.addToTotal(tasks.length); const $fragment = document.createDocumentFragment(); for (let i = 0; i`); const $iconRetry = createElement(`retry`); const $close = qs($page, `img[alt="close"]`); const updateDOMTaskProgress = ($task, text) => $task.firstElementChild.nextElementSibling.textContent = text; const updateDOMTaskSpeed = ($task, text) => $task.firstElementChild.firstElementChild.nextElementSibling.textContent = formatSpeed(text); const updateDOMGlobalSpeed = (function(workersSpeed) { let last = 0; return (nworker, currentWorkerSpeed) => { workersSpeed[nworker] = currentWorkerSpeed; if (new Date().getTime() - last <= 500) return; last = new Date().getTime(); const speed = workersSpeed.reduce((acc, el) => acc + el, 0); const $speed = assert.type($page.firstElementChild?.nextElementSibling?.firstElementChild, HTMLElement); $speed.textContent = formatSpeed(speed); }; }(new Array(MAX_WORKERS).fill(0))); const updateDOMGlobalTitle = ($page, text) => $page.firstElementChild.nextElementSibling.firstChild.textContent = text; const updateDOMWithStatus = ($task, { status, exec, nworker }) => { const cancel = () => exec.cancel(); const executeMutation = (status) => { switch (status) { case "todo": updateDOMGlobalTitle($page, t("Running") + "..."); break; case "doing": const $stop = assert.type($iconStop.cloneNode(true), HTMLElement); updateDOMTaskProgress($task, formatPercent(0)); $task.classList.remove("error_color"); $task.classList.add("todo_color"); $task.setAttribute("data-status", "running"); $task.firstElementChild.nextElementSibling.nextElementSibling.replaceChildren($stop); $stop.onclick = () => { cancel(); $task.removeAttribute("data-status"); $task.firstElementChild.nextElementSibling.nextElementSibling.classList.add("hidden"); }; $close.addEventListener("click", cancel, { once: true }); break; case "done": updateDOMGlobalTitle($page, t("Done")); updateDOMTaskProgress($task, t("Done")); updateDOMGlobalSpeed(nworker, 0); updateDOMTaskSpeed($task, 0); $task.removeAttribute("data-path"); $task.removeAttribute("data-status"); $task.classList.remove("todo_color"); $task.firstElementChild.nextElementSibling.nextElementSibling.classList.add("hidden"); $close.removeEventListener("click", cancel); break; case "error": const $retry = assert.type($iconRetry.cloneNode(true), HTMLElement); updateDOMGlobalTitle($page, t("Error")); updateDOMTaskProgress($task, t("Error")); updateDOMGlobalSpeed(nworker, 0); updateDOMTaskSpeed($task, 0); $task.removeAttribute("data-path"); $task.removeAttribute("data-status"); $task.classList.remove("todo_color"); $task.classList.add("error_color"); $task.firstElementChild.nextElementSibling.nextElementSibling.firstElementChild.remove(); $task.firstElementChild.nextElementSibling.nextElementSibling.appendChild($retry); $retry.onclick = async() => { executeMutation("todo"); executeMutation("doing"); try { await exec.retry(); executeMutation("done"); } catch (err) { executeMutation("error"); } }; $close.removeEventListener("click", cancel); break; default: assert.fail(`UNEXPECTED_STATUS status="${status}" path="${$task.getAttribute("path")}"`); } }; executeMutation(status); }; let tasks = []; const reservations = new Array(MAX_WORKERS).fill(false); const processWorkerQueue = async(nworker) => { while (tasks.length > 0) { updateDOMGlobalTitle($page, t("Running")+"..."); // step1: retrieve next task const task = nextTask(tasks); if (!task) { await new Promise((resolve) => setTimeout(resolve, 1000)); continue; } // step2: validate the task is ready to run now const $tasks = qsa($page, `[data-path="${task.path}"][data-status="running"]`); if ($tasks.length > 0) { await new Promise((resolve) => setTimeout(resolve, 1000)); tasks.unshift(task); continue; } // step3: process the task through its entire lifecycle const $task = qs($page, `[data-path="${task.path}"]`); const exec = task.exec({ progress: (progress) => updateDOMTaskProgress($task, formatPercent(progress)), speed: (speed) => { updateDOMTaskSpeed($task, speed); updateDOMGlobalSpeed(nworker, speed); }, }); updateDOMWithStatus($task, { exec, status: "doing", nworker }); try { await exec.run(task); updateDOMWithStatus($task, { exec, status: "done", nworker }); } catch (err) { updateDOMWithStatus($task, { exec, status: "error", nworker }); } updateTotal.incrementCompleted(); task.done = true; // step4: execute only at task completion if (tasks.length === 0 && // no remaining tasks reservations.filter((t) => t === true).length === 1 // only for the last remaining job ) updateDOMGlobalTitle($page, t("Done")); } }; const nextTask = (tasks) => { for (let i=0; i fn().catch(() => noFailureAllowed(fn)); workers$.subscribe(async({ tasks: newTasks, loading = false }) => { if (loading) return; tasks = tasks.concat(newTasks); // add new tasks to the pool while (!$page.classList.contains("hidden")) { const nworker = reservations.indexOf(false); if (nworker === -1) break; // the pool of workers is already to its max reservations[nworker] = true; noFailureAllowed(processWorkerQueue.bind(null, nworker)).then(() => reservations[nworker] = false); } reservations.fill(false); }); } class IExecutor { contructor() {} cancel() { throw new Error("NOT_IMPLEMENTED"); } retry() { throw new Error("NOT_IMPLEMENTED"); } run() { throw new Error("NOT_IMPLEMENTED"); } } function workerImplFile({ progress, speed }) { return new class Worker extends IExecutor { constructor() { super(); this.xhr = null; } /** * @override */ cancel() { if (this.xhr) assert.type(this.xhr, XMLHttpRequest).abort(); this.xhr = null; } /** * @override */ async run({ file, path, virtual }) { const _file = await file(); const executeJob = () => this.prepareJob({ file: _file, path, virtual }); this.retry = () => executeJob(); return executeJob(); } async prepareJob({ file, path, virtual }) { const chunkSize = getConfig("upload_chunk_size", 0) *1024*1024; const numberOfChunks = Math.ceil(file.size / chunkSize); const headersNoCache = { "Cache-Control": "no-store", "Pragma": "no-cache", }; // Case1: basic upload if (chunkSize === 0 || numberOfChunks === 0 || numberOfChunks === 1) { try { await executeHttp.call(this, toHref(`/api/files/cat?path=${encodeURIComponent(path)}`), { method: "POST", headers: { ...headersNoCache }, body: file, progress, speed, }); virtual.afterSuccess(); } catch (err) { virtual.afterError(); if (err === ABORT_ERROR) return; throw err; } return; } // Case2: chunked upload => TUS: https://www.ietf.org/archive/id/draft-tus-httpbis-resumable-uploads-protocol-00.html try { let resp = await executeHttp.call(this, toHref(`/api/files/cat?path=${encodeURIComponent(path)}&proto=tus`), { method: "POST", headers: { "Upload-Length": file.size, ...headersNoCache, }, body: null, progress: (n) => progress(0), speed, }); const url = resp.headers.location; if (!url.startsWith(toHref("/api/files/cat?"))) { throw new Error("Internal Error"); } for (let i=0; i { const chunksAlreadyDownloaded = i * chunkSize; const currentChunkDownloaded = p / 100 * ( i !== numberOfChunks - 1 ? chunkSize : (file.size % chunkSize) || chunkSize ); progress(Math.floor(100 * (chunksAlreadyDownloaded + currentChunkDownloaded) / file.size)); }, speed, }); } virtual.afterSuccess(); } catch (err) { virtual.afterError(); if (err === ABORT_ERROR) return; throw err; } } }(); } function workerImplDirectory({ progress }) { return new class Worker extends IExecutor { constructor() { super(); this.xhr = null; } /** * @override */ cancel() { assert.type(this.xhr, XMLHttpRequest).abort(); } /** * @override */ async run({ virtual, path }) { const executeJob = () => this.prepareJob({ virtual, path }); this.retry = () => executeJob(); return executeJob(); } async prepareJob({ virtual, path }) { let percent = 0; const id = setInterval(() => { percent += 10; if (percent >= 100) { clearInterval(id); return; } progress(percent); }, 100); try { await executeHttp.call(this, toHref(`/api/files/mkdir?path=${encodeURIComponent(path)}`), { method: "POST", headers: {}, body: null, progress, speed: () => {}, }); clearInterval(id); progress(100); virtual.afterSuccess(); } catch (err) { clearInterval(id); virtual.afterError(); if (err === ABORT_ERROR) return; throw err; } } }(); } function executeHttp(url, { method, headers, body, progress, speed }) { const xhr = new XMLHttpRequest(); const prevProgress = []; this.xhr = xhr; return new Promise((resolve, reject) => { xhr.open(method, forwardURLParams(url, ["share"])); xhr.setRequestHeader("X-Requested-With", "XmlHttpRequest"); xhr.withCredentials = true; for (const key in headers) { xhr.setRequestHeader(key, headers[key]); } xhr.upload.onprogress = (e) => { if (!e.lengthComputable) return; const percent = Math.floor(100 * e.loaded / e.total); progress(percent); if (prevProgress.length === 0) { prevProgress.push(e); return; } prevProgress.push(e); const calculateTime = (p1, pm1) => (p1.timeStamp - pm1.timeStamp)/1000; const calculateBytes = (p1, pm1) => p1.loaded - pm1.loaded; let avgSpeed = 0; for (let i=1; i 5000) { prevProgress.shift(); } }; xhr.upload.onabort = () => reject(ABORT_ERROR); xhr.onerror = (e) => reject(new AjaxError("failed", e, "FAILED")); xhr.onload = () => { if ([200, 201, 204].indexOf(xhr.status) === -1) { reject(new Error(xhr.statusText)); return; } progress(100); resolve({ status: xhr.status, headers: xhr.getAllResponseHeaders() .split("\r\n") .reduce((acc, el) => { const tmp = el.split(": "); if (typeof tmp[0] === "string" && typeof tmp[1] === "string") { acc[tmp[0]] = tmp[1]; } return acc; }, {}) }); }; xhr.send(body); }); } async function processFiles(filelist) { const tasks = []; let size = 0; const detectFiletype = (file) => { // the 4096 is an heuristic observed and taken from: // https://stackoverflow.com/questions/25016442 // however the proposed answer is just wrong as it doesn't consider folder with // name such as: test.png and as Stackoverflow favor consanguinity with their // point system, I couldn't rectify the proposed answer. The following code is // actually working as expected if (file.size % 4096 !== 0) { return Promise.resolve("file"); } return new Promise((resolve) => { const reader = new window.FileReader(); const tid = setTimeout(() => reader.abort(), 1000); reader.onload = () => resolve("file"); reader.onabort = () => resolve("file"); reader.onerror = () => { resolve("directory"); clearTimeout(tid); }; reader.readAsArrayBuffer(file); }); }; for (const currentFile of filelist) { const type = await detectFiletype(currentFile); let path = currentPath() + currentFile.name; let task = null; switch (type) { case "file": task = { type: "file", file: () => new Promise((resolve) => resolve(currentFile)), path, date: currentFile.lastModified, exec: workerImplFile, virtual: save(path, currentFile.size), done: false, ready: () => true, }; size += currentFile.size; break; case "directory": path += "/"; task = { type: "directory", path, date: currentFile.lastModified, exec: workerImplDirectory, virtual: mkdir(path), done: false, ready: () => true, }; size += 4096; break; default: assert.fail(`NOT_SUPPORTED type="${type}"`); } task.virtual.before(); tasks.push(task); } return { tasks, size }; } async function processItems(itemList) { const bfs = async(queue) => { const tasks = []; let size = 0; const basepath = currentPath(); while (queue.length > 0) { const entry = queue.shift(); const path = basepath + entry.fullPath.substring(1); let task = {}; if (entry === null) continue; else if (entry.isFile) { const entrySize = await new Promise((resolve) => { if (typeof entry.getMetadata === "function") { entry.getMetadata(({ size }) => resolve(size)); } else resolve(null); // eg: firefox }); task = { type: "file", file: () => new Promise((resolve, reject) => entry.file( (file) => resolve(file), (error) => reject(error), )), path, exec: workerImplFile, virtual: save(path, entrySize), done: false, ready: () => false, }; size += entrySize; } else if (entry.isDirectory) { task = { type: "directory", path: path + "/", exec: workerImplDirectory, virtual: mkdir(path), done: false, ready: () => false, }; const reader = entry.createReader(); let q = []; do { q = await new Promise((resolve) => reader.readEntries(resolve)); queue = queue.concat(q); } while (q.length > 0); } else { assert.fail("NOT_IMPLEMENTED - unknown entry type in ctrl_upload.js"); } task.ready = () => { const isInDirectory = (filepath, folder) => folder.indexOf(filepath) === 0; for (let i=0; i= thresh && u < units.length - 1); return bytes.toFixed(1) + units[u]; } function formatSpeed(bytes, si = true) { return formatSize(bytes, si)+ "/s"; }