opensource/filestash
1
0
Fork 0
mirror of https://github.com/mickael-kerjean/filestash.git synced 2025-11-02 20:23:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

feature (tags): revamp data model and storage

Browse Source
This commit is contained in:
Mickael Kerjean
2022-10-11 01:45:34 +11:00
parent 9fa8428a28
commit 88bd7d67dc
8 changed files with 141 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
client/helpers/cache.js
View File

@ -1,19 +1,22 @@
"use strict";
const DB_VERSION = 3;
const DB_VERSION = 4;
const FILE_PATH = "file_path";
const FILE_CONTENT = "file_content";
const FILE_TAG = "file_tag";
function DataFromIndexedDB() {
this.db = null;
this.FILE_PATH = FILE_PATH;
this.FILE_CONTENT = FILE_CONTENT;
this.FILE_TAG = FILE_TAG;
return this._init();
}
function DataFromMemory() {
this.data = {};
this.FILE_PATH = FILE_PATH;
this.FILE_CONTENT = FILE_CONTENT;
this.FILE_TAG = FILE_TAG;
return this._init();
}
@ -32,13 +35,21 @@ DataFromIndexedDB.prototype._init = function() {
// we've change the primary key to be a (path,share)
db.deleteObjectStore(FILE_PATH);
db.deleteObjectStore(FILE_CONTENT);
} else if (event.oldVersion == 3) {
// we've added a FILE_TAG to store tag related data and update
// keyPath to have "backend"
db.deleteObjectStore(FILE_PATH);
db.deleteObjectStore(FILE_CONTENT);
}
store = db.createObjectStore(FILE_PATH, { keyPath: ["share", "path"] });
store.createIndex("idx_path", ["share", "path"], { unique: true });
store = db.createObjectStore(FILE_PATH, { keyPath: ["backend", "share", "path"] });
store.createIndex("idx_path", ["backend", "share", "path"], { unique: true });
store = db.createObjectStore(FILE_CONTENT, { keyPath: ["share", "path"] });
store.createIndex("idx_path", ["share", "path"], { unique: true });
store = db.createObjectStore(FILE_CONTENT, { keyPath: ["backend", "share", "path"] });
store.createIndex("idx_path", ["backend", "share", "path"], { unique: true });
store = db.createObjectStore(FILE_TAG, { keyPath: ["backend", "share"] });
store.createIndex("idx_path", ["backend", "share"], { unique: true });
};
this.db = new Promise((done, err) => {
@ -56,7 +67,7 @@ DataFromMemory.prototype._init = function() {
* Fetch a record using its path, can be either a file path or content
*/
DataFromIndexedDB.prototype.get = function(type, key) {
if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject();
if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject();
return this.db.then((db) => {
const tx = db.transaction(type, "readonly");
@ -71,7 +82,7 @@ DataFromIndexedDB.prototype.get = function(type, key) {
});
};
DataFromMemory.prototype.get = function(type, key) {
if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject();
if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject();
const data = this.data[type] || null;
if (data === null) {
@ -87,7 +98,7 @@ DataFromMemory.prototype.get = function(type, key) {
};
DataFromIndexedDB.prototype.update = function(type, key, fn, exact = true) {
if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject();
if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject();
return this.db.then((db) => {
const tx = db.transaction(type, "readwrite");
@ -113,7 +124,7 @@ DataFromIndexedDB.prototype.update = function(type, key, fn, exact = true) {
};
DataFromMemory.prototype.update = function(type, key, fn, exact = true) {
if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject();
if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject();
const data = this.data[type];
if (data === undefined) {
@ -134,7 +145,7 @@ DataFromMemory.prototype.update = function(type, key, fn, exact = true) {
};
DataFromIndexedDB.prototype.upsert = function(type, key, fn) {
if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject();
if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject();
return this.db.then((db) => {
const tx = db.transaction(type, "readwrite");
@ -154,7 +165,7 @@ DataFromIndexedDB.prototype.upsert = function(type, key, fn) {
});
};
DataFromMemory.prototype.upsert = function(type, key, fn) {
if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject();
if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject();
const db = this.data[type] || null;
if (db === null) {
@ -167,7 +178,7 @@ DataFromMemory.prototype.upsert = function(type, key, fn) {
};
DataFromIndexedDB.prototype.add = function(type, key, data) {
if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject();
if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject();
return this.db.then((db) => {
return new Promise((done, error) => {
@ -180,7 +191,7 @@ DataFromIndexedDB.prototype.add = function(type, key, data) {
}).catch(() => Promise.resolve());
};
DataFromMemory.prototype.add = function(type, key, data) {
if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject();
if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject();
if (this.data[type] === undefined) {
this.data[type] = {};
@ -190,7 +201,7 @@ DataFromMemory.prototype.add = function(type, key, data) {
};
DataFromIndexedDB.prototype.remove = function(type, key, exact = true) {
if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject();
if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject();
return this.db.then((db) => {
const tx = db.transaction(type, "readwrite");
@ -223,7 +234,7 @@ DataFromIndexedDB.prototype.remove = function(type, key, exact = true) {
}).catch(() => Promise.resolve());
};
DataFromMemory.prototype.remove = function(type, key, exact = true) {
if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject();
if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject();
const data = this.data[type] || null;
if (data === null) {
@ -242,7 +253,7 @@ DataFromMemory.prototype.remove = function(type, key, exact = true) {
};
DataFromIndexedDB.prototype.fetchAll = function(fn, type = FILE_PATH, key) {
if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject();
if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject();
return this.db.then((db) => {
const tx = db.transaction([type], "readonly");
@ -272,7 +283,7 @@ DataFromIndexedDB.prototype.fetchAll = function(fn, type = FILE_PATH, key) {
}).catch(() => Promise.resolve());
};
DataFromMemory.prototype.fetchAll = function(fn, type = FILE_PATH, key) {
if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject();
if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject();
const data = this.data[type] || null;
if (data === null) {
@ -295,6 +306,8 @@ DataFromIndexedDB.prototype.destroy = function() {
this.db.then((db) => {
purgeAll(db, FILE_PATH);
purgeAll(db, FILE_CONTENT);
// We keep FILE_TAG as this was user generated and potentially frustrating
// for users if they were to lose this
});
done();
@ -311,11 +324,15 @@ DataFromMemory.prototype.destroy = function() {
return Promise.resolve();
};
export let cache = new DataFromMemory();
if ("indexedDB" in window && window.indexedDB !== null) {
const request = indexedDB.open("_indexedDB", 1);
request.onsuccess = (e) => {
export let cache = null;
export function setup_cache() {
cache = new DataFromMemory();
if ("indexedDB" in window && window.indexedDB !== null) {
cache = new DataFromIndexedDB();
indexedDB.deleteDatabase("_indexedDB");
};
return new Promise((done) => {
cache.db.then(() => done());
});
}
return Promise.resolve();
}

6
client/helpers/index.js
View File

@ -5,10 +5,10 @@ export {
export { opener } from "./mimetype";
export { debounce, throttle } from "./backpressure";
export { event } from "./events";
export { cache } from "./cache";
export { cache, setup_cache } from "./cache";
export {
pathBuilder, basename, dirname, absoluteToRelative, filetype, currentShare,
findParams, appendShareToUrl,
pathBuilder, basename, dirname, absoluteToRelative, filetype,
currentShare, currentBackend, findParams, appendShareToUrl,
} from "./path";
export { memory } from "./memory";
export { prepare } from "./navigate";

5
client/helpers/path.js
View File

@ -40,6 +40,11 @@ export function currentShare() {
return findParams("share");
}
export function currentBackend() {
return "";
}
export function findParams(p) {
return new window.URL(location.href).searchParams.get(p) || "";
}

4
client/index.js
View File

@ -3,7 +3,7 @@ import ReactDOM from "react-dom";
import Router from "./router";
import { Config, Log } from "./model/";
import { http_get } from "./helpers/ajax";
import { http_get, setup_cache } from "./helpers/";
import load from "little-loader";
import "./assets/css/reset.scss";
@ -40,7 +40,7 @@ window.addEventListener("DOMContentLoaded", () => {
return Promise.resolve();
}
Promise.all([Config.refresh(), setup_xdg_open(), translation()]).then(() => {
Promise.all([Config.refresh(), setup_xdg_open(), translation(), setup_cache()]).then(() => {
const timeSinceBoot = new Date() - window.initTime;
if (window.CONFIG.name) document.title = window.CONFIG.name;
if (timeSinceBoot >= 1500) {

37
client/model/files.js
View File

@ -2,7 +2,7 @@
import {
http_get, http_post, http_options, prepare, basename, dirname, pathBuilder,
currentShare, appendShareToUrl,
currentShare, currentBackend, appendShareToUrl,
} from "../helpers/";
import { Observable } from "rxjs/Observable";
@ -46,8 +46,9 @@ class FileSystem {
return http_get(url).then((response) => {
response = fileMiddleware(response, path, show_hidden);
return cache.upsert(cache.FILE_PATH, [currentShare(), path], (_files) => {
return cache.upsert(cache.FILE_PATH, [currentBackend(), currentShare(), path], (_files) => {
const store = Object.assign({
backend: currentBackend(),
share: currentShare(),
status: "ok",
path: path,
@ -97,7 +98,7 @@ class FileSystem {
}
_ls_from_cache(path, _record_access = false) {
return cache.get(cache.FILE_PATH, [currentShare(), path]).then((response) => {
return cache.get(cache.FILE_PATH, [currentBackend(), currentShare(), path]).then((response) => {
if (!response || !response.results) return null;
if (this.current_path === path) {
this.obs && this.obs.next({
@ -110,7 +111,7 @@ class FileSystem {
}).then((e) => {
requestAnimationFrame(() => {
if (_record_access === true) {
cache.upsert(cache.FILE_PATH, [currentShare(), path], (response) => {
cache.upsert(cache.FILE_PATH, [currentBackend(), currentShare(), path], (response) => {
if (!response || !response.results) return null;
if (this.current_path === path) {
this.obs && this.obs.next({
@ -136,9 +137,9 @@ class FileSystem {
this._ls_from_cache(dirname(path)) : Promise.resolve(res))
.then(() => http_post(url))
.then((res) => {
return cache.remove(cache.FILE_CONTENT, [currentShare(), path])
.then(cache.remove(cache.FILE_CONTENT, [currentShare(), path], false))
.then(cache.remove(cache.FILE_PATH, [currentShare(), dirname(path)], false))
return cache.remove(cache.FILE_CONTENT, [currentBackend(), currentShare(), path])
.then(cache.remove(cache.FILE_CONTENT, [currentBackend(), currentShare(), path], false))
.then(cache.remove(cache.FILE_PATH, [currentBackend(), currentShare(), dirname(path)], false))
.then(this._remove(path, "loading"))
.then((res) => this.current_path === dirname(path) ?
this._ls_from_cache(dirname(path)) : Promise.resolve(res));
@ -158,8 +159,9 @@ class FileSystem {
if (this.is_binary(res) === true) {
return Promise.reject({ code: "BINARY_FILE" });
}
return cache.upsert(cache.FILE_CONTENT, [currentShare(), path], (response) => {
return cache.upsert(cache.FILE_CONTENT, [currentBackend(), currentShare(), path], (response) => {
const file = response ? response : {
backend: currentBackend(),
share: currentShare(),
path: path,
last_update: null,
@ -245,8 +247,9 @@ class FileSystem {
return this._replace(destination_path, null, "loading")
.then(() => origin_path !== destination_path ?
this._remove(origin_path, "loading") : Promise.resolve())
.then(() => cache.add(cache.FILE_PATH, [currentShare(), destination_path], {
.then(() => cache.add(cache.FILE_PATH, [currentBackend(), currentShare(), destination_path], {
path: destination_path,
backend: currentBackend(),
share: currentShare(),
results: [],
access_count: 0,
@ -353,11 +356,11 @@ class FileSystem {
.then(() => this._replace(destination_path, null, "loading"))
.then(() => this._refresh(origin_path, destination_path))
.then(() => {
cache.update(cache.FILE_PATH, [currentShare(), origin_path], (data) => {
cache.update(cache.FILE_PATH, [currentBackend(), currentShare(), origin_path], (data) => {
data.path = data.path.replace(origin_path, destination_path);
return data;
}, false);
cache.update(cache.FILE_CONTENT, [currentShare(), origin_path], (data) => {
cache.update(cache.FILE_CONTENT, [currentBackend(), currentShare(), origin_path], (data) => {
data.path = data.path.replace(origin_path, destination_path);
return data;
}, false);
@ -389,7 +392,7 @@ class FileSystem {
if (value.access_count >= 1 && value.path !== "/") {
data.push(value);
}
}, cache.FILE_PATH, [currentShare(), "/"]).then(() => {
}, cache.FILE_PATH, [currentBackend(), currentShare(), "/"]).then(() => {
return Promise.resolve(
data
.sort((a, b) => a.access_count > b.access_count? -1 : 1)
@ -410,9 +413,10 @@ class FileSystem {
});
function update_cache(result) {
return cache.upsert(cache.FILE_CONTENT, [currentShare(), path], (response) => {
return cache.upsert(cache.FILE_CONTENT, [currentBackend(), currentShare(), path], (response) => {
if (!response) {
response = {
backend: currentBackend(),
share: currentShare(),
path: path,
last_access: null,
@ -437,7 +441,7 @@ class FileSystem {
}
_replace(path, icon, icon_previous) {
return cache.update(cache.FILE_PATH, [currentShare(), dirname(path)], function(res) {
return cache.update(cache.FILE_PATH, [currentBackend(), currentShare(), dirname(path)], function(res) {
res.results = res.results.map((file) => {
if (file.name === basename(path) && file.icon == icon_previous) {
if (!icon) {
@ -453,10 +457,11 @@ class FileSystem {
});
}
_add(path, icon) {
return cache.upsert(cache.FILE_PATH, [currentShare(), dirname(path)], (res) => {
return cache.upsert(cache.FILE_PATH, [currentBackend(), currentShare(), dirname(path)], (res) => {
if (!res || !res.results) {
res = {
path: path,
backend: currentBackend(),
share: currentShare(),
results: [],
access_count: 0,
@ -475,7 +480,7 @@ class FileSystem {
});
}
_remove(path, previous_icon) {
return cache.update(cache.FILE_PATH, [currentShare(), dirname(path)], function(res) {
return cache.update(cache.FILE_PATH, [currentBackend(), currentShare(), dirname(path)], function(res) {
if (!res) return null;
res.results = res.results.filter((file) => {
return file.name === basename(path) && file.icon == previous_icon ? false : true;

134
client/model/tags.js
View File

@ -1,62 +1,52 @@
let DB = {
tags: {
"Bookmark": ["/home/user/Documents/", "/home/user/Documents/projects/"],
"Customer": ["/home/user/Documents/projects/customers/"],
"wiki": ["/home/user/Documents/test.txt"],
"mit": ["/home/user/Documents/projects/customers/mit/"],
"dhl": ["/home/user/Documents/projects/customers/dhl/"],
"powerstone": ["/home/user/Documents/projects/customers/powerstone/"],
"accounting": [
"/home/user/Documents/projects/customers/mit/accounting/",
"/home/user/Documents/projects/customers/dhl/accounting/",
"/home/user/Documents/projects/customers/powerstone/accounting/",
]
},
weight: { // for sorting
"Bookmark": 2,
},
share: null,
backend: "__hash__",
};
import { cache, currentShare, currentBackend } from "../helpers/";
class TagManager {
all(tagPath = "/", maxSize = -1) {
return Promise.resolve([]); // TODO: Remove this when ready
// return Promise.resolve([]); // TODO: Remove this when ready
if (tagPath == "/") {
const scoreFn = (acc, el) => (acc + el.replace(/[^\/]/g, "").length);
const tags = Object.keys(DB.tags).sort((a, b) => {
if (DB.tags[a].length === DB.tags[b].length) {
return DB.tags[a].reduce(scoreFn, 0) - DB.tags[b].reduce(scoreFn, 0);
}
return DB.tags[a].length < DB.tags[b].length ? 1 : -1;
});
if(tags.length === 0) {
return Promise.resolve(["Bookmark"]);
} else if(tags.length >= 5) {
return Promise.resolve(["All"].concat(tags.slice(0, 5)));
return cache.get(cache.FILE_TAG, [currentBackend(), currentShare()]).then((DB) => {
if (DB === null) {
return [];
}
return Promise.resolve(tags);
}
return Promise.resolve([
// "Bookmark", "wiki", "B", "C", "D", "E", "F"
]);
if (tagPath == "/") {
const scoreFn = (acc, el) => (acc + el.replace(/[^\/]/g, "").length);
const tags = Object.keys(DB.tags).sort((a, b) => {
if (DB.tags[a].length === DB.tags[b].length) {
return DB.tags[a].reduce(scoreFn, 0) - DB.tags[b].reduce(scoreFn, 0);
}
return DB.tags[a].length < DB.tags[b].length ? 1 : -1;
});
if(tags.length === 0) {
return ["Bookmark"];
} else if(tags.length >= 5) {
return ["All"].concat(tags.slice(0, 5));
}
return tags;
}
return [
"Bookmark", "wiki", "B", "C", "D", "E", "F"
];
});
}
files(tagPath) {
const tags = this._tagPathStringToArray(tagPath, false);
if (tags.length === 0) return Promise.resolve([]);
else if(tags.length > 1) return Promise.resolve([]); // TODO
else if (tags.length > 1) return Promise.resolve([]); // TODO
switch(tags[0]) {
case "All":
return this.all()
.then((tags) => (tags.reduce((acc, el) => {
return DB.tags[el] ? acc.concat(DB.tags[el]) : acc;
}, [])));
default:
return Promise.resolve(DB.tags[tags[0]] || []);
}
return cache.get(cache.FILE_TAG, [currentBackend(), currentShare()]).then((DB) => {
if(!DB) return [];
switch(tags[0]) {
case "All":
return this.all()
.then((tags) => (tags.reduce((acc, el) => {
return DB.tags[el] ? acc.concat(DB.tags[el]) : acc;
}, [])));
default:
return Promise.resolve(DB.tags[tags[0]] || []);
}
});
}
_tagPathStringToArray(tagPathString, removeFirst = true) {
@ -66,31 +56,45 @@ class TagManager {
}
addTagToFile(tag, path) {
if(Object.keys(DB.tags).indexOf(tag) === -1) {
DB.tags[tag] = [];
}
if(!DB.tags[tag].indexOf(path) === -1) {
DB.tags[tag].push(path);
}
return cache.upsert(cache.FILE_TAG, [currentBackend(), currentShare()], (DB) => {
if(Object.keys(DB.tags).indexOf(tag) === -1) {
DB.tags[tag] = [];
}
if(!DB.tags[tag].indexOf(path) === -1) {
DB.tags[tag].push(path);
}
return DB;
});
}
removeTagFromFile(tag, path) {
if(!DB.tags[tag]) return;
const idx = DB.tags[tag].indexOf(path);
DB.tags[tag].splice(idx, 1);
return cache.upsert(cache.FILE_TAG, [currentBackend(), currentShare()], (DB) => {
if(!DB.tags[tag]) return;
const idx = DB.tags[tag].indexOf(path);
DB.tags[tag].splice(idx, 1);
if (DB.tags[tag].length === 0) {
delete DB.tags[tag];
delete DB.weight[tag];
}
return DB;
});
}
import(_DB) {
DB = _DB;
return new Promise((done) => {
setTimeout(() => {
done();
}, 5000);
})
import(DB) {
return cache.upsert(cache.FILE_TAG, [currentBackend(), currentShare()], () => {
return DB;
});
}
export() {
return Promise.resolve(DB);
const key = [currentBackend(), currentShare()];
return cache.get(cache.FILE_TAG, key)
.then((a) => {
if (a === null) {
return {tags: {}, weight: {}, share: key[1], backend: key[0]}
}
return a;
});
}
}

1
client/pages/tagspage.js
View File

@ -81,7 +81,6 @@ export function TagsPageComponent({ match }) {
setRefresh(refresh + 1);
}).catch((err) => {
setLoading(false);
notify.send(err, "error")
});
};
reader.readAsText($input.files[0]);

2
client/pages/tagspage.scss
View File

@ -14,7 +14,7 @@
overflow-x: hidden!important;
-webkit-overflow-scrolling: touch;
.component_submenu {
margin: 10px;
margin: 25px 0 0 0;
h1 {
font-weight: 100;
text-transform: uppercase;