improve (efficiency): improve memory efficiency for resource intensive tasks

server/common/error.go

@@ -24,6 +24,8 @@ var (
 	ErrFilesystemError        = NewError("Can't use filesystem", 503)
 	ErrMissingDependency      = NewError("Missing dependency", 424)
 	ErrNotAuthorized          = NewError("Not authorized", 401)
+	ErrCongestion             = NewError("Traffic congestion, try again later", 500)
+	ErrTimeout                = NewError("Timeout", 500)
 )
 
 type AppError struct {

server/plugin/plg_image_light/deps/README.md

@@ -3,8 +3,10 @@ plg_image_light rely on a few libraries for:
 - image resizing: libresize.a: a library built on top of libvips
 
 To create the libraries to be used by Filestash:
+```
 ./create_libtranscode.sh
 ./create_libresize.sh
+```
 
 To test the libraries are working fine:
 ```

server/plugin/plg_image_light/lib_resize.go

@@ -10,25 +10,50 @@ import (
 	. "github.com/mickael-kerjean/filestash/server/common"
 	"golang.org/x/sync/semaphore"
 	"io"
+	"time"
 	"unsafe"
 )
 
-var VIPS_LOCK = semaphore.NewWeighted(int64(10))
+const (
+	THUMBNAIL_TIMEOUT        = 5 * time.Second
+	THUMBNAIL_MAX_CONCURRENT = 50
+)
+
+var VIPS_LOCK = semaphore.NewWeighted(THUMBNAIL_MAX_CONCURRENT)
 
 func CreateThumbnail(t *Transform) (io.ReadCloser, error) {
-	VIPS_LOCK.Acquire(context.Background(), 1)
+	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(THUMBNAIL_TIMEOUT))
+	defer cancel()
+	if err := VIPS_LOCK.Acquire(ctx, 1); err != nil {
+		return nil, ErrCongestion
+	}
 	defer VIPS_LOCK.Release(1)
 
-	filename := C.CString(t.Input)
+	imageChannel := make(chan io.ReadCloser, 1)
-	defer C.free(unsafe.Pointer(filename))
+	go func() {
-	var buffer unsafe.Pointer
+		filename := C.CString(t.Input)
-	len := C.size_t(0)
+		len := C.size_t(0)
-	if C.image_resize(filename, &buffer, &len, C.int(t.Size), boolToCInt(t.Crop), C.int(t.Quality), boolToCInt(t.Exif)) != 0 {
+		var buffer unsafe.Pointer
-		return nil, NewError("", 500)
+		if C.image_resize(filename, &buffer, &len, C.int(t.Size), boolToCInt(t.Crop), C.int(t.Quality), boolToCInt(t.Exif)) != 0 {
+			C.free(unsafe.Pointer(filename))
+			imageChannel <- nil
+			return
+		}
+		C.free(unsafe.Pointer(filename))
+		buf := C.GoBytes(buffer, C.int(len))
+		C.g_free(C.gpointer(buffer))
+		imageChannel <- NewReadCloserFromBytes(buf)
+	}()
+
+	select {
+	case img := <- imageChannel:
+		if img == nil {
+			return nil, ErrNotValid
+		}
+		return img, nil
+	case <- ctx.Done():
+		return nil, ErrTimeout
 	}
-	buf := C.GoBytes(buffer, C.int(len))
-	C.g_free(C.gpointer(buffer))
-	return NewReadCloserFromBytes(buf), nil
 }
 
 func boolToCInt(val bool) C.int {

server/plugin/plg_image_light/lib_transcode.go

@@ -8,10 +8,16 @@ import (
 	"context"
 	"golang.org/x/sync/semaphore"
 	. "github.com/mickael-kerjean/filestash/server/common"
+	"time"
 	"unsafe"
 )
 
-var LIBRAW_LOCK = semaphore.NewWeighted(int64(5))
+const (
+	TRANSCODE_TIMEOUT        = 10 * time.Second
+	TRANSCODE_MAX_CONCURRENT = 5
+)
+
+var LIBRAW_LOCK = semaphore.NewWeighted(int64(TRANSCODE_MAX_CONCURRENT))
 
 func IsRaw(mType string) bool {
 	switch mType {
@@ -45,14 +51,28 @@ func IsRaw(mType string) bool {
 }
 
 func ExtractPreview(t *Transform) error {
-	LIBRAW_LOCK.Acquire(context.Background(), 1)
+	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(TRANSCODE_TIMEOUT))
+	defer cancel()
+	
+	if err := LIBRAW_LOCK.Acquire(ctx, 1); err != nil {
+		return ErrCongestion
+	}
 	defer LIBRAW_LOCK.Release(1)
 
-	filename := C.CString(t.Input)
+	transcodeChannel := make(chan error, 1)
-	defer C.free(unsafe.Pointer(filename))
+	go func() {
+		filename := C.CString(t.Input)
+		defer C.free(unsafe.Pointer(filename))
+		if err := C.image_transcode_compute(filename, C.int(t.Size)); err != 0 {
+			transcodeChannel <- ErrNotValid
+		}
+		transcodeChannel <- nil
+	}()
 
-	if err := C.image_transcode_compute(filename, C.int(t.Size)); err != 0 {
+	select {
-		return ErrNotValid
+	case err := <- transcodeChannel:
+		return err
+	case <- ctx.Done():
+		return ErrTimeout		
 	}
-	return nil
 }