Implement directory download as ZIP

Thanks to the fdzipstream library [1] and zlib [2], it is possible to
generate ZIP files on-the-fly, therefore requiring no extra disk space
usage and only a small amount of memory.

Unfortunately, as of the time of this writing fdzipstream is not
packaged by any distributions yet [3], so it had to be imported as a git
submodule as a workaround.

While libarchive [4] could be an interesting alternative, writing ZIP
files is only supported by very recent versions (>= 3.8.0), which are
still not packaged by many distributions [5], either.

Moreover, libarchive is a package with several dependencies other than
zlib and is significantly larger compared to fdzipstreams, so
fdzipstreams was ultimately considered a better fit for this purpose.

[1]: https://github.com/CTrabant/fdzipstream.git
[2]: http://zlib.net/
[3]: https://repology.org/projects/?search=fdzipstream
[4]: https://www.libarchive.org/
[5]: https://repology.org/project/libarchive/versions

12 files changed, 487 insertions, 11 deletions

diff --git a/.gitignore b/.gitignore
index c99e607..dbef56b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 build/
 slcl
 thumbnail/thumbnail
+*.a
 *.o
 *.d
 /Makefile
diff --git a/CMakeLists.txt b/CMakeLists.txt
index d34a876..4d234c3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -10,6 +10,7 @@ add_executable(${PROJECT_NAME}
     main.c
     page.c
     style.c
+    zip.c
 )
 target_compile_options(${PROJECT_NAME} PRIVATE -Wall)
 target_compile_definitions(${PROJECT_NAME} PRIVATE _FILE_OFFSET_BITS=64)
@@ -26,7 +27,15 @@ endif()
 
 find_package(cJSON 1.0 REQUIRED)
 find_package(OpenSSL 1.0 REQUIRED)
-target_link_libraries(${PROJECT_NAME} PRIVATE web dynstr cjson OpenSSL::SSL)
+find_package(fdzipstream)
+
+if(NOT fdzipstream_FOUND)
+    message(STATUS "Using in-tree fdzipstream")
+    add_subdirectory(fdzipstream)
+endif()
+
+target_link_libraries(${PROJECT_NAME} PRIVATE web dynstr cjson OpenSSL::SSL
+    fdzipstream)
 install(TARGETS ${PROJECT_NAME})
 install(FILES usergen
     TYPE BIN
diff --git a/README.md b/README.md
index e34d401..80e850f 100644
--- a/README.md
+++ b/README.md
@@ -20,6 +20,7 @@ portability, minimalism, simplicity and efficiency.
 
 - Private access directory with file uploading, with configurable quota.
 - Read-only public file sharing.
+- Download directories as `.zip` files.
 - Uses [`libweb`](https://gitea.privatedns.org/xavi/libweb), a tiny web framework.
 - A simple JSON file as the credentials database.
 - No JavaScript.
@@ -50,10 +51,13 @@ to `slcl`. If required, encryption should be done before uploading e.g.: using
 - A POSIX environment.
 - OpenSSL >= 2.0.
 - cJSON >= 1.7.15.
+- ZLIB.
 - [`dynstr`](https://gitea.privatedns.org/xavi/dynstr)
 (provided as a `git` submodule by `libweb`).
 - [`libweb`](https://gitea.privatedns.org/xavi/libweb)
 (provided as a `git` submodule).
+- [`fdzipstream`](https://github.com/CTrabant/fdzipstream.git)
+(provided as a `git` submodule).
 - `jq` (for [`usergen`](usergen) only).
 - CMake (optional).
 
@@ -62,7 +66,7 @@ to `slcl`. If required, encryption should be done before uploading e.g.: using
 #### Mandatory packages
 
 ```sh
-sudo apt install build-essential libcjson-dev libssl-dev m4 jq
+sudo apt install build-essential libcjson-dev libssl-dev m4 jq zlib1g
 ```
 
 #### Optional packages
diff --git a/cmake/Findfdzipstream.cmake b/cmake/Findfdzipstream.cmake
new file mode 100644
index 0000000..b26a515
--- /dev/null
+++ b/cmake/Findfdzipstream.cmake
@@ -0,0 +1,17 @@
+mark_as_advanced(FDZIPSTREAM_LIBRARY FDZIPSTREAM_INCLUDE_DIR)
+find_library(FDZIPSTREAM_LIBRARY NAMES libfdzipstream fdzipstream)
+
+find_path(FDZIPSTREAM_INCLUDE_DIR NAMES fdzipstream.h)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(fdzipstream
+    DEFAULT_MSG FDZIPSTREAM_LIBRARY FDZIPSTREAM_INCLUDE_DIR)
+
+if(fdzipstream_FOUND)
+    if(NOT TARGET fdzipstream)
+        add_library(fdzipstream UNKNOWN IMPORTED)
+        set_target_properties(fdzipstream PROPERTIES
+            INTERFACE_INCLUDE_DIRECTORIES "${FDZIPSTREAM_INCLUDE_DIR}"
+            IMPORTED_LOCATION "${FDZIPSTREAM_LIBRARY}")
+    endif()
+endif()
diff --git a/configure b/configure
index 0c5a7b3..3328319 100755
--- a/configure
+++ b/configure
@@ -71,6 +71,18 @@ else
 	LDFLAGS="$LDFLAGS -Llibweb -lweb"
 fi
 
+if pkg-config fdzipstream
+then
+	in_tree_fdzipstream=0
+	CFLAGS="$CFLAGS $(pkg-config --cflags fdzipstream)"
+	LDFLAGS="$LDFLAGS $(pkg-config --libs fdzipstream)"
+else
+	echo "Info: fdzipstream not found. Using in-tree copy" >&2
+	in_tree_fdzipstream=1
+	CFLAGS="$CFLAGS -Ifdzipstream/fdzipstream"
+	LDFLAGS="$LDFLAGS -Lfdzipstream -lfdzipstream -lz"
+fi
+
 cleanup()
 {
 	rm -f $F
@@ -101,7 +113,8 @@ OBJECTS = \
 	jwt.o \
 	main.o \
 	page.o \
-	style.o
+	style.o \
+	zip.o
 
 all: $(PROJECT)
 
@@ -138,6 +151,16 @@ $(LIBWEB): FORCE
 EOF
 fi
 
+if [ $in_tree_fdzipstream -ne 0 ]
+then
+cat <<"EOF" >> $F
+FDZIPSTREAM = fdzipstream/libfdzipstream.a
+$(PROJECT): $(FDZIPSTREAM)
+$(FDZIPSTREAM): FORCE
+	+cd fdzipstream && $(MAKE) CC=$(CC)
+EOF
+fi
+
 cat <<"EOF" >> $F
 clean:
 	rm -f $(OBJECTS) $(DEPS)
@@ -157,6 +180,13 @@ cat <<"EOF" >> $F
 EOF
 fi
 
+if [ $in_tree_fdzipstream -ne 0 ]
+then
+cat <<"EOF" >> $F
+	+cd fdzipstream && $(MAKE) clean
+EOF
+fi
+
 cat <<"EOF" >> $F
 distclean: clean
 	rm -f slcl
diff --git a/fdzipstream/CMakeLists.txt b/fdzipstream/CMakeLists.txt
new file mode 100644
index 0000000..101a117
--- /dev/null
+++ b/fdzipstream/CMakeLists.txt
@@ -0,0 +1,6 @@
+cmake_minimum_required(VERSION 3.13)
+project(fdzipstream LANGUAGES C)
+find_package(ZLIB REQUIRED)
+add_library(${PROJECT_NAME} fdzipstream/fdzipstream.c)
+target_include_directories(${PROJECT_NAME} PUBLIC fdzipstream)
+target_link_libraries(${PROJECT_NAME} PUBLIC ${ZLIB_LIBRARIES})
diff --git a/fdzipstream/Makefile b/fdzipstream/Makefile
new file mode 100644
index 0000000..7e81636
--- /dev/null
+++ b/fdzipstream/Makefile
@@ -0,0 +1,21 @@
+.POSIX:
+
+PROJECT = libfdzipstream.a
+CFLAGS = -O1 -g -D_POSIX_C_SOURCE=200809L
+
+OBJECTS = \
+	fdzipstream.o
+
+all: $(PROJECT)
+
+clean:
+	rm -f $(OBJECTS)
+
+distclean: clean
+	rm -f $(PROJECT)
+
+$(PROJECT): $(OBJECTS)
+	$(AR) $(ARFLAGS) $@ $(OBJECTS)
+
+%.o: fdzipstream/%.c
+	$(CC) $(CFLAGS) -c $< -o $@
diff --git a/libweb b/libweb
-Subproject db9cb051c4ee05c07ab32dfed5bae8b7dc0916b
+Subproject b80741b3e30444b30514b3a4432f40294034c4f
diff --git a/main.c b/main.c
index 104a381..7a367b0 100644
--- a/main.c
+++ b/main.c
@@ -822,7 +822,7 @@ static int search(const struct http_payload *const p,
     *r = (const struct http_response)
     {
         .step.payload = search_step,
-        .step_args = s
+        .args = s
     };
 
     s->search.root = s->root.str;
@@ -1014,10 +1014,10 @@ static int check_quota(const unsigned long long len,
 
 static int check_length_step(const unsigned long long len,
     const struct http_cookie *const c, struct http_response *const r,
-    void *const user, void *const step_args)
+    void *const user, void *const args)
 {
     int ret = 0;
-    struct quota *const q = step_args;
+    struct quota *const q = args;
 
     switch (cftw_step(q->cftw))
     {
@@ -1079,7 +1079,7 @@ static int check_length(const unsigned long long len,
         *r = (const struct http_response)
         {
             .step.length = check_length_step,
-            .step_args = q
+            .args = q
         };
     }
 
@@ -1143,10 +1143,10 @@ end:
 }
 
 static int getnode_step(const struct http_payload *const p,
-    struct http_response *const r, void *const user, void *const step_args)
+    struct http_response *const r, void *const user, void *const args)
 {
     int ret = 0;
-    struct quota *const q = step_args;
+    struct quota *const q = args;
 
     switch (cftw_step(q->cftw))
     {
@@ -1214,7 +1214,7 @@ static int getnode(const struct http_payload *const p,
         *r = (const struct http_response)
         {
             .step.payload = getnode_step,
-            .step_args = q
+            .args = q
         };
     }
     else
diff --git a/page.c b/page.c
index af24e70..9d1ea8e 100644
--- a/page.c
+++ b/page.c
@@ -1,6 +1,7 @@
 #define _POSIX_C_SOURCE 200809L
 
 #include "page.h"
+#include "zip.h"
 #include <libweb/html.h>
 #include <libweb/http.h>
 #include <dynstr.h>
@@ -1812,6 +1813,21 @@ static bool preview(const struct page_resource *const pr)
     return false;
 }
 
+static bool download(const struct page_resource *const pr)
+{
+    for (size_t i = 0; i < pr->n_args; i++)
+    {
+        const struct http_arg *const a = &pr->args[i];
+
+        if (!strcmp(a->key, "download")
+            && (!strcmp(a->value, "1")
+                || !strcasecmp(a->value, "true")))
+            return true;
+    }
+
+    return false;
+}
+
 int page_resource(const struct page_resource *const pr)
 {
     int ret = -1;
@@ -1840,7 +1856,15 @@ int page_resource(const struct page_resource *const pr)
 
     if (S_ISDIR(m))
     {
-        if ((ret = list_dir(pr)))
+        if (download(pr))
+        {
+            if ((ret = zip(pr->res, pr->r)))
+            {
+                fprintf(stderr, "%s: zip_dir failed\n", __func__);
+                goto end;
+            }
+        }
+        else if ((ret = list_dir(pr)))
         {
             if (ret < 0)
             {
diff --git a/zip.c b/zip.c
new file mode 100644
index 0000000..704bf36
--- /dev/null
+++ b/zip.c
@@ -0,0 +1,356 @@
+#define _POSIX_C_SOURCE 200809L
+
+#include "zip.h"
+#include "cftw.h"
+#include <dynstr.h>
+#include <libweb/http.h>
+#include <fdzipstream.h>
+#include <fcntl.h>
+#include <libgen.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct zip
+{
+    bool finished;
+    char *basedir;
+    struct cftw *cftw;
+    ZIPstream *stream;
+    ZIPentry *entry;
+    int fds[2], fd;
+    off_t filesz, read, lread;
+    int (*next)(void *, size_t, bool *, void *, struct zip *);
+};
+
+static int recurse(void *, size_t, bool *, void *, struct zip *);
+static int read_file(void *, size_t, bool *, void *, struct zip *);
+
+static void free_zip(void *const p)
+{
+    struct zip *const z = p;
+
+    if (!z)
+        return;
+    else if (z->entry)
+        zs_entryend(z->stream, z->entry, NULL);
+
+    zs_free(z->stream);
+
+    if ((z->fds[0] >= 0 && close(z->fds[0]))
+        || (z->fds[1] >= 0 && close(z->fds[1]))
+        || (z->fd >= 0 && close(z->fd)))
+        fprintf(stderr, "%s: close(2): %s\n", __func__, strerror(errno));
+
+    cftw_free(z->cftw);
+    free(z->basedir);
+    free(z);
+}
+
+static int dump_final(void *const buf, const size_t n, bool *const done,
+    void *const user, struct zip *const z)
+{
+    const ssize_t r = read(z->fds[0], buf, z->lread);
+
+    if (r < 0)
+    {
+        if (errno == EAGAIN || errno == EWOULDBLOCK)
+        {
+            free_zip(z);
+            *done = true;
+            return 0;
+        }
+        else
+            fprintf(stderr, "%s: read(2): %s\n", __func__, strerror(errno));
+    }
+
+    return r;
+}
+
+static int dump_end(void *const buf, const size_t n, bool *const done,
+    void *const user, struct zip *const z)
+{
+    const ssize_t r = read(z->fds[0], buf, z->lread);
+
+    if (r < 0)
+    {
+        if (errno == EAGAIN || errno == EWOULDBLOCK)
+        {
+            if (close(z->fd))
+            {
+                fprintf(stderr, "%s: close(2): %s\n", __func__, strerror(errno));
+                return -1;
+            }
+
+            z->fd = -1;
+            z->next = recurse;
+            return 0;
+        }
+    }
+
+    return r;
+}
+
+static int dumpz(void *const buf, const size_t n, bool *const done,
+    void *const user, struct zip *const z)
+{
+    const ssize_t r = read(z->fds[0], buf, z->lread);
+
+    if (r < 0)
+    {
+        if (errno != EAGAIN && errno != EWOULDBLOCK)
+        {
+            fprintf(stderr, "%s: read(2): %s\n", __func__, strerror(errno));
+            return -1;
+        }
+        else if (z->read == z->filesz)
+        {
+            if (!zs_entryend(z->stream, z->entry, NULL))
+            {
+                fprintf(stderr, "%s: zs_entryend failed\n", __func__);
+                return -1;
+            }
+
+            z->entry = NULL;
+            z->next = dump_end;
+        }
+        else
+            z->next = read_file;
+
+        return 0;
+    }
+
+    return r;
+}
+
+static int read_file(void *const buf, const size_t n, bool *const done,
+    void *const user, struct zip *const z)
+{
+    const off_t pend = z->filesz - z->read;
+    size_t rem = pend > n ? n : pend;
+    ssize_t r;
+
+    if ((r = read(z->fd, buf, rem)) < 0)
+    {
+        fprintf(stderr, "%s: read(2): %s\n", __func__, strerror(errno));
+        return -1;
+    }
+    else if (!zs_entrydata(z->stream, z->entry, buf, r, NULL))
+    {
+        fprintf(stderr, "%s: zs_entrydata failed\n", __func__);
+        return -1;
+    }
+
+    z->lread = r;
+    z->read += r;
+    z->next = dumpz;
+    return 0;
+}
+
+static int setup_file(struct zip *const z, const char *const fpath,
+    const time_t t)
+{
+    struct stat sb;
+    ZIPentry *entry = NULL;
+    const int fd = open(fpath, O_RDONLY | O_NONBLOCK);
+    /* zs_entrybegin is not const-correct. */
+    char *path = (char *)fpath + strlen(z->basedir);
+
+    if (fd < 0)
+    {
+        fprintf(stderr, "%s: open(2) %s: %s\n", __func__, fpath,
+            strerror(errno));
+        goto failure;
+    }
+    else if (fstat(fd, &sb))
+    {
+        fprintf(stderr, "%s: fstat(2) %s: %s\n", __func__, fpath,
+            strerror(errno));
+        goto failure;
+    }
+    else if (!(entry = zs_entrybegin(z->stream, path, t, ZS_DEFLATE,
+        NULL)))
+    {
+        fprintf(stderr, "%s: zs_entrybegin failed\n", __func__);
+        goto failure;
+    }
+
+    z->fd = fd;
+    z->read = 0;
+    z->entry = entry;
+    z->filesz = sb.st_size;
+    z->next = read_file;
+    return 0;
+
+failure:
+
+    if (fd && close(fd))
+        fprintf(stderr, "%s: close(2) %s: %s\n", __func__, fpath,
+            strerror(errno));
+
+    return -1;
+}
+
+static int setup(const char *const fpath, const struct stat *sb,
+    bool *const done, void *const user)
+{
+    struct zip *const z = user;
+
+    if (S_ISREG(sb->st_mode) && setup_file(z, fpath, sb->st_ctim.tv_sec))
+    {
+        fprintf(stderr, "%s: setup_file failed\n", __func__);
+        return -1;
+    }
+
+    return 0;
+}
+
+static int finalize(struct zip *const z)
+{
+    if (zs_finish(z->stream, NULL))
+    {
+        fprintf(stderr, "%s: zs_finish failed\n", __func__);
+        return -1;
+    }
+
+    z->finished = true;
+    z->next = dump_final;
+    return 0;
+}
+
+static int recurse(void *const buf, const size_t n, bool *const done,
+    void *const user, struct zip *const z)
+{
+    switch (cftw_step(z->cftw))
+    {
+        case CFTW_AGAIN:
+            break;
+
+        case CFTW_FATAL:
+            fprintf(stderr, "%s: cftw_step failed\n", __func__);
+            return -1;
+
+        case CFTW_OK:
+            if (finalize(z))
+            {
+                fprintf(stderr, "%s: finalize end\n", __func__);
+                return -1;
+            }
+
+            break;
+    }
+
+    return 0;
+}
+
+static int step(void *const buf, const size_t n, bool *const done,
+    void *const user, void *const args)
+{
+    struct zip *const z = args;
+    const int ret = z->next(buf, n, done, user, z);
+
+    if (ret < 0)
+        free_zip(z);
+
+    return ret;
+}
+
+int zip(const char *const dir, struct http_response *const r)
+{
+    struct cftw *c = NULL;
+    ZIPstream *stream = NULL;
+    struct zip *z = NULL;
+    char *basedir = NULL, *bndup = NULL, *bn = NULL;
+    int fds[2] = {-1, -1}, flags;
+    struct dynstr d;
+
+    dynstr_init(&d);
+
+    if (pipe(fds))
+    {
+        fprintf(stderr, "%s: pipe(2): %s\n", __func__, strerror(errno));
+        goto failure;
+    }
+    else if ((flags = fcntl(fds[0], F_GETFL)) == -1
+        || fcntl(fds[0], F_SETFL, flags | O_NONBLOCK) == -1)
+    {
+        fprintf(stderr, "%s: fcntl(2): %s\n", __func__, strerror(errno));
+        goto failure;
+    }
+    else if (!(z = malloc(sizeof *z)))
+    {
+        fprintf(stderr, "%s: malloc(3): %s\n", __func__, strerror(errno));
+        goto failure;
+    }
+    else if (!(c = cftw(dir, setup, z)))
+    {
+        fprintf(stderr, "%s: cftw failed\n", __func__);
+        goto failure;
+    }
+    else if (!(basedir = strdup(dir))
+        || !(bndup = strdup(dir)))
+    {
+        fprintf(stderr, "%s: strdup(3): %s\n", __func__, strerror(errno));
+        goto failure;
+    }
+    else if (!(bn = basename(bndup)))
+    {
+        fprintf(stderr, "%s: basename(3) failed\n", __func__);
+        goto failure;
+    }
+    else if (!(stream = zs_init(fds[1], NULL)))
+    {
+        fprintf(stderr, "%s: zs_init failed\n", __func__);
+        goto failure;
+    }
+    else if (dynstr_append(&d, "attachment; filename=\"%s.zip\"", bn))
+    {
+        fprintf(stderr, "%s: dynstr_append failed\n", __func__);
+        goto failure;
+    }
+
+    *z = (const struct zip)
+    {
+        .fds = {[0] = fds[0], [1] = fds[1]},
+        .basedir = basedir,
+        .stream = stream,
+        .cftw = c,
+        .next = recurse
+    };
+
+    *r = (const struct http_response)
+    {
+        .status = HTTP_STATUS_OK,
+        .chunk = step,
+        .args = z
+    };
+
+    if (http_response_add_header(r, "Content-Type", "application/zip")
+        || http_response_add_header(r, "Content-Disposition", d.str))
+    {
+        fprintf(stderr, "%s: http_response_add_header failed\n", __func__);
+        goto failure;
+    }
+
+    dynstr_free(&d);
+    free(bndup);
+    return 0;
+
+failure:
+
+    if ((fds[0] >= 0 && close(fds[0]))
+        || (fds[1] >= 0 && close(fds[1])))
+        fprintf(stderr, "%s: close(2): %s\n", __func__, strerror(errno));
+
+    dynstr_free(&d);
+    zs_free(stream);
+    cftw_free(c);
+    free(basedir);
+    free(bndup);
+    free(z);
+    return -1;
+}
diff --git a/zip.h b/zip.h
new file mode 100644
index 0000000..79524b5
--- /dev/null
+++ b/zip.h
@@ -0,0 +1,8 @@
+#ifndef ZIP_H
+#define ZIP_H
+
+#include <libweb/http.h>
+
+int zip(const char *dir, struct http_response *r);
+
+#endif