Make search non-blocking

Thanks to a new feature in libweb, it is now possible to generate HTTP
responses asynchronously i.e., without blocking other clients if the
response takes a long time to generate.

This now allow users to search for files or directories without blocking
other users, regardless how much time the search operation takes.

This required cftw to deviate from the POSIX-like, blocking interface it
had so far, and has been replaced now with a non-blocking interface, so
that directories are inspected one entry at a time.

4 files changed, 286 insertions, 95 deletions

diff --git a/cftw.c b/cftw.c
index 4b8b013..87684fa 100644
--- a/cftw.c
+++ b/cftw.c
@@ -7,91 +7,233 @@
 #include <errno.h>
 #include <stdio.h>
 #include <stdint.h>
+#include <stdlib.h>
 #include <string.h>
 
-static int do_cftw(const char *const dirpath, int (*const fn)(const char *,
-    const struct stat *, bool *, void *), bool *const done, void *const user)
+struct cftw_entry
 {
-    int ret = -1;
+    DIR *d;
+    char *dirpath;
+    struct cftw_entry *child;
+};
+
+struct cftw
+{
+    bool done;
+    char *dirpath;
+    int (*fn)(const char *, const struct stat *, bool *, void *);
+    void *user;
+    struct cftw_entry *root;
+};
+
+static int free_entry(struct cftw_entry *const e)
+{
+    int ret = 0;
+
+    if (e->d && closedir(e->d))
+    {
+        fprintf(stderr, "%s: closedir(2) %s: %s\n", __func__, e->dirpath,
+            strerror(errno));
+        ret = -1;
+    }
+
+    free(e->dirpath);
+    free(e);
+    return ret;
+}
+
+void cftw_free(struct cftw *const c)
+{
+    if (!c)
+        return;
+
+    for (struct cftw_entry *e = c->root; e;)
+    {
+        struct cftw_entry *const child = e->child;
+
+        free_entry(e);
+        e = child;
+    }
+
+    free(c);
+}
+
+static struct cftw_entry *entry(const char *const dirpath)
+{
+    struct cftw_entry *ret = NULL;
+    char *dirdup = NULL;
     DIR *const d = opendir(dirpath);
 
     if (!d)
     {
-        fprintf(stderr, "%s: opendir(2): %s\n", __func__, strerror(errno));
-        goto end;
+        fprintf(stderr, "%s: opendir(2) %s: %s\n", __func__, dirpath,
+            strerror(errno));
+        goto failure;
+    }
+    else if (!(dirdup = strdup(dirpath)))
+    {
+        fprintf(stderr, "%s: strdup(3): %s\n", __func__, strerror(errno));
+        goto failure;
+    }
+    else if (!(ret = malloc(sizeof *ret)))
+    {
+        fprintf(stderr, "%s: malloc(3): %s\n", __func__, strerror(errno));
+        goto failure;
     }
 
-    for (;;)
+    *ret = (const struct cftw_entry)
     {
-        errno = 0;
-        struct dirent *const de = readdir(d);
+        .dirpath = dirdup,
+        .d = d
+    };
 
-        if (errno)
-        {
-            fprintf(stderr, "%s: readdir(3): %s\n", __func__, strerror(errno));
-            goto end;
-        }
-        else if (!de)
-            break;
+    return ret;
+
+failure:
 
-        const char *const path = de->d_name;
+    if (d && closedir(d))
+        fprintf(stderr, "%s: closedir(2) %s: %s\n", __func__, dirpath,
+            strerror(errno));
 
-        if (!strcmp(path, ".") || !strcmp(path, ".."))
-            continue;
+    free(dirdup);
+    free(ret);
+    return NULL;
+}
 
-        const char *const sep = dirpath[strlen(dirpath) - 1] == '/' ? "" : "/";
-        struct stat sb;
-        struct dynstr d;
+static enum cftw_state run(struct cftw *const c, struct cftw_entry *const e)
+{
+    int error;
+    struct dirent *de;
+    struct dynstr d;
 
-        dynstr_init(&d);
+    dynstr_init(&d);
+    errno = 0;
+    de = readdir(e->d);
 
-        if (dynstr_append(&d, "%s%s%s", dirpath, sep, path))
-        {
-            fprintf(stderr, "%s: dynstr_append failed\n", __func__);
-            return -1;
-        }
+    if (errno)
+    {
+        fprintf(stderr, "%s: readdir(3): %s\n", __func__, strerror(errno));
+        goto failure;
+    }
+    else if (!de)
+        return CFTW_OK;
 
-        const int r = stat(d.str, &sb);
+    const char *const path = de->d_name, *const dirpath = e->dirpath;
 
-        if (r)
-            fprintf(stderr, "%s: stat(2) %s: %s\n",
-                __func__, path, strerror(errno));
-        else if (S_ISDIR(sb.st_mode))
-        {
-            if ((ret = do_cftw(d.str, fn, done, user)))
-                ;
-            else if ((ret = fn(d.str, &sb, done, user)))
-                ;
-        }
-        else if (S_ISREG(sb.st_mode))
-            ret = fn(d.str, &sb, done, user);
-        else
-            fprintf(stderr, "%s: unexpected st_mode %ju\n",
-                __func__, (uintmax_t)sb.st_mode);
+    if (!strcmp(path, ".") || !strcmp(path, ".."))
+        return CFTW_AGAIN;
 
-        dynstr_free(&d);
+    const char *const sep = dirpath[strlen(dirpath) - 1] == '/' ? "" : "/";
+    struct stat sb;
 
-        if (ret || *done)
-            goto end;
+    if (dynstr_append(&d, "%s%s%s", dirpath, sep, path))
+    {
+        fprintf(stderr, "%s: dynstr_append failed\n", __func__);
+        goto failure;
     }
 
-    ret = 0;
+    const int r = stat(d.str, &sb);
 
-end:
+    if (r)
+    {
+        fprintf(stderr, "%s: stat(2) %s: %s\n", __func__, path,
+            strerror(errno));
+        goto failure;
+    }
+    else if (S_ISDIR(sb.st_mode))
+    {
+        if (!(e->child = entry(d.str)))
+            goto failure;
 
-    if (d && closedir(d))
+        error = c->fn(d.str, &sb, &c->done, c->user);
+    }
+    else if (S_ISREG(sb.st_mode))
+        error = c->fn(d.str, &sb, &c->done, c->user);
+    else
     {
-        fprintf(stderr, "%s: closedir(2): %s\n", __func__, strerror(errno));
-        ret = -1;
+        fprintf(stderr, "%s: unexpected st_mode %ju\n",
+            __func__, (uintmax_t)sb.st_mode);
+        goto failure;
     }
 
-    return ret;
+    dynstr_free(&d);
+
+    if (error)
+        return CFTW_FATAL;
+    else if (c->done)
+        return CFTW_OK;
+
+    return CFTW_AGAIN;
+
+failure:
+    dynstr_free(&d);
+    return CFTW_FATAL;
+}
+
+static enum cftw_state step(struct cftw *const c, struct cftw_entry *const e)
+{
+    if (e->child)
+    {
+        switch (step(c, e->child))
+        {
+            case CFTW_AGAIN:
+                return CFTW_AGAIN;
+
+            case CFTW_FATAL:
+               free_entry(e->child);
+               return CFTW_FATAL;
+
+            case CFTW_OK:
+                if (free_entry(e->child))
+                    return CFTW_FATAL;
+
+                e->child = NULL;
+                break;
+        }
+
+        return CFTW_AGAIN;
+    }
+
+    return run(c, e);
 }
 
-int cftw(const char *const dirpath, int (*const fn)(const char *,
+enum cftw_state cftw_step(struct cftw *c)
+{
+    return step(c, c->root);
+}
+
+struct cftw *cftw(const char *const dirpath, int (*const fn)(const char *,
     const struct stat *, bool *, void *), void *const user)
 {
-    bool done = false;
+    struct cftw *ret = NULL;
+    char *const dirdup = strdup(dirpath);
+
+    if (!dirdup)
+    {
+        fprintf(stderr, "%s: strdup(3): %s\n", __func__, strerror(errno));
+        goto failure;
+    }
+    else if (!(ret = malloc(sizeof *ret)))
+    {
+        fprintf(stderr, "%s: malloc(3): %s\n", __func__, strerror(errno));
+        goto failure;
+    }
+
+    *ret = (const struct cftw)
+    {
+        .dirpath = dirdup,
+        .fn = fn,
+        .user = user,
+        .root = entry(dirpath)
+    };
+
+    if (!ret->root)
+        goto failure;
+
+    return ret;
 
-    return do_cftw(dirpath, fn, &done, user);
+failure:
+    free(dirdup);
+    free(ret);
+    return NULL;
 }
diff --git a/cftw.h b/cftw.h
index 663960c..645e082 100644
--- a/cftw.h
+++ b/cftw.h
@@ -4,9 +4,18 @@
 #include <stdbool.h>
 #include <sys/stat.h>
 
-/* Thread-safe variant of ftw(3) and nftw(3) that allows passing an
- * opaque pointer and removes some unneeded parameters. */
-int cftw(const char *dirpath, int (*fn)(const char *fpath,
+enum cftw_state
+{
+    CFTW_OK,
+    CFTW_AGAIN,
+    CFTW_FATAL
+};
+
+/* Thread-safe and non-blocking variant of ftw(3) and nftw(3) that allows
+ * passing an opaque pointer and removes some unneeded parameters. */
+struct cftw *cftw(const char *dirpath, int (*fn)(const char *fpath,
     const struct stat *sb, bool *done, void *user), void *user);
+enum cftw_state cftw_step(struct cftw *c);
+void cftw_free(struct cftw *c);
 
 #endif /* CFTW_H */
diff --git a/libweb b/libweb
-Subproject f7864cb7d49a8ca5bddf8d1f68b71ecd5ed85ad
+Subproject 5a6f30440b66fe6713acb9d979dc3e6624e4c36
diff --git a/main.c b/main.c
index 3702706..e42236d 100644
--- a/main.c
+++ b/main.c
@@ -46,6 +46,14 @@ struct user_args
     struct dynstr d;
     int fd;
     bool fifo;
+    void *state;
+};
+
+struct search
+{
+    struct cftw *cftw;
+    struct dynstr root, res;
+    struct page_search search;
 };
 
 static struct handler *handler;
@@ -647,24 +655,18 @@ static void search_results_free(struct page_search *const s)
     }
 }
 
-struct search_args
-{
-    const char *root, *res;
-    struct page_search *s;
-};
-
 static int search_fn(const char *const fpath, const struct stat *const sb,
     bool *const done, void *const user)
 {
     static const size_t limit = 200;
-    const struct search_args *const sa = user;
-    const char *rel = fpath + strlen(sa->root);
-    struct page_search *const res = sa->s;
+    struct search *const s = user;
+    const char *rel = fpath + strlen(s->root.str);
+    struct page_search *const res = &s->search;
     struct page_search_result *results = NULL, *r = NULL;
 
     rel += strspn(rel, "/");
 
-    if (wildcard_cmp(rel, sa->res, false))
+    if (wildcard_cmp(rel, s->res.str, false))
         return 0;
     else if (!(results = realloc(res->results,
         (res->n + 1) * sizeof *res->results)))
@@ -689,7 +691,7 @@ static int search_fn(const char *const fpath, const struct stat *const sb,
 
     if (++res->n >= limit)
     {
-        sa->s->limit_exceeded = true;
+        s->search.limit_exceeded = true;
         *done = true;
     }
 
@@ -701,42 +703,62 @@ failure:
     return -1;
 }
 
-static int do_search(const char *const abs, const char *const root,
-    const char *const res, struct page_search *const s)
+static void free_search(struct search *const s)
 {
-    struct search_args sa =
-    {
-        .root = root,
-        .res = res,
-        .s = s
-    };
+    if (!s)
+        return;
+
+    search_results_free(&s->search);
+    dynstr_free(&s->root);
+    dynstr_free(&s->res);
+    cftw_free(s->cftw);
+    free(s);
+}
 
-    s->root = root;
+static int search_step(const struct http_payload *const p,
+    struct http_response *const r, void *const user)
+{
+    const struct user_args *const ua = user;
+    struct search *const s = ua->state;
 
-    if (cftw(abs, search_fn, &sa))
+    switch (cftw_step(s->cftw))
     {
-        fprintf(stderr, "%s: cftw failed\n", __func__);
-        return -1;
+        case CFTW_OK:
+
+            if (page_search(r, &s->search))
+            {
+                fprintf(stderr, "%s: page_search failed\n", __func__);
+                goto failure;
+            }
+
+            free_search(s);
+            break;
+
+        case CFTW_FATAL:
+            goto failure;
+
+        case CFTW_AGAIN:
+            break;
     }
 
     return 0;
+
+failure:
+    free_search(s);
+    return -1;
 }
 
 static int search(const struct http_payload *const p,
     struct http_response *const r, void *const user)
 {
     int ret = -1;
-    const struct user_args *const args = user;
+    struct search *s = NULL;
+    struct user_args *const args = user;
     const struct auth *const a = args->a;
     const char *const username = p->cookie.field, *const root = auth_dir(a);
     int (*f)(struct http_response *);
     char *dir = NULL;
     struct dynstr userd, d, res;
-    struct page_search s =
-    {
-        .username = username,
-        .adir = root
-    };
 
     dynstr_init(&userd);
     dynstr_init(&d);
@@ -767,25 +789,42 @@ static int search(const struct http_payload *const p,
         fprintf(stderr, "%s: dynstr_append d failed\n", __func__);
         goto end;
     }
-    else if ((ret = do_search(d.str, userd.str, res.str, &s)))
+    else if ((!(s = malloc(sizeof *s))))
     {
-        if (ret < 0)
-            fprintf(stderr, "%s: do_search failed\n", __func__);
+        fprintf(stderr, "%s: malloc(3): %s\n", __func__, strerror(errno));
+        goto end;
+    }
+
+    *s = (const struct search){0};
+    dynstr_init(&s->root);
+    dynstr_init(&s->res);
 
+    if (!(s->cftw = cftw(d.str, search_fn, s)))
+    {
+        fprintf(stderr, "%s: cftw failed\n", __func__);
         goto end;
     }
-    else if ((ret = page_search(r, &s)))
+    else if (dynstr_dup(&s->root, &userd)
+        || dynstr_dup(&s->res, &res))
     {
-        fprintf(stderr, "%s: page_search failed\n", __func__);
+        fprintf(stderr, "%s: dynstr_dup failed\n", __func__);
         goto end;
     }
 
+    *r = (const struct http_response){.step = search_step};
+    s->search.root = s->root.str;
+    args->state = s;
+    ret = 0;
+
 end:
     free(dir);
-    dynstr_free(&userd);
     dynstr_free(&d);
+    dynstr_free(&userd);
     dynstr_free(&res);
-    search_results_free(&s);
+
+    if (ret)
+        free_search(s);
+
     return ret;
 }
 
@@ -947,7 +986,8 @@ static int check_length(const unsigned long long len,
     const struct http_cookie *const c, struct http_response *const r,
     void *const user)
 {
-    struct auth *const a = user;
+    const struct user_args *const ua = user;
+    const struct auth *const a = ua->a;
     const char *const username = c->field;
     bool has_quota;
     unsigned long long quota;