From ce60ea3e4cf733aadd4ecb8ca08e2f81b498d67c Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Tue, 27 Nov 2018 02:03:17 +0000 Subject: serve /dev*/usage requests from memory Filesystems may become unwritable and out-of-date "usage" files will cause trackers to see out-of-date information. We still write to the filesystem by default for compatibility with existing HTTP servers. However, giving the "usage" file a 0000 mode will prevent cmogstored from overwriting it. This allows admins to also reduce wear on storage devices: chmod 0000 $mogroot/dev*/usage --- cmogstored.h | 12 +++++-- dev.c | 72 +++++++++++++++++++++++++++++++++----- fs.c | 13 +++++++ fs.h | 2 ++ http.c | 32 +++++++++-------- http_get.c | 93 +++++++++++++++++++++++++++++++++++++++++++++----- http_parser.rl | 10 ++++-- test/cmogstored-cfg.rb | 4 ++- test/http-parser-1.c | 35 +++++++++++++++++++ 9 files changed, 235 insertions(+), 38 deletions(-) diff --git a/cmogstored.h b/cmogstored.h index db87ac0..61a7ffc 100644 --- a/cmogstored.h +++ b/cmogstored.h @@ -109,6 +109,10 @@ struct mog_wbuf; struct mog_dev { dev_t st_dev; uint32_t devid; + pthread_mutex_t usage_lock; /* protects usage_txt */ + unsigned usage_len; + char *usage_txt; + time_t usage_mtime; struct mog_ioq ioq; /* normal requests */ struct mog_ioq fsckq; /* low-priority for MogileFS fsck */ }; @@ -206,8 +210,9 @@ struct mog_http { unsigned has_range:1; /* for GET */ unsigned bad_range:1; unsigned skip_rbuf_defer:1; + unsigned usage_txt:1; enum mog_chunk_state chunk_state:2; - unsigned unused_padding:2; + unsigned unused_padding:1; uint8_t path_tip; uint8_t path_end; uint16_t line_end; @@ -374,7 +379,7 @@ void mog_set_maxconns(unsigned long); /* svc.c */ struct mog_svc *mog_svc_new(const char *docroot); -typedef int (*mog_scandev_cb)(const struct mog_dev *, struct mog_svc *); +typedef int (*mog_scandev_cb)(struct mog_dev *, struct mog_svc *); size_t mog_svc_each(Hash_processor processor, void *data); void mog_svc_upgrade_prepare(void); bool mog_svc_start_each(void *svc_ptr, void *have_mgmt_ptr); @@ -385,12 +390,13 @@ bool mog_svc_atfork_child(void *svc_ptr, void *parent); /* dev.c */ struct mog_dev *mog_dev_for(struct mog_svc *, uint32_t mog_devid, bool update); -int mog_dev_mkusage(const struct mog_dev *, struct mog_svc *); +int mog_dev_mkusage(struct mog_dev *, struct mog_svc *); size_t mog_dev_hash(const void *, size_t tablesize); bool mog_dev_cmp(const void *a, const void *b); void mog_dev_free(void *devptr); bool mog_dev_user_rescale_i(void *devp, void *svcp); bool mog_dev_requeue_prepare(void *devp, void *ign); +void mog_dev_usage_update(struct mog_dev *, struct mog_svc *); /* valid_path.rl */ int mog_valid_path(const char *buf, size_t len); diff --git a/dev.c b/dev.c index 4cb15d2..3b52023 100644 --- a/dev.c +++ b/dev.c @@ -33,6 +33,10 @@ static struct mog_dev *mog_dev_new(struct mog_svc *svc, uint32_t mog_devid) dev->st_dev = sb.st_dev; mog_ioq_init(&dev->fsckq, svc, 1); mog_ioq_init(&dev->ioq, svc, svc->thr_per_dev); + dev->usage_txt = 0; + dev->usage_len = 0; + dev->usage_mtime = 0; + CHECK(int, 0, pthread_mutex_init(&dev->usage_lock, NULL)); return dev; } @@ -108,8 +112,7 @@ bool mog_dev_cmp(const void *a, const void *b) } static int -emit_usage( -const struct mog_dev *dev, struct mog_svc *svc, int fd, struct statvfs *v) +emit_usage(struct mog_dev *dev, struct mog_svc *svc, int fd, struct statvfs *v) { int rc = -1; unsigned long long available = v->f_bavail; @@ -129,6 +132,7 @@ const struct mog_dev *dev, struct mog_svc *svc, int fd, struct statvfs *v) me = mog_mnt_acquire(dev->st_dev); if (me) { + char *usage_txt; static const char usage_fmt[] = "available: %llu\n" "device: %s\n" @@ -139,14 +143,34 @@ const struct mog_dev *dev, struct mog_svc *svc, int fd, struct statvfs *v) "used: %llu\n"; errno = 0; - rc = dprintf(fd, usage_fmt, + rc = asprintf(&usage_txt, usage_fmt, available, me->me_devname, svc->docroot, (unsigned)dev->devid, now, total, use, used); + if (rc > 0) { + char *old_usage; + + CHECK(int, 0, pthread_mutex_lock(&dev->usage_lock)); + old_usage = dev->usage_txt; + dev->usage_txt = usage_txt; + dev->usage_len = rc; + dev->usage_mtime = (time_t)now; + CHECK(int, 0, pthread_mutex_unlock(&dev->usage_lock)); + + free(old_usage); + if (fd >= 0) { + ssize_t w = write(fd, usage_txt, rc); + + if (w >= 0 && w != rc) + errno = ENOSPC; + else if (w < 0) + rc = -1; + } + } PRESERVE_ERRNO( mog_mnt_release(me) ); if (rc < 0 || errno == ENOSPC) { PRESERVE_ERRNO(do { - syslog(LOG_ERR, "dprintf(%s/dev%u/usage): %m", + syslog(LOG_ERR, "write(%s/dev%u/usage): %m", svc->docroot, (unsigned)dev->devid); } while (0)); } @@ -159,12 +183,32 @@ const struct mog_dev *dev, struct mog_svc *svc, int fd, struct statvfs *v) return rc; } -int mog_dev_mkusage(const struct mog_dev *dev, struct mog_svc *svc) +static void +dev_usage_update(struct mog_dev *dev, struct mog_svc *svc, struct statvfs *v) +{ + if (mog_statvfs(svc, dev, v) < 0) { + syslog(LOG_ERR, "statvfs error: %s/dev%u/usage (%m)", + svc->docroot, (unsigned)dev->devid); + return; + } + (void)emit_usage(dev, svc, -1, v); +} + +void +mog_dev_usage_update(struct mog_dev *dev, struct mog_svc *svc) +{ + struct statvfs v; + + dev_usage_update(dev, svc, &v); +} + +int mog_dev_mkusage(struct mog_dev *dev, struct mog_svc *svc) { struct statvfs v; char *usage_path; char *tmp_path; int fd = -1; + struct stat sb; if (!svc->mgmt_mfd) return 0; @@ -174,6 +218,17 @@ int mog_dev_mkusage(const struct mog_dev *dev, struct mog_svc *svc) (unsigned)dev->devid); return 0; } + + /* + * allow chmod 0000 on devNNN/usage files to prevent us from + * overwriting them + */ + if (mog_stat(svc, usage_path, &sb) == 0 && + ((sb.st_mode & (S_IRWXU|S_IRWXG|S_IRWXO)) == 0)) { + dev_usage_update(dev, svc, &v); + tmp_path = 0; + goto out; + } if (asprintf(&tmp_path, "%s.%x", usage_path, (unsigned)getpid()) < 0) { syslog(LOG_ERR, "error generating path: /dev%u/usage.%u (%m)", (unsigned)dev->devid, (unsigned)getpid()); @@ -185,15 +240,12 @@ int mog_dev_mkusage(const struct mog_dev *dev, struct mog_svc *svc) errno = 0; fd = mog_open_put(svc, tmp_path, O_EXCL|O_CREAT); - if (fd < 0) { - if (mog_open_expire_retry(svc)) - fd = mog_open_put(svc, tmp_path, O_EXCL|O_CREAT); - } if (fd < 0) { PRESERVE_ERRNO(do { syslog(LOG_ERR, "open(%s%s): %m", svc->docroot, tmp_path); } while (0)); + dev_usage_update(dev, svc, &v); goto out; } if (fstatvfs(fd, &v) < 0) { @@ -252,6 +304,8 @@ void mog_dev_free(void *ptr) mog_ioq_destroy(&dev->fsckq); mog_ioq_destroy(&dev->ioq); + free(dev->usage_txt); + CHECK(int, 0, pthread_mutex_destroy(&dev->usage_lock)); free(dev); } diff --git a/fs.c b/fs.c index 1c51977..a0f1147 100644 --- a/fs.c +++ b/fs.c @@ -126,3 +126,16 @@ int mog_mkdir(struct mog_svc *svc, const char *path, mode_t mode) return mkdir(fspath, mode); } #endif /* !HAVE_MKDIRAT */ + +int mog_statvfs(struct mog_svc *svc, struct mog_dev *dev, struct statvfs *v) +{ + char fspath[MY_PATHMAX]; + int rc = snprintf(fspath, sizeof(fspath), "%s/dev%u", + svc->docroot, dev->devid); + + if (rc <= 0) { + errno = ENAMETOOLONG; + return -1; + } + return statvfs(fspath, v); +} diff --git a/fs.h b/fs.h index 19d3961..63e16e8 100644 --- a/fs.h +++ b/fs.h @@ -45,3 +45,5 @@ int mog_open_read(struct mog_svc *svc, const char *path); #ifdef O_CLOEXEC void mog_cloexec_works(void); #endif + +int mog_statvfs(struct mog_svc *, struct mog_dev *, struct statvfs *); diff --git a/http.c b/http.c index 1061e63..e685273 100644 --- a/http.c +++ b/http.c @@ -447,25 +447,29 @@ void mog_httpget_post_accept(int fd, struct mog_accept *ac, */ char *mog_http_path(struct mog_http *http, char *buf) { - char *path = buf + http->_p.path_tip; - size_t len = http->_p.path_end - http->_p.path_tip; - - assert(http->_p.path_end > http->_p.path_tip - && "bad HTTP path from parser"); - - if (! mog_valid_path(path, len)) + if (http->_p.usage_txt) { + errno = EACCES; return NULL; + } else { + char *path = buf + http->_p.path_tip; + size_t len = http->_p.path_end - http->_p.path_tip; - if (http->_p.http_method == MOG_HTTP_METHOD_PUT) { - if (!mog_valid_put_path(path, len)) { - errno = EINVAL; + assert(http->_p.path_end > http->_p.path_tip + && "bad HTTP path from parser"); + + if (! mog_valid_path(path, len)) return NULL; - } - } - path[len] = '\0'; + if (http->_p.http_method == MOG_HTTP_METHOD_PUT) { + if (!mog_valid_put_path(path, len)) { + errno = EINVAL; + return NULL; + } + } - return path; + path[len] = '\0'; + return path; + } } diff --git a/http_get.c b/http_get.c index 154d8c5..1af4fd2 100644 --- a/http_get.c +++ b/http_get.c @@ -47,6 +47,16 @@ static ssize_t linux_sendfile(int sockfd, int filefd, off_t *off, size_t count) #define ERR416 "416 Requested Range Not Satisfiable" +static void +http_hdr_prepare(char **buf, char **modified, size_t *len, time_t *mtime) +{ + /* single buffer so we can use MSG_MORE */ + *buf = mog_fsbuf_get(len); + *modified = *buf + *len / 2; + assert((*len / 2) > MOG_HTTPDATE_CAPA && "fsbuf too small"); + mog_http_date(*modified, MOG_HTTPDATE_CAPA, mtime); +} + /* * TODO: refactor this * @@ -58,18 +68,13 @@ static ssize_t linux_sendfile(int sockfd, int filefd, off_t *off, size_t count) static off_t http_get_resp_hdr(struct mog_fd *mfd, struct stat *sb) { struct mog_http *http = &mfd->as.http; - char *modified; - char *buf; + char *buf, *modified; size_t len; struct mog_now *now = mog_now(); long long count; int rc; - /* single buffer so we can use MSG_MORE */ - buf = mog_fsbuf_get(&len); - modified = buf + len / 2; - assert((len / 2) > MOG_HTTPDATE_CAPA && "fsbuf too small"); - mog_http_date(modified, MOG_HTTPDATE_CAPA, &sb->st_mtime); + http_hdr_prepare(&buf, &modified, &len, &sb->st_mtime); /* validate ranges */ if (http->_p.has_range) { @@ -194,14 +199,86 @@ bad_range: return (off_t)count; } +static void emit_dev_usage(struct mog_fd *mfd) +{ + struct mog_http *http = &mfd->as.http; + struct mog_dev *dev = mog_dev_for(http->svc, http->_p.mog_devid, false); + void *ok = NULL; + + if (dev) { + char *buf, *modified; + size_t len, ilen; + struct mog_now *now; + int rc; + bool retried = false; + struct iovec iov; + +retry: + now = mog_now(); + CHECK(int, 0, pthread_mutex_lock(&dev->usage_lock)); + ok = dev->usage_txt; + if (!ok) { + if (retried) + goto out_unlock; + retried = true; + CHECK(int, 0, pthread_mutex_unlock(&dev->usage_lock)); + mog_dev_usage_update(dev, http->svc); + goto retry; + } + + http_hdr_prepare(&buf, &modified, &len, + &dev->usage_mtime); + ilen = len; + rc = snprintf(buf, len, + "HTTP/1.1 200 OK\r\n" + "Date: %s\r\n" + "Last-Modified: %s\r\n" + "Content-Length: %u\r\n" + "Content-Type: text/plain\r\n" + "Accept-Ranges: bytes\r\n" + "Connection: %s\r\n" + "\r\n", + now->httpdate, + modified, + dev->usage_len, + http->_p.persistent ? "keep-alive" : "close"); + + ok = NULL; + if (rc > 0) { + len -= rc; + if (http->_p.http_method == MOG_HTTP_METHOD_HEAD) { + ok = iov.iov_base = buf; + iov.iov_len = rc; + } else if (len >= dev->usage_len && len < ilen) { + memcpy(buf + rc, dev->usage_txt, + dev->usage_len); + ok = iov.iov_base = buf; + iov.iov_len = rc + dev->usage_len; + } + } +out_unlock: + CHECK(int, 0, pthread_mutex_unlock(&dev->usage_lock)); + if (ok) + http->wbuf = mog_trywritev(mfd->fd, &iov, 1); + } + if (!dev || !ok) + mog_http_resp(mfd, "404 Not Found", true); +} + void mog_http_get_open(struct mog_fd *mfd, char *buf) { struct mog_http *http = &mfd->as.http; struct stat sb; struct mog_file *file = NULL; - char *path = mog_http_path(http, buf); + char *path; off_t len; + if (http->_p.usage_txt) { + emit_dev_usage(mfd); + return; + } + + path = mog_http_path(http, buf); if (!path) goto forbidden; /* path traversal attack */ assert(http->forward == NULL && "already have http->forward"); assert(path[0] == '/' && "bad path"); diff --git a/http_parser.rl b/http_parser.rl index 300b571..ae4217a 100644 --- a/http_parser.rl +++ b/http_parser.rl @@ -48,12 +48,16 @@ static char *skip_header(struct mog_http *http, char *buf, const char *pe) DELETE = "DELETE "> { http->_p.http_method = MOG_HTTP_METHOD_DELETE; }; MKCOL = "MKCOL "> { http->_p.http_method = MOG_HTTP_METHOD_MKCOL; }; + mog_fs_path = (mog_path) > { http->_p.path_tip = to_u8(fpc - buf); } + # TODO: maybe folks use query string/fragments for logging... + (" HTTP/1.") > { http->_p.path_end = to_u8(fpc - buf); }; + + usage_path = ('/' devid "usage HTTP/1.") @ { http->_p.usage_txt = 1; }; + # no HTTP/0.9 for now, sorry (not :P) req_line = (HEAD|GET|PUT|DELETE|MKCOL) ("http://" [^/]+)? - '/'*(mog_path) > { http->_p.path_tip = to_u8(fpc - buf); } - # TODO: maybe folks use query string/fragments for logging... - (" HTTP/1.") > { http->_p.path_end = to_u8(fpc - buf); } + '/'*(usage_path | mog_fs_path) ('0'|'1'> { http->_p.persistent = 1; }) '\r'LF; content_length = "Content-Length:"i sep diff --git a/test/cmogstored-cfg.rb b/test/cmogstored-cfg.rb index 07dfc6a..7cb862e 100644 --- a/test/cmogstored-cfg.rb +++ b/test/cmogstored-cfg.rb @@ -367,7 +367,9 @@ class TestCmogstoredConfig < Test::Unit::TestCase get_client Net::HTTP.start(@host, @port) do |http| resp = http.request(Net::HTTP::Get.new("/dev666/usage")) - assert_kind_of Net::HTTPNotFound, resp + assert_kind_of Net::HTTPOK, resp + assert resp.body.size > 0 + assert ! File.exist?("#@tmpdir/dev666/usage") resp = http.request(Net::HTTP::Get.new("/dev666/cmogstored.test")) assert_kind_of Net::HTTPOK, resp assert_equal "HI", resp.body diff --git a/test/http-parser-1.c b/test/http-parser-1.c index 9125b9c..f699611 100644 --- a/test/http-parser-1.c +++ b/test/http-parser-1.c @@ -41,6 +41,7 @@ int main(void) assert(http->_p.persistent && "not persistent"); assert(state == MOG_PARSER_DONE && "parser not done"); assert_path_equal("/foo"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("normal HTTP GET request with redundant leading slash") { @@ -51,6 +52,7 @@ int main(void) assert(http->_p.persistent && "not persistent"); assert(state == MOG_PARSER_DONE && "parser not done"); assert_path_equal("/foo"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("HTTP/1.1 request with explicit close") { @@ -64,6 +66,7 @@ int main(void) assert(http->_p.persistent == 0 && "should not be persistent"); assert(state == MOG_PARSER_DONE && "parser not done"); assert_path_equal("/foo"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("HTTP/1.0 request with keepalive") { @@ -76,6 +79,7 @@ int main(void) assert(http->_p.persistent == 1 && "should be persistent"); assert(state == MOG_PARSER_DONE && "parser not done"); assert_path_equal("/foo"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("bogus HTTP/1.0 request") { @@ -84,6 +88,7 @@ int main(void) "\r\n"); state = mog_http_parse(http, buf, len); assert(state == MOG_PARSER_ERROR && "parser not errored"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("bogus request") { @@ -92,6 +97,7 @@ int main(void) "\r\n"); state = mog_http_parse(http, buf, len); assert(state == MOG_PARSER_ERROR && "parser not errored"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("HTTP/1.1 HEAD request") { @@ -104,6 +110,7 @@ int main(void) assert(http->_p.persistent == 1 && "should be persistent"); assert(state == MOG_PARSER_DONE && "parser not done"); assert_path_equal("/foo"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("HTTP/1.1 PUT request") { @@ -121,6 +128,7 @@ int main(void) assert_path_equal("/foo"); assert(strcmp(buf + http->_p.buf_off, "partial body request") == 0 && "buffer repositioned to body start"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("HTTP/1.1 PUT chunked request header") { @@ -139,6 +147,7 @@ int main(void) assert_path_equal("/foo"); assert(strcmp(buf + http->_p.buf_off, "16\r\npartial...") == 0 && "buffer repositioned to body start"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("HTTP/1.1 PUT with Content-Range") { @@ -160,6 +169,7 @@ int main(void) assert_path_equal("/foo"); assert(strcmp(buf + http->_p.buf_off, "16\r\npartial...") == 0 && "buffer repositioned to body start"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("HTTP/1.1 PUT chunked request header w/Trailer") { @@ -179,6 +189,7 @@ int main(void) assert_path_equal("/foo"); assert(strcmp(buf + http->_p.buf_off, "16\r\npartial...") == 0 && "buffer repositioned to body start"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("HTTP/1.1 DELETE request") { @@ -193,6 +204,7 @@ int main(void) assert(http->_p.persistent == 1 && "should be persistent"); assert(state == MOG_PARSER_DONE && "parser not done"); assert_path_equal("/foo"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("HTTP/1.1 MKCOL request") { @@ -207,6 +219,7 @@ int main(void) assert(http->_p.persistent == 1 && "should be persistent"); assert(state == MOG_PARSER_DONE && "parser not done"); assert_path_equal("/foo"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("HTTP/1.1 Range (mid) GET request") { @@ -223,6 +236,7 @@ int main(void) assert(http->_p.persistent == 1 && "should be persistent"); assert(state == MOG_PARSER_DONE && "parser not done"); assert_path_equal("/foo"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("HTTP/1.1 Range (tip) GET request") { @@ -239,6 +253,7 @@ int main(void) assert(http->_p.persistent == 1 && "should be persistent"); assert(state == MOG_PARSER_DONE && "parser not done"); assert_path_equal("/foo"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("HTTP/1.1 Range (end) GET request") { @@ -255,6 +270,7 @@ int main(void) assert(http->_p.persistent == 1 && "should be persistent"); assert(state == MOG_PARSER_DONE && "parser not done"); assert_path_equal("/foo"); + assert(!http->_p.usage_txt && "not a usage request"); } if ("HTTP/1.1 devid parse") { @@ -266,6 +282,25 @@ int main(void) assert(http->_p.mog_devid == 666 && "dev666 set"); assert(state == MOG_PARSER_DONE && "parser not done"); assert_path_equal("/dev666/0/1.fid"); + assert(!http->_p.usage_txt && "not a usage request"); + } + + if ("HTTP/1.0 devN/usage request") { + buf_set("GET /dev666/usage HTTP/1.0\r\n\r\n"); + state = mog_http_parse(http, buf, len); + assert(http->_p.http_method == MOG_HTTP_METHOD_GET + && "http_method should be GET"); + assert(http->_p.mog_devid == 666 && "dev666 set"); + assert(state == MOG_PARSER_DONE && "parser not done"); + assert(http->_p.usage_txt && "a usage request"); + + buf_set("GET /dev666/usager HTTP/1.0\r\n\r\n"); + state = mog_http_parse(http, buf, len); + assert(!http->_p.usage_txt && "a usage request"); + + buf_set("GET /dev666/usag HTTP/1.0\r\n\r\n"); + state = mog_http_parse(http, buf, len); + assert(!http->_p.usage_txt && "a usage request"); } reset(); -- cgit v1.2.3-24-ge0c7