In preparation for libdaxctl and daxctl to grow operational modes for DAX devices, add the following supporting APIs: daxctl_dev_get_ctx daxctl_dev_is_enabled It also adds and uses a helper to verify the device model for the _is_enabled API, since enable/disable only make sense for the dax-bus model. Cc: Dan Williams <dan.j.williams(a)intel.com&gt; Reviewed-by: Dan Williams <dan.j.williams(a)intel.com&gt; Signed-off-by: Vishal Verma <vishal.l.verma(a)intel.com&gt; --- daxctl/lib/libdaxctl.c | 70 ++++++++++++++++++++++++++++++++++++++++ daxctl/lib/libdaxctl.sym | 6 ++++ daxctl/libdaxctl.h | 2 ++ 3 files changed, 78 insertions(+) diff --git a/daxctl/lib/libdaxctl.c b/daxctl/lib/libdaxctl.c index c2e3a52..916a49e 100644 --- a/daxctl/lib/libdaxctl.c +++ b/daxctl/lib/libdaxctl.c @@ -306,6 +306,43 @@ DAXCTL_EXPORT struct daxctl_region *daxctl_new_region(struct daxctl_ctx *ctx, return region; } +static bool device_model_is_dax_bus(struct daxctl_dev *dev) +{ + const char *devname = daxctl_dev_get_devname(dev); + struct daxctl_ctx *ctx = daxctl_dev_get_ctx(dev); + char *path = dev->dev_buf, *resolved; + size_t len = dev->buf_len; + struct stat sb; + + if (snprintf(path, len, "/dev/%s", devname) < 0) + return false; + + if (lstat(path, &sb) < 0) { + err(ctx, "%s: stat for %s failed: %s\n", + devname, path, strerror(errno)); + return false; + } + + if (snprintf(path, len, "/sys/dev/char/%d:%d/subsystem", + major(sb.st_rdev), minor(sb.st_rdev)) < 0) + return false; + + resolved = realpath(path, NULL); + if (!resolved) { + err(ctx, "%s: unable to determine subsys: %s\n", + devname, strerror(errno)); + return false; + } + + if (strcmp(resolved, "/sys/bus/dax") == 0) { + free(resolved); + return true; + } + + free(resolved); + return false; +} + static void *add_dax_dev(void *parent, int id, const char *daxdev_base) { const char *devname = devpath_to_devname(daxdev_base); @@ -559,6 +596,39 @@ static void dax_regions_init(struct daxctl_ctx *ctx) } } +static int is_enabled(const char *drvpath) +{ + struct stat st; + + if (lstat(drvpath, &st) < 0 || !S_ISLNK(st.st_mode)) + return 0; + else + return 1; +} + +DAXCTL_EXPORT int daxctl_dev_is_enabled(struct daxctl_dev *dev) +{ + struct daxctl_ctx *ctx = daxctl_dev_get_ctx(dev); + char *path = dev->dev_buf; + int len = dev->buf_len; + + if (!device_model_is_dax_bus(dev)) + return 1; + + if (snprintf(path, len, "%s/driver", dev->dev_path) >= len) { + err(ctx, "%s: buffer too small!\n", + daxctl_dev_get_devname(dev)); + return 0; + } + + return is_enabled(path); +} + +DAXCTL_EXPORT struct daxctl_ctx *daxctl_dev_get_ctx(struct daxctl_dev *dev) +{ + return dev->region->ctx; +} + DAXCTL_EXPORT struct daxctl_dev *daxctl_dev_get_first(struct daxctl_region *region) { dax_devices_init(region); diff --git a/daxctl/lib/libdaxctl.sym b/daxctl/lib/libdaxctl.sym index 84d3a69..c4af9a7 100644 --- a/daxctl/lib/libdaxctl.sym +++ b/daxctl/lib/libdaxctl.sym @@ -50,3 +50,9 @@ LIBDAXCTL_5 { global: daxctl_region_get_path; } LIBDAXCTL_4; + +LIBDAXCTL_6 { +global: + daxctl_dev_get_ctx; + daxctl_dev_is_enabled; +} LIBDAXCTL_5; diff --git a/daxctl/libdaxctl.h b/daxctl/libdaxctl.h index 1d13ea2..e20ccb4 100644 --- a/daxctl/libdaxctl.h +++ b/daxctl/libdaxctl.h @@ -67,6 +67,8 @@ const char *daxctl_dev_get_devname(struct daxctl_dev *dev); int daxctl_dev_get_major(struct daxctl_dev *dev); int daxctl_dev_get_minor(struct daxctl_dev *dev); unsigned long long daxctl_dev_get_size(struct daxctl_dev *dev); +struct daxctl_ctx *daxctl_dev_get_ctx(struct daxctl_dev *dev); +int daxctl_dev_is_enabled(struct daxctl_dev *dev); #define daxctl_dev_foreach(region, dev) \ for (dev = daxctl_dev_get_first(region); \ -- 2.20.1

Add an interface to retrieve the 'resource' attribute for a dax device. Attempt to retrieve it as usual via sysfs, but since older kernels may be missing this attribute, as a fallback, attempt to retrieve it from /proc/iomem Cc: Dan Williams <dan.j.williams(a)intel.com&gt; [fscanf format string problem and diagnosis] Reported-by: Tony Luck <tony.luck(a)intel.com&gt; Reviewed-by: Dan Williams <dan.j.williams(a)intel.com&gt; Signed-off-by: Vishal Verma <vishal.l.verma(a)intel.com&gt; --- Makefile.am | 3 ++- daxctl/lib/Makefile.am | 2 ++ daxctl/lib/libdaxctl-private.h | 1 + daxctl/lib/libdaxctl.c | 12 +++++++++++ daxctl/lib/libdaxctl.sym | 1 + daxctl/libdaxctl.h | 1 + util/iomem.c | 37 ++++++++++++++++++++++++++++++++++ util/iomem.h | 12 +++++++++++ 8 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 util/iomem.c create mode 100644 util/iomem.h diff --git a/Makefile.am b/Makefile.am index df8797e..8d10a10 100644 --- a/Makefile.am +++ b/Makefile.am @@ -74,6 +74,7 @@ libutil_a_SOURCES = \ util/wrapper.c \ util/filter.c \ util/bitmap.c \ - util/abspath.c + util/abspath.c \ + util/iomem.c nobase_include_HEADERS = daxctl/libdaxctl.h diff --git a/daxctl/lib/Makefile.am b/daxctl/lib/Makefile.am index 9f0e444..7704b1b 100644 --- a/daxctl/lib/Makefile.am +++ b/daxctl/lib/Makefile.am @@ -9,6 +9,8 @@ lib_LTLIBRARIES = libdaxctl.la libdaxctl_la_SOURCES =\ ../libdaxctl.h \ libdaxctl-private.h \ + ../../util/iomem.c \ + ../../util/iomem.h \ ../../util/sysfs.c \ ../../util/sysfs.h \ ../../util/log.c \ diff --git a/daxctl/lib/libdaxctl-private.h b/daxctl/lib/libdaxctl-private.h index 120137f..fee67d1 100644 --- a/daxctl/lib/libdaxctl-private.h +++ b/daxctl/lib/libdaxctl-private.h @@ -65,6 +65,7 @@ struct daxctl_dev { size_t buf_len; char *dev_path; struct list_node list; + unsigned long long resource; unsigned long long size; struct kmod_module *module; struct kmod_list *kmod_list; diff --git a/daxctl/lib/libdaxctl.c b/daxctl/lib/libdaxctl.c index caf661e..aa0d2f2 100644 --- a/daxctl/lib/libdaxctl.c +++ b/daxctl/lib/libdaxctl.c @@ -24,6 +24,7 @@ #include <util/log.h> #include <util/sysfs.h> +#include <util/iomem.h> #include <daxctl/libdaxctl.h> #include "libdaxctl-private.h" @@ -406,6 +407,12 @@ static void *add_dax_dev(void *parent, int id, const char *daxdev_base) dev->major = major(st.st_rdev); dev->minor = minor(st.st_rdev); + sprintf(path, "%s/resource", daxdev_base); + if (sysfs_read_attr(ctx, path, buf) == 0) + dev->resource = strtoull(buf, NULL, 0); + else + dev->resource = iomem_get_dev_resource(ctx, daxdev_base); + sprintf(path, "%s/size", daxdev_base); if (sysfs_read_attr(ctx, path, buf) < 0) goto err_read; @@ -928,6 +935,11 @@ DAXCTL_EXPORT int daxctl_dev_get_minor(struct daxctl_dev *dev) return dev->minor; } +DAXCTL_EXPORT unsigned long long daxctl_dev_get_resource(struct daxctl_dev *dev) +{ + return dev->resource; +} + DAXCTL_EXPORT unsigned long long daxctl_dev_get_size(struct daxctl_dev *dev) { return dev->size; diff --git a/daxctl/lib/libdaxctl.sym b/daxctl/lib/libdaxctl.sym index 19904a2..1692624 100644 --- a/daxctl/lib/libdaxctl.sym +++ b/daxctl/lib/libdaxctl.sym @@ -58,4 +58,5 @@ global: daxctl_dev_disable; daxctl_dev_enable_devdax; daxctl_dev_enable_ram; + daxctl_dev_get_resource; } LIBDAXCTL_5; diff --git a/daxctl/libdaxctl.h b/daxctl/libdaxctl.h index 407f459..adf55f3 100644 --- a/daxctl/libdaxctl.h +++ b/daxctl/libdaxctl.h @@ -66,6 +66,7 @@ int daxctl_dev_get_id(struct daxctl_dev *dev); const char *daxctl_dev_get_devname(struct daxctl_dev *dev); int daxctl_dev_get_major(struct daxctl_dev *dev); int daxctl_dev_get_minor(struct daxctl_dev *dev); +unsigned long long daxctl_dev_get_resource(struct daxctl_dev *dev); unsigned long long daxctl_dev_get_size(struct daxctl_dev *dev); struct daxctl_ctx *daxctl_dev_get_ctx(struct daxctl_dev *dev); int daxctl_dev_is_enabled(struct daxctl_dev *dev); diff --git a/util/iomem.c b/util/iomem.c new file mode 100644 index 0000000..a3c23f5 --- /dev/null +++ b/util/iomem.c @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2019 Intel Corporation. All rights reserved. */ + +#include <stdio.h> +#include <errno.h> +#include <string.h> +#include <util/log.h> +#include <util/iomem.h> +#include <util/sysfs.h> + +unsigned long long __iomem_get_dev_resource(struct log_ctx *ctx, + const char *devpath) +{ + const char *devname = devpath_to_devname(devpath); + FILE *fp = fopen("/proc/iomem", "r"); + unsigned long long res; + char name[256]; + + if (fp == NULL) { + log_err(ctx, "%s: open /proc/iomem: %s\n", devname, + strerror(errno)); + return 0; + } + + while (fscanf(fp, "%llx-%*x : %254[^\n]\n", &res, name) == 2) { + if (strcmp(name, devname) == 0) { + log_dbg(ctx, "%s: got resource via iomem: %#llx\n", + devname, res); + fclose(fp); + return res; + } + } + + log_dbg(ctx, "%s: not found in iomem\n", devname); + fclose(fp); + return 0; +} diff --git a/util/iomem.h b/util/iomem.h new file mode 100644 index 0000000..aaaf6a7 --- /dev/null +++ b/util/iomem.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2019 Intel Corporation. All rights reserved. */ +#ifndef _NDCTL_IOMEM_H_ +#define _NDCTL_IOMEM_H_ + +struct log_ctx; +unsigned long long __iomem_get_dev_resource(struct log_ctx *ctx, + const char *path); + +#define iomem_get_dev_resource(c, p) __iomem_get_dev_resource(&(c)->ctx, (p)) + +#endif /* _NDCTL_IOMEM_H_ */ -- 2.20.1

On Wed, 2019-07-31 at 18:29 -0600, Vishal Verma wrote: +++++++++++++++++++++++++++++++++ [..] *memblock) Static analysis complains that this can potentially cause a NULL dereference. Fix it by passing the mem object to memblock_in_dev(), since it has already been validated by that time. 3<---- From e8bf803e359b784259f645d1ff68e964b2c8618f Mon Sep 17 00:00:00 2001 From: Vishal Verma <vishal.l.verma(a)intel.com&gt; Date: Fri, 3 May 2019 13:27:35 -0600 Subject: [ndctl PATCH] libdaxctl: add a 'daxctl_memory' object for memory based operations Introduce a new 'daxctl_memory' object, which will be used for operations related to managing dax devices in 'system-memory' modes. Add libdaxctl APIs to get the target_node of a DAX device, and to online, offline, and query the state of hotplugged memory sections associated with a given device. This adds the following new interfaces: daxctl_dev_get_target_node daxctl_dev_get_memory daxctl_memory_get_dev daxctl_memory_get_node_path daxctl_memory_get_block_size daxctl_memory_online daxctl_memory_offline daxctl_memory_is_online daxctl_memory_num_sections Cc: Pavel Tatashin <pasha.tatashin(a)soleen.com&gt; Cc: Dave Hansen <dave.hansen(a)linux.intel.com&gt; Cc: Dan Williams <dan.j.williams(a)intel.com&gt; [for the memblock-already-online TOCTOU hole] Reported-by: Fan Du <fan.du(a)intel.com&gt; Tested-by: Fan Du <fan.du(a)intel.com&gt; Signed-off-by: Vishal Verma <vishal.l.verma(a)intel.com&gt; --- daxctl/lib/libdaxctl-private.h | 18 ++ daxctl/lib/libdaxctl.c | 384 +++++++++++++++++++++++++++++++++ daxctl/lib/libdaxctl.sym | 9 + daxctl/libdaxctl.h | 11 + 4 files changed, 422 insertions(+) diff --git a/daxctl/lib/libdaxctl-private.h b/daxctl/lib/libdaxctl- private.h index fee67d1..01091de 100644 --- a/daxctl/lib/libdaxctl-private.h +++ b/daxctl/lib/libdaxctl-private.h @@ -39,6 +39,13 @@ static const char *dax_modules[] = { [DAXCTL_DEV_MODE_RAM] = "kmem", }; +enum memory_op { + MEM_SET_OFFLINE, + MEM_SET_ONLINE, + MEM_IS_ONLINE, + MEM_COUNT, +}; + /** * struct daxctl_region - container for dax_devices */ @@ -70,8 +77,19 @@ struct daxctl_dev { struct kmod_module *module; struct kmod_list *kmod_list; struct daxctl_region *region; + struct daxctl_memory *mem; + int target_node; +}; + +struct daxctl_memory { + struct daxctl_dev *dev; + void *mem_buf; + size_t buf_len; + char *node_path; + unsigned long block_size; }; + static inline int check_kmod(struct kmod_ctx *kmod_ctx) { return kmod_ctx ? 0 : -ENXIO; diff --git a/daxctl/lib/libdaxctl.c b/daxctl/lib/libdaxctl.c index aa0d2f2..bcc77b6 100644 --- a/daxctl/lib/libdaxctl.c +++ b/daxctl/lib/libdaxctl.c @@ -200,6 +200,15 @@ DAXCTL_EXPORT void daxctl_region_get_uuid(struct daxctl_region *region, uuid_t u uuid_copy(uu, region->uuid); } +static void free_mem(struct daxctl_dev *dev) +{ + if (dev && dev->mem) { + free(dev->mem->node_path); + free(dev->mem); + dev->mem = NULL; + } +} + static void free_dev(struct daxctl_dev *dev, struct list_head *head) { if (head) @@ -207,6 +216,7 @@ static void free_dev(struct daxctl_dev *dev, struct list_head *head) kmod_module_unref_list(dev->kmod_list); free(dev->dev_buf); free(dev->dev_path); + free_mem(dev); free(dev); } @@ -380,6 +390,94 @@ static struct kmod_list *to_module_list(struct daxctl_ctx *ctx, return list; } +static int dev_is_system_ram_capable(struct daxctl_dev *dev) +{ + const char *devname = daxctl_dev_get_devname(dev); + struct daxctl_ctx *ctx = daxctl_dev_get_ctx(dev); + char *mod_path, *mod_base; + char path[200]; + const int len = sizeof(path); + + if (!device_model_is_dax_bus(dev)) + return false; + + if (!daxctl_dev_is_enabled(dev)) + return false; + + if (snprintf(path, len, "%s/driver/module", dev->dev_path) >= len) { + err(ctx, "%s: buffer too small!\n", devname); + return false; + } + + mod_path = realpath(path, NULL); + if (!mod_path) + return false; + + mod_base = basename(mod_path); + if (strcmp(mod_base, dax_modules[DAXCTL_DEV_MODE_RAM]) == 0) { + free(mod_path); + return true; + } + + free(mod_path); + return false; +} + +/* + * This checks for the device to be in system-ram mode, so calling + * daxctl_dev_get_memory() on a devdax mode device will always return NULL. + */ +static struct daxctl_memory *daxctl_dev_alloc_mem(struct daxctl_dev *dev) +{ + const char *size_path = "/sys/devices/system/memory/block_size_bytes"; + const char *node_base = "/sys/devices/system/node/node"; + const char *devname = daxctl_dev_get_devname(dev); + struct daxctl_ctx *ctx = daxctl_dev_get_ctx(dev); + struct daxctl_memory *mem; + char buf[SYSFS_ATTR_SIZE]; + int node_num; + + if (!dev_is_system_ram_capable(dev)) + return NULL; + + mem = calloc(1, sizeof(*mem)); + if (!mem) + return NULL; + + mem->dev = dev; + + if (sysfs_read_attr(ctx, size_path, buf) == 0) { + mem->block_size = strtoul(buf, NULL, 16); + if (mem->block_size == 0 || mem->block_size == ULONG_MAX) { + err(ctx, "%s: Unable to determine memblock size: %s\n", + devname, strerror(errno)); + mem->block_size = 0; + } + } + + node_num = daxctl_dev_get_target_node(dev); + if (node_num >= 0) { + if (asprintf(&mem->node_path, "%s%d", node_base, + node_num) < 0) { + err(ctx, "%s: Unable to set node_path\n", devname); + goto err_mem; + } + } + + mem->mem_buf = calloc(1, strlen(node_base) + 256); + if (!mem->mem_buf) + goto err_node; + mem->buf_len = strlen(node_base) + 256; + + return mem; + +err_node: + free(mem->node_path); +err_mem: + free(mem); + return NULL; +} + static void *add_dax_dev(void *parent, int id, const char *daxdev_base) { const char *devname = devpath_to_devname(daxdev_base); @@ -435,6 +533,12 @@ static void *add_dax_dev(void *parent, int id, const char *daxdev_base) if (rc == 0) dev->kmod_list = to_module_list(ctx, buf); + sprintf(path, "%s/target_node", daxdev_base); + if (sysfs_read_attr(ctx, path, buf) == 0) + dev->target_node = strtol(buf, NULL, 0); + else + dev->target_node = -1; + daxctl_dev_foreach(region, dev_dup) if (dev_dup->id == dev->id) { free_dev(dev, NULL); @@ -862,6 +966,9 @@ DAXCTL_EXPORT int daxctl_dev_disable(struct daxctl_dev *dev) if (!daxctl_dev_is_enabled(dev)) return 0; + /* If there is a memory object, first free that */ + free_mem(dev); + daxctl_unbind(ctx, dev->dev_path); if (daxctl_dev_is_enabled(dev)) { @@ -944,3 +1051,280 @@ DAXCTL_EXPORT unsigned long long daxctl_dev_get_size(struct daxctl_dev *dev) { return dev->size; } + +DAXCTL_EXPORT int daxctl_dev_get_target_node(struct daxctl_dev *dev) +{ + return dev->target_node; +} + +DAXCTL_EXPORT struct daxctl_memory *daxctl_dev_get_memory(struct daxctl_dev *dev) +{ + if (dev->mem) + return dev->mem; + else + return daxctl_dev_alloc_mem(dev); +} + +DAXCTL_EXPORT struct daxctl_dev *daxctl_memory_get_dev(struct daxctl_memory *mem) +{ + return mem->dev; +} + +DAXCTL_EXPORT const char *daxctl_memory_get_node_path(struct daxctl_memory *mem) +{ + return mem->node_path; +} + +DAXCTL_EXPORT unsigned long daxctl_memory_get_block_size(struct daxctl_memory *mem) +{ + return mem->block_size; +} + +static int online_one_memblock(struct daxctl_dev *dev, char *path) +{ + const char *devname = daxctl_dev_get_devname(dev); + struct daxctl_ctx *ctx = daxctl_dev_get_ctx(dev); + const char *mode = "online_movable"; + char buf[SYSFS_ATTR_SIZE]; + int rc; + + rc = sysfs_read_attr(ctx, path, buf); + if (rc) { + err(ctx, "%s: Failed to read %s: %s\n", + devname, path, strerror(-rc)); + return rc; + } + + /* + * if already online, possibly due to kernel config or a udev rule, + * there is nothing to do and we can skip over the memblock + */ + if (strncmp(buf, "online", 6) == 0) + return 1; + + rc = sysfs_write_attr_quiet(ctx, path, mode); + if (rc) { + /* + * While we performed an already-online check above, there + * is still a TOCTOU hole where someone (such as a udev rule) + * may have raced to online the memory. In such a case, + * the sysfs store will fail, however we can check for this + * by simply reading the state again. If it changed to the + * desired state, then we don't have to error out. + */ + if (sysfs_read_attr(ctx, path, buf) == 0) { + if (strncmp(buf, "online", 6) == 0) + return 1; + } + err(ctx, "%s: Failed to online %s: %s\n", + devname, path, strerror(-rc)); + } + return rc; +} + +static int offline_one_memblock(struct daxctl_dev *dev, char *path) +{ + const char *devname = daxctl_dev_get_devname(dev); + struct daxctl_ctx *ctx = daxctl_dev_get_ctx(dev); + const char *mode = "offline"; + char buf[SYSFS_ATTR_SIZE]; + int rc; + + rc = sysfs_read_attr(ctx, path, buf); + if (rc) { + err(ctx, "%s: Failed to read %s: %s\n", + devname, path, strerror(-rc)); + return rc; + } + + /* if already offline, there is nothing to do */ + if (strncmp(buf, "offline", 7) == 0) + return 1; + + rc = sysfs_write_attr_quiet(ctx, path, mode); + if (rc) { + /* Close the TOCTOU hole like in online_one_memblock() above */ + if (sysfs_read_attr(ctx, path, buf) == 0) { + if (strncmp(buf, "offline", 7) == 0) + return 1; + } + err(ctx, "%s: Failed to offline %s: %s\n", + devname, path, strerror(-rc)); + } + return rc; +} + +static int memblock_is_online(struct daxctl_dev *dev, char *path) +{ + const char *devname = daxctl_dev_get_devname(dev); + struct daxctl_ctx *ctx = daxctl_dev_get_ctx(dev); + char buf[SYSFS_ATTR_SIZE]; + int rc; + + rc = sysfs_read_attr(ctx, path, buf); + if (rc) { + err(ctx, "%s: Failed to read %s: %s\n", + devname, path, strerror(-rc)); + return rc; + } + + if (strncmp(buf, "online", 6) == 0) + return 1; + + /* offline */ + return 0; +} + +static bool memblock_in_dev(struct daxctl_memory *mem, const char *memblock) +{ + const char *mem_base = "/sys/devices/system/memory/"; + struct daxctl_dev *dev = daxctl_memory_get_dev(mem); + unsigned long long memblock_res, dev_start, dev_end; + const char *devname = daxctl_dev_get_devname(dev); + struct daxctl_ctx *ctx = daxctl_dev_get_ctx(dev); + unsigned long memblock_size; + int path_len = mem->buf_len; + char buf[SYSFS_ATTR_SIZE]; + unsigned long phys_index; + char *path = mem->mem_buf; + + if (snprintf(path, path_len, "%s/%s/phys_index", + mem_base, memblock) < 0) + return false; + + if (sysfs_read_attr(ctx, path, buf) == 0) { + phys_index = strtoul(buf, NULL, 16); + if (phys_index == 0 || phys_index == ULONG_MAX) { + err(ctx, "%s: %s: Unable to determine phys_index: %s\n", + devname, memblock, strerror(errno)); + return false; + } + } else { + err(ctx, "%s: %s: Unable to determine phys_index: %s\n", + devname, memblock, strerror(errno)); + return false; + } + + dev_start = daxctl_dev_get_resource(dev); + if (!dev_start) { + err(ctx, "%s: Unable to determine resource\n", devname); + return false; + } + dev_end = dev_start + daxctl_dev_get_size(dev); + + memblock_size = daxctl_memory_get_block_size(mem); + if (!memblock_size) { + err(ctx, "%s: Unable to determine memory block size\n", + devname); + return false; + } + memblock_res = phys_index * memblock_size; + + if (memblock_res >= dev_start && memblock_res <= dev_end) + return true; + + return false; +} + +static int op_for_one_memblock(struct daxctl_memory *mem, char *path, + enum memory_op op) +{ + struct daxctl_dev *dev = daxctl_memory_get_dev(mem); + const char *devname = daxctl_dev_get_devname(dev); + struct daxctl_ctx *ctx = daxctl_dev_get_ctx(dev); + int rc; + + switch (op) { + case MEM_SET_ONLINE: + return online_one_memblock(dev, path); + case MEM_SET_OFFLINE: + return offline_one_memblock(dev, path); + case MEM_IS_ONLINE: + rc = memblock_is_online(dev, path); + if (rc < 0) + return rc; + /* + * Retain the 'normal' semantics for if (memblock_is_online()), + * but since count needs rc == 0, we'll just flip rc for this op + */ + return !rc; + case MEM_COUNT: + return 0; + } + + err(ctx, "%s: BUG: unknown op: %d\n", devname, op); + return -EINVAL; +} + +static int daxctl_memory_op(struct daxctl_memory *mem, enum memory_op op) +{ + struct daxctl_dev *dev = daxctl_memory_get_dev(mem); + const char *devname = daxctl_dev_get_devname(dev); + struct daxctl_ctx *ctx = daxctl_dev_get_ctx(dev); + const char *node_path; + int rc, count = 0; + struct dirent *de; + DIR *node_dir; + + node_path = daxctl_memory_get_node_path(mem); + if (!node_path) { + err(ctx, "%s: Failed to get node_path\n", devname); + return -ENXIO; + } + + node_dir = opendir(node_path); + if (!node_dir) + return -errno; + + errno = 0; + while ((de = readdir(node_dir)) != NULL) { + char *path = mem->mem_buf; + int len = mem->buf_len; + + if (strncmp(de->d_name, "memory", 6) == 0) { + if (!memblock_in_dev(mem, de->d_name)) + continue; + rc = snprintf(path, len, "%s/%s/state", + node_path, de->d_name); + if (rc < 0) { + rc = -ENOMEM; + goto out_dir; + } + rc = op_for_one_memblock(mem, path, op); + if (rc < 0) + goto out_dir; + if (rc == 0) + count++; + } + errno = 0; + } + if (errno) { + rc = -errno; + goto out_dir; + } + rc = count; + +out_dir: + closedir(node_dir); + return rc; +} + +DAXCTL_EXPORT int daxctl_memory_online(struct daxctl_memory *mem) +{ + return daxctl_memory_op(mem, MEM_SET_ONLINE); +} + +DAXCTL_EXPORT int daxctl_memory_offline(struct daxctl_memory *mem) +{ + return daxctl_memory_op(mem, MEM_SET_OFFLINE); +} + +DAXCTL_EXPORT int daxctl_memory_is_online(struct daxctl_memory *mem) +{ + return daxctl_memory_op(mem, MEM_IS_ONLINE); +} + +DAXCTL_EXPORT int daxctl_memory_num_sections(struct daxctl_memory *mem) +{ + return daxctl_memory_op(mem, MEM_COUNT); +} diff --git a/daxctl/lib/libdaxctl.sym b/daxctl/lib/libdaxctl.sym index 1692624..bc18604 100644 --- a/daxctl/lib/libdaxctl.sym +++ b/daxctl/lib/libdaxctl.sym @@ -59,4 +59,13 @@ global: daxctl_dev_enable_devdax; daxctl_dev_enable_ram; daxctl_dev_get_resource; + daxctl_dev_get_target_node; + daxctl_dev_get_memory; + daxctl_memory_get_dev; + daxctl_memory_get_node_path; + daxctl_memory_get_block_size; + daxctl_memory_online; + daxctl_memory_offline; + daxctl_memory_is_online; + daxctl_memory_num_sections; } LIBDAXCTL_5; diff --git a/daxctl/libdaxctl.h b/daxctl/libdaxctl.h index adf55f3..fb6c3b1 100644 --- a/daxctl/libdaxctl.h +++ b/daxctl/libdaxctl.h @@ -73,6 +73,17 @@ int daxctl_dev_is_enabled(struct daxctl_dev *dev); int daxctl_dev_disable(struct daxctl_dev *dev); int daxctl_dev_enable_devdax(struct daxctl_dev *dev); int daxctl_dev_enable_ram(struct daxctl_dev *dev); +int daxctl_dev_get_target_node(struct daxctl_dev *dev); + +struct daxctl_memory; +struct daxctl_memory *daxctl_dev_get_memory(struct daxctl_dev *dev); +struct daxctl_dev *daxctl_memory_get_dev(struct daxctl_memory *mem); +const char *daxctl_memory_get_node_path(struct daxctl_memory *mem); +unsigned long daxctl_memory_get_block_size(struct daxctl_memory *mem); +int daxctl_memory_online(struct daxctl_memory *mem); +int daxctl_memory_offline(struct daxctl_memory *mem); +int daxctl_memory_is_online(struct daxctl_memory *mem); +int daxctl_memory_num_sections(struct daxctl_memory *mem); #define daxctl_dev_foreach(region, dev) \ for (dev = daxctl_dev_get_first(region); \ -- 2.20.1

In preparation for a reconfigure-device command, allow JSON listings to display the 'mode' of a dax device. This will allow the reconfigure-device command (and via daxctl_dev_to_json(), also daxctl-list) to print the mode in device listings via a 'daxctl-list' command or immediately after a mode change. Add a 'state' attribute to the json listings for devices, since a device could end up in a state where it is not bound to any driver, and hence, 'disabled'. The state attribute is only displayed for disabled devices. Cc: Dave Hansen <dave.hansen(a)linux.intel.com&gt; Cc: Dan Williams <dan.j.williams(a)intel.com&gt; Signed-off-by: Vishal Verma <vishal.l.verma(a)intel.com&gt; --- daxctl/lib/libdaxctl.c | 2 ++ util/json.c | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/daxctl/lib/libdaxctl.c b/daxctl/lib/libdaxctl.c index 949c56f..edb8257 100644 --- a/daxctl/lib/libdaxctl.c +++ b/daxctl/lib/libdaxctl.c @@ -12,6 +12,8 @@ */ #include <stdio.h> #include <errno.h> +#include <limits.h> +#include <libgen.h> #include <stdlib.h> #include <dirent.h> #include <unistd.h> diff --git a/util/json.c b/util/json.c index f521337..9f80b5b 100644 --- a/util/json.c +++ b/util/json.c @@ -269,6 +269,7 @@ struct json_object *util_dimm_to_json(struct ndctl_dimm *dimm, struct json_object *util_daxctl_dev_to_json(struct daxctl_dev *dev, unsigned long flags) { + struct daxctl_memory *mem = daxctl_dev_get_memory(dev); const char *devname = daxctl_dev_get_devname(dev); struct json_object *jdev, *jobj; int node; @@ -292,6 +293,19 @@ struct json_object *util_daxctl_dev_to_json(struct daxctl_dev *dev, json_object_object_add(jdev, "target_node", jobj); } + if (mem) + jobj = json_object_new_string("system-ram"); + else + jobj = json_object_new_string("devdax"); + if (jobj) + json_object_object_add(jdev, "mode", jobj); + + if (!daxctl_dev_is_enabled(dev)) { + jobj = json_object_new_string("disabled"); + if (jobj) + json_object_object_add(jdev, "state", jobj); + } + return jdev; } -- 2.20.1

Add a man page describing the new daxctl-reconfigure-device command. Cc: Pavel Tatashin <pasha.tatashin(a)soleen.com&gt; Cc: Dave Hansen <dave.hansen(a)linux.intel.com&gt; Cc: Dan Williams <dan.j.williams(a)intel.com&gt; Signed-off-by: Vishal Verma <vishal.l.verma(a)intel.com&gt; --- Documentation/daxctl/Makefile.am | 3 +- .../daxctl/daxctl-reconfigure-device.txt | 157 ++++++++++++++++++ 2 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 Documentation/daxctl/daxctl-reconfigure-device.txt diff --git a/Documentation/daxctl/Makefile.am b/Documentation/daxctl/Makefile.am index 6aba035..715fbad 100644 --- a/Documentation/daxctl/Makefile.am +++ b/Documentation/daxctl/Makefile.am @@ -28,7 +28,8 @@ endif man1_MANS = \ daxctl.1 \ daxctl-list.1 \ - daxctl-migrate-device-model.1 + daxctl-migrate-device-model.1 \ + daxctl-reconfigure-device.1 CLEANFILES = $(man1_MANS) diff --git a/Documentation/daxctl/daxctl-reconfigure-device.txt b/Documentation/daxctl/daxctl-reconfigure-device.txt new file mode 100644 index 0000000..196d692 --- /dev/null +++ b/Documentation/daxctl/daxctl-reconfigure-device.txt @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0 + +daxctl-reconfigure-device(1) +============================ + +NAME +---- +daxctl-reconfigure-device - Reconfigure a dax device into a different mode + +SYNOPSIS +-------- +[verse] +'daxctl reconfigure-device' <dax0.0> [<dax1.0>...<daxY.Z>] [<options>] + +EXAMPLES +-------- + +* Reconfigure dax0.0 to system-ram mode, don't online the memory +---- +# daxctl reconfigure-device --mode=system-ram --no-online dax0.0 +[ + { + "chardev":"dax0.0", + "size":16777216000, + "target_node":2, + "mode":"system-ram" + } +] +---- + +* Reconfigure dax0.0 to devdax mode, attempt to offline the memory +---- +# daxctl reconfigure-device --human --mode=devdax --force dax0.0 +{ + "chardev":"dax0.0", + "size":"15.63 GiB (16.78 GB)", + "target_node":2, + "mode":"devdax" +} +---- + +* Reconfigure all dax devices on region0 to system-ram mode +---- +# daxctl reconfigure-device --mode=system-ram --region=0 all +[ + { + "chardev":"dax0.0", + "size":16777216000, + "target_node":2, + "mode":"system-ram" + }, + { + "chardev":"dax0.1", + "size":16777216000, + "target_node":3, + "mode":"system-ram" + } +] +---- + +* Run a process called 'some-service' using numactl to restrict its cpu +nodes to '0' and '1', and memory allocations to node 2 (determined using +daxctl_dev_get_target_node() or 'daxctl list') +---- +# daxctl reconfigure-device --mode=system-ram dax0.0 +[ + { + "chardev":"dax0.0", + "size":16777216000, + "target_node":2, + "mode":"system-ram" + } +] + +# numactl --cpunodebind=0-1 --membind=2 -- some-service --opt1 --opt2 +---- + +DESCRIPTION +----------- + +Reconfigure the operational mode of a dax device. This can be used to convert +a regular 'devdax' mode device to the 'system-ram' mode which arranges for the +dax range to be hot-plugged into the system as regular memory. + +NOTE: This is a destructive operation. Any data on the dax device *will* be +lost. + +NOTE: Device reconfiguration depends on the dax-bus device model. See +linkdaxctl:daxctl-migrate-device-model[1] for more information. If dax-class is +in use (via the dax_pmem_compat driver), the reconfiguration will fail with an +error such as the following: +---- +# daxctl reconfigure-device --mode=system-ram --region=0 all +libdaxctl: daxctl_dev_disable: dax3.0: error: device model is dax-class +dax3.0: disable failed: Operation not supported +error reconfiguring devices: Operation not supported +reconfigured 0 devices +---- + +OPTIONS +------- +-r:: +--region=:: + Restrict the operation to devices belonging to the specified region(s). + A device-dax region is a contiguous range of memory that hosts one or + more /dev/daxX.Y devices, where X is the region id and Y is the device + instance id. + +-m:: +--mode=:: + Specify the mode to which the dax device(s) should be reconfigured. + - "system-ram": hotplug the device into system memory. + + - "devdax": switch to the normal "device dax" mode. This requires the + kernel to support hot-unplugging 'kmem' based memory. If this is not + available, a reboot is the only way to switch back to 'devdax' mode. + +-N:: +--no-online:: + By default, memory sections provided by system-ram devices will be + brought online automatically and immediately with the 'online_movable' + policy. Use this option to disable the automatic onlining behavior. + + NOTE: While this option prevents daxctl from automatically onlining + the memory sections, there may be other agents, notably system udev + rules, that online new memory sections as they appear. Coordinating + with such rules is out of scope of this utility, and the system + administrator is expected to remove them if they are undesirable. + If such an agent races to online memory sections, daxctl is prepared + to lose the race, and not fail the onlining operation as it only + cares that the memory section was onlined, not that it was the one + to do so. + +-f:: +--force:: + When converting from "system-ram" mode to "devdax", it is expected + that all the memory sections are first made offline. By default, + daxctl won't touch online memory. However with this option, attempt + to offline the memory on the NUMA node associated with the dax device + before converting it back to "devdax" mode. + +-u:: +--human:: + By default the command will output machine-friendly raw-integer + data. Instead, with this flag, numbers representing storage size + will be formatted as human readable strings with units, other + fields are converted to hexadecimal strings. + +-v:: +--verbose:: + Emit more debug messages + +include::../copyright.txt[] + +SEE ALSO +-------- +linkdaxctl:daxctl-list[1],daxctl-migrate-device-model[1] -- 2.20.1

Add man pages for the two new commands: daxctl-online-memory, and daxctl-offline-memory. Cc: Dan Williams <dan.j.williams(a)intel.com&gt; Cc: Dave Hansen <dave.hansen(a)linux.intel.com&gt; Reviewed-by: Dan Williams <dan.j.williams(a)intel.com&gt; Signed-off-by: Vishal Verma <vishal.l.verma(a)intel.com&gt; --- Documentation/daxctl/Makefile.am | 4 +- .../daxctl/daxctl-offline-memory.txt | 72 +++++++++++++++++ Documentation/daxctl/daxctl-online-memory.txt | 80 +++++++++++++++++++ 3 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 Documentation/daxctl/daxctl-offline-memory.txt create mode 100644 Documentation/daxctl/daxctl-online-memory.txt diff --git a/Documentation/daxctl/Makefile.am b/Documentation/daxctl/Makefile.am index 715fbad..37c3bde 100644 --- a/Documentation/daxctl/Makefile.am +++ b/Documentation/daxctl/Makefile.am @@ -29,7 +29,9 @@ man1_MANS = \ daxctl.1 \ daxctl-list.1 \ daxctl-migrate-device-model.1 \ - daxctl-reconfigure-device.1 + daxctl-reconfigure-device.1 \ + daxctl-online-memory.1 \ + daxctl-offline-memory.1 CLEANFILES = $(man1_MANS) diff --git a/Documentation/daxctl/daxctl-offline-memory.txt b/Documentation/daxctl/daxctl-offline-memory.txt new file mode 100644 index 0000000..ba06287 --- /dev/null +++ b/Documentation/daxctl/daxctl-offline-memory.txt @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 + +daxctl-offline-memory(1) +======================== + +NAME +---- +daxctl-offline-memory - Offline the memory for a device that is in system-ram mode + +SYNOPSIS +-------- +[verse] +'daxctl offline-memory' <dax0.0> [<dax1.0>...<daxY.Z>] [<options>] + +EXAMPLES +-------- + +* Reconfigure dax0.0 to system-ram mode +---- +# daxctl reconfigure-device --mode=system-ram --human dax0.0 +{ + "chardev":"dax0.0", + "size":"7.87 GiB (8.45 GB)", + "target_node":2, + "mode":"system-ram" +} +---- + +* Offline the memory +---- +# daxctl offline-memory dax0.0 +dax0.0: 62 sections offlined +offlined memory for 1 device +---- + +DESCRIPTION +----------- + +Offline the memory sections associated with a device that has been converted +to the system-ram mode. If one or more blocks are already offline, attempt to +offline the remaining blocks. If all blocks were already offline, print a +message and return success without actually doing anything. + +This is complementary to the 'daxctl-online-memory' command, and may be used +when it is wished to offline the memory sections, but not convert the device +back to 'devdax' mode. + +OPTIONS +------- +-r:: +--region=:: + Restrict the operation to devices belonging to the specified region(s). + A device-dax region is a contiguous range of memory that hosts one or + more /dev/daxX.Y devices, where X is the region id and Y is the device + instance id. + +-u:: +--human:: + By default the command will output machine-friendly raw-integer + data. Instead, with this flag, numbers representing storage size + will be formatted as human readable strings with units, other + fields are converted to hexadecimal strings. + +-v:: +--verbose:: + Emit more debug messages + +include::../copyright.txt[] + +SEE ALSO +-------- +linkdaxctl:daxctl-reconfigure-device[1],daxctl-online-memory[1] diff --git a/Documentation/daxctl/daxctl-online-memory.txt b/Documentation/daxctl/daxctl-online-memory.txt new file mode 100644 index 0000000..5ac1cbf --- /dev/null +++ b/Documentation/daxctl/daxctl-online-memory.txt @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 + +daxctl-online-memory(1) +======================= + +NAME +---- +daxctl-online-memory - Online the memory for a device that is in system-ram mode + +SYNOPSIS +-------- +[verse] +'daxctl online-memory' <dax0.0> [<dax1.0>...<daxY.Z>] [<options>] + +EXAMPLES +-------- + +* Reconfigure dax0.0 to system-ram mode, don't online the memory +---- +# daxctl reconfigure-device --mode=system-ram --no-online --human dax0.0 +{ + "chardev":"dax0.0", + "size":"7.87 GiB (8.45 GB)", + "target_node":2, + "mode":"system-ram" +} +---- + +* Online the memory separately +---- +# daxctl online-memory dax0.0 +dax0.0: 62 new sections onlined +onlined memory for 1 device +---- + +* Onlining memory when some sections were already online +---- +# daxctl online-memory dax0.0 +dax0.0: 1 section already online +dax0.0: 61 new sections onlined +onlined memory for 1 device +---- + +DESCRIPTION +----------- + +Online the memory sections associated with a device that has been converted +to the system-ram mode. If one or more blocks are already online, print a +message about them, and attempt to online the remaining blocks. + +This is complementary to the 'daxctl-reconfigure-device' command, when used with +the '--no-online' option to skip onlining memory sections immediately after the +reconfigure. In these scenarios, the memory can be onlined at a later time using +'daxctl-online-memory'. + +OPTIONS +------- +-r:: +--region=:: + Restrict the operation to devices belonging to the specified region(s). + A device-dax region is a contiguous range of memory that hosts one or + more /dev/daxX.Y devices, where X is the region id and Y is the device + instance id. + +-u:: +--human:: + By default the command will output machine-friendly raw-integer + data. Instead, with this flag, numbers representing storage size + will be formatted as human readable strings with units, other + fields are converted to hexadecimal strings. + +-v:: +--verbose:: + Emit more debug messages + +include::../copyright.txt[] + +SEE ALSO +-------- +linkdaxctl:daxctl-reconfigure-device[1],daxctl-offline-memory[1] -- 2.20.1

Add bash completion helpers for the new daxctl-reconfigure-device, daxctl-online-memory, and daxctl-offline-memory commands. Cc: Dan Williams <dan.j.williams(a)intel.com&gt; Reviewed-by: Dan Williams <dan.j.williams(a)intel.com&gt; Signed-off-by: Vishal Verma <vishal.l.verma(a)intel.com&gt; --- contrib/ndctl | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/contrib/ndctl b/contrib/ndctl index cacee2d..680fe6a 100755 --- a/contrib/ndctl +++ b/contrib/ndctl @@ -547,7 +547,7 @@ __daxctlcomp() COMPREPLY=( $( compgen -W "$1" -- "$2" ) ) for cword in "${COMPREPLY[@]}"; do - if [[ "$cword" == @(--region|--dev) ]]; then + if [[ "$cword" == @(--region|--dev|--mode) ]]; then COMPREPLY[$i]="${cword}=" else COMPREPLY[$i]="${cword} " @@ -572,6 +572,9 @@ __daxctl_comp_options() --dev) opts="$(__daxctl_get_devs -i)" ;; + --mode) + opts="system-ram devdax" + ;; *) return ;; @@ -582,8 +585,23 @@ __daxctl_comp_options() __daxctl_comp_non_option_args() { - # there aren't any commands that accept non option arguments yet - return + local subcmd=$1 + local cur=$2 + local opts + + case $subcmd in + reconfigure-device) + ;& + online-memory) + ;& + offline-memory) + opts="$(__daxctl_get_devs -i) all" + ;; + *) + return + ;; + esac + __daxctlcomp "$opts" "$cur" } __daxctl_main() -- 2.20.1