[linux-block.git] / arch / s390 / kernel / topology.c

// SPDX-License-Identifier: GPL-2.0
/*
 *    Copyright IBM Corp. 2007, 2011
 */

#define KMSG_COMPONENT "cpu"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/workqueue.h>
#include <linux/memblock.h>
#include <linux/uaccess.h>
#include <linux/sysctl.h>
#include <linux/cpuset.h>
#include <linux/device.h>
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/sched/topology.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/cpu.h>
#include <linux/smp.h>
#include <linux/mm.h>
#include <linux/nodemask.h>
#include <linux/node.h>
#include <asm/sysinfo.h>

#define PTF_HORIZONTAL	(0UL)
#define PTF_VERTICAL	(1UL)
#define PTF_CHECK	(2UL)

enum {
	TOPOLOGY_MODE_HW,
	TOPOLOGY_MODE_SINGLE,
	TOPOLOGY_MODE_PACKAGE,
	TOPOLOGY_MODE_UNINITIALIZED
};

struct mask_info {
	struct mask_info *next;
	unsigned char id;
	cpumask_t mask;
};

static int topology_mode = TOPOLOGY_MODE_UNINITIALIZED;
static void set_topology_timer(void);
static void topology_work_fn(struct work_struct *work);
static struct sysinfo_15_1_x *tl_info;

static DECLARE_WORK(topology_work, topology_work_fn);

/*
 * Socket/Book linked lists and cpu_topology updates are
 * protected by "sched_domains_mutex".
 */
static struct mask_info socket_info;
static struct mask_info book_info;
static struct mask_info drawer_info;

struct cpu_topology_s390 cpu_topology[NR_CPUS];
EXPORT_SYMBOL_GPL(cpu_topology);

static void cpu_group_map(cpumask_t *dst, struct mask_info *info, unsigned int cpu)
{
	static cpumask_t mask;

	cpumask_clear(&mask);
	if (!cpumask_test_cpu(cpu, &cpu_setup_mask))
		goto out;
	cpumask_set_cpu(cpu, &mask);
	switch (topology_mode) {
	case TOPOLOGY_MODE_HW:
		while (info) {
			if (cpumask_test_cpu(cpu, &info->mask)) {
				cpumask_copy(&mask, &info->mask);
				break;
			}
			info = info->next;
		}
		break;
	case TOPOLOGY_MODE_PACKAGE:
		cpumask_copy(&mask, cpu_present_mask);
		break;
	default:
		fallthrough;
	case TOPOLOGY_MODE_SINGLE:
		break;
	}
	cpumask_and(&mask, &mask, &cpu_setup_mask);
out:
	cpumask_copy(dst, &mask);
}

static void cpu_thread_map(cpumask_t *dst, unsigned int cpu)
{
	static cpumask_t mask;
	int i;

	cpumask_clear(&mask);
	if (!cpumask_test_cpu(cpu, &cpu_setup_mask))
		goto out;
	cpumask_set_cpu(cpu, &mask);
	if (topology_mode != TOPOLOGY_MODE_HW)
		goto out;
	cpu -= cpu % (smp_cpu_mtid + 1);
	for (i = 0; i <= smp_cpu_mtid; i++) {
		if (cpumask_test_cpu(cpu + i, &cpu_setup_mask))
			cpumask_set_cpu(cpu + i, &mask);
	}
out:
	cpumask_copy(dst, &mask);
}

#define TOPOLOGY_CORE_BITS	64

static void add_cpus_to_mask(struct topology_core *tl_core,
			     struct mask_info *drawer,
			     struct mask_info *book,
			     struct mask_info *socket)
{
	struct cpu_topology_s390 *topo;
	unsigned int core;

	for_each_set_bit(core, &tl_core->mask, TOPOLOGY_CORE_BITS) {
		unsigned int rcore;
		int lcpu, i;

		rcore = TOPOLOGY_CORE_BITS - 1 - core + tl_core->origin;
		lcpu = smp_find_processor_id(rcore << smp_cpu_mt_shift);
		if (lcpu < 0)
			continue;
		for (i = 0; i <= smp_cpu_mtid; i++) {
			topo = &cpu_topology[lcpu + i];
			topo->drawer_id = drawer->id;
			topo->book_id = book->id;
			topo->socket_id = socket->id;
			topo->core_id = rcore;
			topo->thread_id = lcpu + i;
			topo->dedicated = tl_core->d;
			cpumask_set_cpu(lcpu + i, &drawer->mask);
			cpumask_set_cpu(lcpu + i, &book->mask);
			cpumask_set_cpu(lcpu + i, &socket->mask);
			smp_cpu_set_polarization(lcpu + i, tl_core->pp);
		}
	}
}

static void clear_masks(void)
{
	struct mask_info *info;

	info = &socket_info;
	while (info) {
		cpumask_clear(&info->mask);
		info = info->next;
	}
	info = &book_info;
	while (info) {
		cpumask_clear(&info->mask);
		info = info->next;
	}
	info = &drawer_info;
	while (info) {
		cpumask_clear(&info->mask);
		info = info->next;
	}
}

static union topology_entry *next_tle(union topology_entry *tle)
{
	if (!tle->nl)
		return (union topology_entry *)((struct topology_core *)tle + 1);
	return (union topology_entry *)((struct topology_container *)tle + 1);
}

static void tl_to_masks(struct sysinfo_15_1_x *info)
{
	struct mask_info *socket = &socket_info;
	struct mask_info *book = &book_info;
	struct mask_info *drawer = &drawer_info;
	union topology_entry *tle, *end;

	clear_masks();
	tle = info->tle;
	end = (union topology_entry *)((unsigned long)info + info->length);
	while (tle < end) {
		switch (tle->nl) {
		case 3:
			drawer = drawer->next;
			drawer->id = tle->container.id;
			break;
		case 2:
			book = book->next;
			book->id = tle->container.id;
			break;
		case 1:
			socket = socket->next;
			socket->id = tle->container.id;
			break;
		case 0:
			add_cpus_to_mask(&tle->cpu, drawer, book, socket);
			break;
		default:
			clear_masks();
			return;
		}
		tle = next_tle(tle);
	}
}

static void topology_update_polarization_simple(void)
{
	int cpu;

	for_each_possible_cpu(cpu)
		smp_cpu_set_polarization(cpu, POLARIZATION_HRZ);
}

static int ptf(unsigned long fc)
{
	int rc;

	asm volatile(
		"	.insn	rre,0xb9a20000,%1,%1\n"
		"	ipm	%0\n"
		"	srl	%0,28\n"
		: "=d" (rc)
		: "d" (fc)  : "cc");
	return rc;
}

int topology_set_cpu_management(int fc)
{
	int cpu, rc;

	if (!MACHINE_HAS_TOPOLOGY)
		return -EOPNOTSUPP;
	if (fc)
		rc = ptf(PTF_VERTICAL);
	else
		rc = ptf(PTF_HORIZONTAL);
	if (rc)
		return -EBUSY;
	for_each_possible_cpu(cpu)
		smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
	return rc;
}

void update_cpu_masks(void)
{
	struct cpu_topology_s390 *topo, *topo_package, *topo_sibling;
	int cpu, sibling, pkg_first, smt_first, id;

	for_each_possible_cpu(cpu) {
		topo = &cpu_topology[cpu];
		cpu_thread_map(&topo->thread_mask, cpu);
		cpu_group_map(&topo->core_mask, &socket_info, cpu);
		cpu_group_map(&topo->book_mask, &book_info, cpu);
		cpu_group_map(&topo->drawer_mask, &drawer_info, cpu);
		topo->booted_cores = 0;
		if (topology_mode != TOPOLOGY_MODE_HW) {
			id = topology_mode == TOPOLOGY_MODE_PACKAGE ? 0 : cpu;
			topo->thread_id = cpu;
			topo->core_id = cpu;
			topo->socket_id = id;
			topo->book_id = id;
			topo->drawer_id = id;
		}
	}
	for_each_online_cpu(cpu) {
		topo = &cpu_topology[cpu];
		pkg_first = cpumask_first(&topo->core_mask);
		topo_package = &cpu_topology[pkg_first];
		if (cpu == pkg_first) {
			for_each_cpu(sibling, &topo->core_mask) {
				topo_sibling = &cpu_topology[sibling];
				smt_first = cpumask_first(&topo_sibling->thread_mask);
				if (sibling == smt_first)
					topo_package->booted_cores++;
			}
		} else {
			topo->booted_cores = topo_package->booted_cores;
		}
	}
}

void store_topology(struct sysinfo_15_1_x *info)
{
	stsi(info, 15, 1, topology_mnest_limit());
}

static void __arch_update_dedicated_flag(void *arg)
{
	if (topology_cpu_dedicated(smp_processor_id()))
		set_cpu_flag(CIF_DEDICATED_CPU);
	else
		clear_cpu_flag(CIF_DEDICATED_CPU);
}

static int __arch_update_cpu_topology(void)
{
	struct sysinfo_15_1_x *info = tl_info;
	int rc = 0;

	mutex_lock(&smp_cpu_state_mutex);
	if (MACHINE_HAS_TOPOLOGY) {
		rc = 1;
		store_topology(info);
		tl_to_masks(info);
	}
	update_cpu_masks();
	if (!MACHINE_HAS_TOPOLOGY)
		topology_update_polarization_simple();
	mutex_unlock(&smp_cpu_state_mutex);
	return rc;
}

int arch_update_cpu_topology(void)
{
	struct device *dev;
	int cpu, rc;

	rc = __arch_update_cpu_topology();
	on_each_cpu(__arch_update_dedicated_flag, NULL, 0);
	for_each_online_cpu(cpu) {
		dev = get_cpu_device(cpu);
		if (dev)
			kobject_uevent(&dev->kobj, KOBJ_CHANGE);
	}
	return rc;
}

static void topology_work_fn(struct work_struct *work)
{
	rebuild_sched_domains();
}

void topology_schedule_update(void)
{
	schedule_work(&topology_work);
}

static void topology_flush_work(void)
{
	flush_work(&topology_work);
}

static void topology_timer_fn(struct timer_list *unused)
{
	if (ptf(PTF_CHECK))
		topology_schedule_update();
	set_topology_timer();
}

static struct timer_list topology_timer;

static atomic_t topology_poll = ATOMIC_INIT(0);

static void set_topology_timer(void)
{
	if (atomic_add_unless(&topology_poll, -1, 0))
		mod_timer(&topology_timer, jiffies + msecs_to_jiffies(100));
	else
		mod_timer(&topology_timer, jiffies + msecs_to_jiffies(60 * MSEC_PER_SEC));
}

void topology_expect_change(void)
{
	if (!MACHINE_HAS_TOPOLOGY)
		return;
	/* This is racy, but it doesn't matter since it is just a heuristic.
	 * Worst case is that we poll in a higher frequency for a bit longer.
	 */
	if (atomic_read(&topology_poll) > 60)
		return;
	atomic_add(60, &topology_poll);
	set_topology_timer();
}

static int cpu_management;

static ssize_t dispatching_show(struct device *dev,
				struct device_attribute *attr,
				char *buf)
{
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	count = sprintf(buf, "%d\n", cpu_management);
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}

static ssize_t dispatching_store(struct device *dev,
				 struct device_attribute *attr,
				 const char *buf,
				 size_t count)
{
	int val, rc;
	char delim;

	if (sscanf(buf, "%d %c", &val, &delim) != 1)
		return -EINVAL;
	if (val != 0 && val != 1)
		return -EINVAL;
	rc = 0;
	cpus_read_lock();
	mutex_lock(&smp_cpu_state_mutex);
	if (cpu_management == val)
		goto out;
	rc = topology_set_cpu_management(val);
	if (rc)
		goto out;
	cpu_management = val;
	topology_expect_change();
out:
	mutex_unlock(&smp_cpu_state_mutex);
	cpus_read_unlock();
	return rc ? rc : count;
}
static DEVICE_ATTR_RW(dispatching);

static ssize_t cpu_polarization_show(struct device *dev,
				     struct device_attribute *attr, char *buf)
{
	int cpu = dev->id;
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	switch (smp_cpu_get_polarization(cpu)) {
	case POLARIZATION_HRZ:
		count = sprintf(buf, "horizontal\n");
		break;
	case POLARIZATION_VL:
		count = sprintf(buf, "vertical:low\n");
		break;
	case POLARIZATION_VM:
		count = sprintf(buf, "vertical:medium\n");
		break;
	case POLARIZATION_VH:
		count = sprintf(buf, "vertical:high\n");
		break;
	default:
		count = sprintf(buf, "unknown\n");
		break;
	}
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}
static DEVICE_ATTR(polarization, 0444, cpu_polarization_show, NULL);

static struct attribute *topology_cpu_attrs[] = {
	&dev_attr_polarization.attr,
	NULL,
};

static struct attribute_group topology_cpu_attr_group = {
	.attrs = topology_cpu_attrs,
};

static ssize_t cpu_dedicated_show(struct device *dev,
				  struct device_attribute *attr, char *buf)
{
	int cpu = dev->id;
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	count = sprintf(buf, "%d\n", topology_cpu_dedicated(cpu));
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}
static DEVICE_ATTR(dedicated, 0444, cpu_dedicated_show, NULL);

static struct attribute *topology_extra_cpu_attrs[] = {
	&dev_attr_dedicated.attr,
	NULL,
};

static struct attribute_group topology_extra_cpu_attr_group = {
	.attrs = topology_extra_cpu_attrs,
};

int topology_cpu_init(struct cpu *cpu)
{
	int rc;

	rc = sysfs_create_group(&cpu->dev.kobj, &topology_cpu_attr_group);
	if (rc || !MACHINE_HAS_TOPOLOGY)
		return rc;
	rc = sysfs_create_group(&cpu->dev.kobj, &topology_extra_cpu_attr_group);
	if (rc)
		sysfs_remove_group(&cpu->dev.kobj, &topology_cpu_attr_group);
	return rc;
}

static const struct cpumask *cpu_thread_mask(int cpu)
{
	return &cpu_topology[cpu].thread_mask;
}


const struct cpumask *cpu_coregroup_mask(int cpu)
{
	return &cpu_topology[cpu].core_mask;
}

static const struct cpumask *cpu_book_mask(int cpu)
{
	return &cpu_topology[cpu].book_mask;
}

static const struct cpumask *cpu_drawer_mask(int cpu)
{
	return &cpu_topology[cpu].drawer_mask;
}

static struct sched_domain_topology_level s390_topology[] = {
	{ cpu_thread_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
	{ cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
	{ cpu_book_mask, SD_INIT_NAME(BOOK) },
	{ cpu_drawer_mask, SD_INIT_NAME(DRAWER) },
	{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
	{ NULL, },
};

static void __init alloc_masks(struct sysinfo_15_1_x *info,
			       struct mask_info *mask, int offset)
{
	int i, nr_masks;

	nr_masks = info->mag[TOPOLOGY_NR_MAG - offset];
	for (i = 0; i < info->mnest - offset; i++)
		nr_masks *= info->mag[TOPOLOGY_NR_MAG - offset - 1 - i];
	nr_masks = max(nr_masks, 1);
	for (i = 0; i < nr_masks; i++) {
		mask->next = memblock_alloc(sizeof(*mask->next), 8);
		if (!mask->next)
			panic("%s: Failed to allocate %zu bytes align=0x%x\n",
			      __func__, sizeof(*mask->next), 8);
		mask = mask->next;
	}
}

void __init topology_init_early(void)
{
	struct sysinfo_15_1_x *info;

	set_sched_topology(s390_topology);
	if (topology_mode == TOPOLOGY_MODE_UNINITIALIZED) {
		if (MACHINE_HAS_TOPOLOGY)
			topology_mode = TOPOLOGY_MODE_HW;
		else
			topology_mode = TOPOLOGY_MODE_SINGLE;
	}
	if (!MACHINE_HAS_TOPOLOGY)
		goto out;
	tl_info = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
	if (!tl_info)
		panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
		      __func__, PAGE_SIZE, PAGE_SIZE);
	info = tl_info;
	store_topology(info);
	pr_info("The CPU configuration topology of the machine is: %d %d %d %d %d %d / %d\n",
		info->mag[0], info->mag[1], info->mag[2], info->mag[3],
		info->mag[4], info->mag[5], info->mnest);
	alloc_masks(info, &socket_info, 1);
	alloc_masks(info, &book_info, 2);
	alloc_masks(info, &drawer_info, 3);
out:
	cpumask_set_cpu(0, &cpu_setup_mask);
	__arch_update_cpu_topology();
	__arch_update_dedicated_flag(NULL);
}

static inline int topology_get_mode(int enabled)
{
	if (!enabled)
		return TOPOLOGY_MODE_SINGLE;
	return MACHINE_HAS_TOPOLOGY ? TOPOLOGY_MODE_HW : TOPOLOGY_MODE_PACKAGE;
}

static inline int topology_is_enabled(void)
{
	return topology_mode != TOPOLOGY_MODE_SINGLE;
}

static int __init topology_setup(char *str)
{
	bool enabled;
	int rc;

	rc = kstrtobool(str, &enabled);
	if (rc)
		return rc;
	topology_mode = topology_get_mode(enabled);
	return 0;
}
early_param("topology", topology_setup);

static int topology_ctl_handler(struct ctl_table *ctl, int write,
				void *buffer, size_t *lenp, loff_t *ppos)
{
	int enabled = topology_is_enabled();
	int new_mode;
	int rc;
	struct ctl_table ctl_entry = {
		.procname	= ctl->procname,
		.data		= &enabled,
		.maxlen		= sizeof(int),
		.extra1		= SYSCTL_ZERO,
		.extra2		= SYSCTL_ONE,
	};

	rc = proc_douintvec_minmax(&ctl_entry, write, buffer, lenp, ppos);
	if (rc < 0 || !write)
		return rc;

	mutex_lock(&smp_cpu_state_mutex);
	new_mode = topology_get_mode(enabled);
	if (topology_mode != new_mode) {
		topology_mode = new_mode;
		topology_schedule_update();
	}
	mutex_unlock(&smp_cpu_state_mutex);
	topology_flush_work();

	return rc;
}

static struct ctl_table topology_ctl_table[] = {
	{
		.procname	= "topology",
		.mode		= 0644,
		.proc_handler	= topology_ctl_handler,
	},
	{ },
};

static struct ctl_table topology_dir_table[] = {
	{
		.procname	= "s390",
		.maxlen		= 0,
		.mode		= 0555,
		.child		= topology_ctl_table,
	},
	{ },
};

static int __init topology_init(void)
{
	timer_setup(&topology_timer, topology_timer_fn, TIMER_DEFERRABLE);
	if (MACHINE_HAS_TOPOLOGY)
		set_topology_timer();
	else
		topology_update_polarization_simple();
	register_sysctl_table(topology_dir_table);
	return device_create_file(cpu_subsys.dev_root, &dev_attr_dispatching);
}
device_initcall(topology_init);
Commit	Line	Data
a17ae4c3	1	// SPDX-License-Identifier: GPL-2.0
dbd70fb4	2	/*
a53c8fab	3	* Copyright IBM Corp. 2007, 2011
dbd70fb4 HC	4	*/
dbd70fb4 HC	5
395d31d4 MS	6	#define KMSG_COMPONENT "cpu"
	7	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
	8
83a24e32	9	#include <linux/workqueue.h>
57c8a661	10	#include <linux/memblock.h>
51dce386 HC	11	#include <linux/uaccess.h>
51dce386 HC	12	#include <linux/sysctl.h>
83a24e32 HC	13	#include <linux/cpuset.h>
83a24e32 HC	14	#include <linux/device.h>
80020fbd	15	#include <linux/export.h>
83a24e32	16	#include <linux/kernel.h>
dbd70fb4	17	#include <linux/sched.h>
105ab3d8	18	#include <linux/sched/topology.h>
83a24e32	19	#include <linux/delay.h>
d05d15da HC	20	#include <linux/init.h>
d05d15da HC	21	#include <linux/slab.h>
dbd70fb4 HC	22	#include <linux/cpu.h>
dbd70fb4 HC	23	#include <linux/smp.h>
83a24e32	24	#include <linux/mm.h>
3a368f74 PH	25	#include <linux/nodemask.h>
3a368f74 PH	26	#include <linux/node.h>
78609132	27	#include <asm/sysinfo.h>
dbd70fb4	28
c10fde0d HC	29	#define PTF_HORIZONTAL (0UL)
	30	#define PTF_VERTICAL (1UL)
	31	#define PTF_CHECK (2UL)
dbd70fb4	32
1b25fda0 HC	33	enum {
	34	TOPOLOGY_MODE_HW,
	35	TOPOLOGY_MODE_SINGLE,
	36	TOPOLOGY_MODE_PACKAGE,
	37	TOPOLOGY_MODE_UNINITIALIZED
	38	};
	39
4cb14bc8 HC	40	struct mask_info {
4cb14bc8 HC	41	struct mask_info *next;
10d38589	42	unsigned char id;
dbd70fb4 HC	43	cpumask_t mask;
	44	};
	45
1b25fda0	46	static int topology_mode = TOPOLOGY_MODE_UNINITIALIZED;
d1e57508	47	static void set_topology_timer(void);
dbd70fb4	48	static void topology_work_fn(struct work_struct *work);
c30f91b6	49	static struct sysinfo_15_1_x *tl_info;
dbd70fb4	50
d1e57508	51	static DECLARE_WORK(topology_work, topology_work_fn);
d00aa4e7	52
3a3814c2	53	/*
30fc4ca2	54	* Socket/Book linked lists and cpu_topology updates are
3a3814c2 MH	55	* protected by "sched_domains_mutex".
3a3814c2 MH	56	*/
d1e57508	57	static struct mask_info socket_info;
4cb14bc8	58	static struct mask_info book_info;
adac0f1e	59	static struct mask_info drawer_info;
d1e57508	60
30fc4ca2 HC	61	struct cpu_topology_s390 cpu_topology[NR_CPUS];
30fc4ca2 HC	62	EXPORT_SYMBOL_GPL(cpu_topology);
83a24e32	63
da6d2c28	64	static void cpu_group_map(cpumask_t dst, struct mask_info info, unsigned int cpu)
dbd70fb4	65	{
da6d2c28	66	static cpumask_t mask;
dbd70fb4	67
9e3d62d5	68	cpumask_clear(&mask);
a052096b	69	if (!cpumask_test_cpu(cpu, &cpu_setup_mask))
9e3d62d5 SS	70	goto out;
9e3d62d5 SS	71	cpumask_set_cpu(cpu, &mask);
1b25fda0 HC	72	switch (topology_mode) {
	73	case TOPOLOGY_MODE_HW:
	74	while (info) {
	75	if (cpumask_test_cpu(cpu, &info->mask)) {
da6d2c28	76	cpumask_copy(&mask, &info->mask);
1b25fda0 HC	77	break;
	78	}
	79	info = info->next;
	80	}
1b25fda0 HC	81	break;
	82	case TOPOLOGY_MODE_PACKAGE:
	83	cpumask_copy(&mask, cpu_present_mask);
	84	break;
	85	default:
2c7749b9	86	fallthrough;
1b25fda0	87	case TOPOLOGY_MODE_SINGLE:
1b25fda0	88	break;
0b52783d	89	}
a052096b	90	cpumask_and(&mask, &mask, &cpu_setup_mask);
9e3d62d5	91	out:
da6d2c28	92	cpumask_copy(dst, &mask);
dbd70fb4 HC	93	}
dbd70fb4 HC	94
da6d2c28	95	static void cpu_thread_map(cpumask_t *dst, unsigned int cpu)
10ad34bc	96	{
da6d2c28	97	static cpumask_t mask;
10ad34bc MS	98	int i;
10ad34bc MS	99
9e3d62d5	100	cpumask_clear(&mask);
a052096b	101	if (!cpumask_test_cpu(cpu, &cpu_setup_mask))
9e3d62d5 SS	102	goto out;
9e3d62d5 SS	103	cpumask_set_cpu(cpu, &mask);
1b25fda0	104	if (topology_mode != TOPOLOGY_MODE_HW)
da6d2c28	105	goto out;
10ad34bc	106	cpu -= cpu % (smp_cpu_mtid + 1);
a052096b SS	107	for (i = 0; i <= smp_cpu_mtid; i++) {
a052096b SS	108	if (cpumask_test_cpu(cpu + i, &cpu_setup_mask))
10ad34bc	109	cpumask_set_cpu(cpu + i, &mask);
a052096b	110	}
da6d2c28 HC	111	out:
da6d2c28 HC	112	cpumask_copy(dst, &mask);
10ad34bc MS	113	}
10ad34bc MS	114
251ea0ca HC	115	#define TOPOLOGY_CORE_BITS 64
251ea0ca HC	116
86d18a55 HC	117	static void add_cpus_to_mask(struct topology_core *tl_core,
	118	struct mask_info *drawer,
	119	struct mask_info *book,
	120	struct mask_info *socket)
dbd70fb4	121	{
439eb131	122	struct cpu_topology_s390 *topo;
10ad34bc	123	unsigned int core;
dbd70fb4	124
251ea0ca	125	for_each_set_bit(core, &tl_core->mask, TOPOLOGY_CORE_BITS) {
10ad34bc MS	126	unsigned int rcore;
10ad34bc MS	127	int lcpu, i;
dbd70fb4	128
10ad34bc MS	129	rcore = TOPOLOGY_CORE_BITS - 1 - core + tl_core->origin;
10ad34bc MS	130	lcpu = smp_find_processor_id(rcore << smp_cpu_mt_shift);
d1e57508 HC	131	if (lcpu < 0)
d1e57508 HC	132	continue;
10ad34bc	133	for (i = 0; i <= smp_cpu_mtid; i++) {
30fc4ca2	134	topo = &cpu_topology[lcpu + i];
adac0f1e	135	topo->drawer_id = drawer->id;
439eb131	136	topo->book_id = book->id;
86d18a55	137	topo->socket_id = socket->id;
439eb131 HC	138	topo->core_id = rcore;
439eb131 HC	139	topo->thread_id = lcpu + i;
1887aa07	140	topo->dedicated = tl_core->d;
adac0f1e	141	cpumask_set_cpu(lcpu + i, &drawer->mask);
10ad34bc MS	142	cpumask_set_cpu(lcpu + i, &book->mask);
10ad34bc MS	143	cpumask_set_cpu(lcpu + i, &socket->mask);
10ad34bc	144	smp_cpu_set_polarization(lcpu + i, tl_core->pp);
dbd70fb4 HC	145	}
	146	}
	147	}
	148
4cb14bc8	149	static void clear_masks(void)
dbd70fb4	150	{
4cb14bc8	151	struct mask_info *info;
dbd70fb4	152
d1e57508	153	info = &socket_info;
4cb14bc8	154	while (info) {
0f1959f5	155	cpumask_clear(&info->mask);
4cb14bc8 HC	156	info = info->next;
4cb14bc8 HC	157	}
4cb14bc8 HC	158	info = &book_info;
4cb14bc8 HC	159	while (info) {
0f1959f5	160	cpumask_clear(&info->mask);
4cb14bc8	161	info = info->next;
dbd70fb4	162	}
adac0f1e HC	163	info = &drawer_info;
	164	while (info) {
	165	cpumask_clear(&info->mask);
	166	info = info->next;
	167	}
dbd70fb4 HC	168	}
dbd70fb4 HC	169
c30f91b6	170	static union topology_entry next_tle(union topology_entry tle)
dbd70fb4	171	{
c30f91b6	172	if (!tle->nl)
10ad34bc	173	return (union topology_entry )((struct topology_core )tle + 1);
c30f91b6	174	return (union topology_entry )((struct topology_container )tle + 1);
dbd70fb4 HC	175	}
dbd70fb4 HC	176
86d18a55	177	static void tl_to_masks(struct sysinfo_15_1_x *info)
dbd70fb4	178	{
d1e57508	179	struct mask_info *socket = &socket_info;
83a24e32	180	struct mask_info *book = &book_info;
adac0f1e	181	struct mask_info *drawer = &drawer_info;
c30f91b6	182	union topology_entry tle, end;
4cb14bc8	183
86d18a55	184	clear_masks();
c10fde0d	185	tle = info->tle;
c30f91b6	186	end = (union topology_entry *)((unsigned long)info + info->length);
dbd70fb4 HC	187	while (tle < end) {
dbd70fb4 HC	188	switch (tle->nl) {
adac0f1e HC	189	case 3:
	190	drawer = drawer->next;
	191	drawer->id = tle->container.id;
	192	break;
dbd70fb4	193	case 2:
4cb14bc8 HC	194	book = book->next;
4cb14bc8 HC	195	book->id = tle->container.id;
dbd70fb4 HC	196	break;
dbd70fb4 HC	197	case 1:
d1e57508 HC	198	socket = socket->next;
d1e57508 HC	199	socket->id = tle->container.id;
dbd70fb4 HC	200	break;
dbd70fb4 HC	201	case 0:
86d18a55	202	add_cpus_to_mask(&tle->cpu, drawer, book, socket);
4baeb964 HC	203	break;
	204	default:
	205	clear_masks();
	206	return;
	207	}
	208	tle = next_tle(tle);
	209	}
	210	}
	211
c10fde0d HC	212	static void topology_update_polarization_simple(void)
	213	{
	214	int cpu;
	215
5439050f	216	for_each_possible_cpu(cpu)
50ab9a9a	217	smp_cpu_set_polarization(cpu, POLARIZATION_HRZ);
c10fde0d HC	218	}
	219
	220	static int ptf(unsigned long fc)
dbd70fb4 HC	221	{
	222	int rc;
	223
	224	asm volatile(
	225	" .insn rre,0xb9a20000,%1,%1\n"
	226	" ipm %0\n"
	227	" srl %0,28\n"
	228	: "=d" (rc)
c10fde0d HC	229	: "d" (fc) : "cc");
	230	return rc;
	231	}
	232
	233	int topology_set_cpu_management(int fc)
	234	{
83a24e32	235	int cpu, rc;
c10fde0d	236
9186d7a9	237	if (!MACHINE_HAS_TOPOLOGY)
c10fde0d HC	238	return -EOPNOTSUPP;
	239	if (fc)
	240	rc = ptf(PTF_VERTICAL);
	241	else
	242	rc = ptf(PTF_HORIZONTAL);
	243	if (rc)
	244	return -EBUSY;
5439050f	245	for_each_possible_cpu(cpu)
50ab9a9a	246	smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
dbd70fb4 HC	247	return rc;
	248	}
	249
52aeda7a	250	void update_cpu_masks(void)
d00aa4e7	251	{
95968497 AG	252	struct cpu_topology_s390 topo, topo_package, *topo_sibling;
95968497 AG	253	int cpu, sibling, pkg_first, smt_first, id;
d00aa4e7	254
4cb14bc8	255	for_each_possible_cpu(cpu) {
30fc4ca2	256	topo = &cpu_topology[cpu];
da6d2c28 HC	257	cpu_thread_map(&topo->thread_mask, cpu);
	258	cpu_group_map(&topo->core_mask, &socket_info, cpu);
	259	cpu_group_map(&topo->book_mask, &book_info, cpu);
	260	cpu_group_map(&topo->drawer_mask, &drawer_info, cpu);
95968497	261	topo->booted_cores = 0;
1b25fda0 HC	262	if (topology_mode != TOPOLOGY_MODE_HW) {
1b25fda0 HC	263	id = topology_mode == TOPOLOGY_MODE_PACKAGE ? 0 : cpu;
439eb131 HC	264	topo->thread_id = cpu;
439eb131 HC	265	topo->core_id = cpu;
1b25fda0 HC	266	topo->socket_id = id;
	267	topo->book_id = id;
	268	topo->drawer_id = id;
d1e57508	269	}
4cb14bc8	270	}
95968497 AG	271	for_each_online_cpu(cpu) {
	272	topo = &cpu_topology[cpu];
	273	pkg_first = cpumask_first(&topo->core_mask);
	274	topo_package = &cpu_topology[pkg_first];
	275	if (cpu == pkg_first) {
	276	for_each_cpu(sibling, &topo->core_mask) {
	277	topo_sibling = &cpu_topology[sibling];
	278	smt_first = cpumask_first(&topo_sibling->thread_mask);
	279	if (sibling == smt_first)
	280	topo_package->booted_cores++;
	281	}
	282	} else {
	283	topo->booted_cores = topo_package->booted_cores;
	284	}
	285	}
4cb14bc8 HC	286	}
4cb14bc8 HC	287
96f4a70d	288	void store_topology(struct sysinfo_15_1_x *info)
4cb14bc8	289	{
ae5ca67a	290	stsi(info, 15, 1, topology_mnest_limit());
d00aa4e7 HC	291	}
d00aa4e7 HC	292
1887aa07 MS	293	static void __arch_update_dedicated_flag(void *arg)
	294	{
	295	if (topology_cpu_dedicated(smp_processor_id()))
	296	set_cpu_flag(CIF_DEDICATED_CPU);
	297	else
	298	clear_cpu_flag(CIF_DEDICATED_CPU);
	299	}
	300
8c910580	301	static int __arch_update_cpu_topology(void)
dbd70fb4	302	{
c30f91b6	303	struct sysinfo_15_1_x *info = tl_info;
8c910580	304	int rc = 0;
dbd70fb4	305
51dce386	306	mutex_lock(&smp_cpu_state_mutex);
3a368f74 PH	307	if (MACHINE_HAS_TOPOLOGY) {
	308	rc = 1;
	309	store_topology(info);
	310	tl_to_masks(info);
c10fde0d	311	}
d1e57508	312	update_cpu_masks();
3a368f74 PH	313	if (!MACHINE_HAS_TOPOLOGY)
3a368f74 PH	314	topology_update_polarization_simple();
51dce386	315	mutex_unlock(&smp_cpu_state_mutex);
8c910580 HC	316	return rc;
	317	}
	318
	319	int arch_update_cpu_topology(void)
	320	{
	321	struct device *dev;
	322	int cpu, rc;
	323
	324	rc = __arch_update_cpu_topology();
1887aa07	325	on_each_cpu(__arch_update_dedicated_flag, NULL, 0);
dbd70fb4	326	for_each_online_cpu(cpu) {
8a25a2fd	327	dev = get_cpu_device(cpu);
f3122a79 VG	328	if (dev)
f3122a79 VG	329	kobject_uevent(&dev->kobj, KOBJ_CHANGE);
dbd70fb4	330	}
3a368f74	331	return rc;
dbd70fb4 HC	332	}
dbd70fb4 HC	333
fd781fa2 HC	334	static void topology_work_fn(struct work_struct *work)
fd781fa2 HC	335	{
f414f5f1	336	rebuild_sched_domains();
dbd70fb4 HC	337	}
dbd70fb4 HC	338
c10fde0d HC	339	void topology_schedule_update(void)
	340	{
	341	schedule_work(&topology_work);
	342	}
	343
51dce386 HC	344	static void topology_flush_work(void)
	345	{
	346	flush_work(&topology_work);
	347	}
	348
5cd79d6a	349	static void topology_timer_fn(struct timer_list *unused)
dbd70fb4	350	{
c10fde0d HC	351	if (ptf(PTF_CHECK))
c10fde0d HC	352	topology_schedule_update();
dbd70fb4 HC	353	set_topology_timer();
	354	}
	355
5cd79d6a	356	static struct timer_list topology_timer;
d68bddb7 HC	357
	358	static atomic_t topology_poll = ATOMIC_INIT(0);
	359
dbd70fb4 HC	360	static void set_topology_timer(void)
dbd70fb4 HC	361	{
d68bddb7	362	if (atomic_add_unless(&topology_poll, -1, 0))
0188d08a	363	mod_timer(&topology_timer, jiffies + msecs_to_jiffies(100));
d68bddb7	364	else
0188d08a	365	mod_timer(&topology_timer, jiffies + msecs_to_jiffies(60 * MSEC_PER_SEC));
d68bddb7 HC	366	}
	367
	368	void topology_expect_change(void)
	369	{
	370	if (!MACHINE_HAS_TOPOLOGY)
	371	return;
	372	/* This is racy, but it doesn't matter since it is just a heuristic.
	373	* Worst case is that we poll in a higher frequency for a bit longer.
	374	*/
	375	if (atomic_read(&topology_poll) > 60)
	376	return;
	377	atomic_add(60, &topology_poll);
	378	set_topology_timer();
dbd70fb4 HC	379	}
dbd70fb4 HC	380
83a24e32 HC	381	static int cpu_management;
83a24e32 HC	382
72f31889 LT	383	static ssize_t dispatching_show(struct device *dev,
72f31889 LT	384	struct device_attribute *attr,
83a24e32 HC	385	char *buf)
	386	{
	387	ssize_t count;
	388
	389	mutex_lock(&smp_cpu_state_mutex);
	390	count = sprintf(buf, "%d\n", cpu_management);
	391	mutex_unlock(&smp_cpu_state_mutex);
	392	return count;
	393	}
	394
72f31889 LT	395	static ssize_t dispatching_store(struct device *dev,
72f31889 LT	396	struct device_attribute *attr,
83a24e32 HC	397	const char *buf,
	398	size_t count)
	399	{
	400	int val, rc;
	401	char delim;
	402
	403	if (sscanf(buf, "%d %c", &val, &delim) != 1)
	404	return -EINVAL;
	405	if (val != 0 && val != 1)
	406	return -EINVAL;
	407	rc = 0;
a73de293	408	cpus_read_lock();
83a24e32 HC	409	mutex_lock(&smp_cpu_state_mutex);
	410	if (cpu_management == val)
	411	goto out;
	412	rc = topology_set_cpu_management(val);
d68bddb7 HC	413	if (rc)
	414	goto out;
	415	cpu_management = val;
	416	topology_expect_change();
83a24e32 HC	417	out:
83a24e32 HC	418	mutex_unlock(&smp_cpu_state_mutex);
a73de293	419	cpus_read_unlock();
83a24e32 HC	420	return rc ? rc : count;
83a24e32 HC	421	}
b6b996b6	422	static DEVICE_ATTR_RW(dispatching);
83a24e32	423
72f31889 LT	424	static ssize_t cpu_polarization_show(struct device *dev,
72f31889 LT	425	struct device_attribute attr, char buf)
83a24e32 HC	426	{
	427	int cpu = dev->id;
	428	ssize_t count;
	429
	430	mutex_lock(&smp_cpu_state_mutex);
50ab9a9a	431	switch (smp_cpu_get_polarization(cpu)) {
83a24e32 HC	432	case POLARIZATION_HRZ:
	433	count = sprintf(buf, "horizontal\n");
	434	break;
	435	case POLARIZATION_VL:
	436	count = sprintf(buf, "vertical:low\n");
	437	break;
	438	case POLARIZATION_VM:
	439	count = sprintf(buf, "vertical:medium\n");
	440	break;
	441	case POLARIZATION_VH:
	442	count = sprintf(buf, "vertical:high\n");
	443	break;
	444	default:
	445	count = sprintf(buf, "unknown\n");
	446	break;
	447	}
	448	mutex_unlock(&smp_cpu_state_mutex);
	449	return count;
	450	}
72f31889	451	static DEVICE_ATTR(polarization, 0444, cpu_polarization_show, NULL);
83a24e32 HC	452
83a24e32 HC	453	static struct attribute *topology_cpu_attrs[] = {
72f31889	454	&dev_attr_polarization.attr,
83a24e32 HC	455	NULL,
	456	};
	457
	458	static struct attribute_group topology_cpu_attr_group = {
	459	.attrs = topology_cpu_attrs,
	460	};
	461
1887aa07 MS	462	static ssize_t cpu_dedicated_show(struct device *dev,
	463	struct device_attribute attr, char buf)
	464	{
	465	int cpu = dev->id;
	466	ssize_t count;
	467
	468	mutex_lock(&smp_cpu_state_mutex);
	469	count = sprintf(buf, "%d\n", topology_cpu_dedicated(cpu));
	470	mutex_unlock(&smp_cpu_state_mutex);
	471	return count;
	472	}
	473	static DEVICE_ATTR(dedicated, 0444, cpu_dedicated_show, NULL);
	474
	475	static struct attribute *topology_extra_cpu_attrs[] = {
	476	&dev_attr_dedicated.attr,
	477	NULL,
	478	};
	479
	480	static struct attribute_group topology_extra_cpu_attr_group = {
	481	.attrs = topology_extra_cpu_attrs,
	482	};
	483
83a24e32 HC	484	int topology_cpu_init(struct cpu *cpu)
83a24e32 HC	485	{
1887aa07 MS	486	int rc;
	487
	488	rc = sysfs_create_group(&cpu->dev.kobj, &topology_cpu_attr_group);
	489	if (rc \|\| !MACHINE_HAS_TOPOLOGY)
	490	return rc;
	491	rc = sysfs_create_group(&cpu->dev.kobj, &topology_extra_cpu_attr_group);
	492	if (rc)
	493	sysfs_remove_group(&cpu->dev.kobj, &topology_cpu_attr_group);
	494	return rc;
83a24e32 HC	495	}
83a24e32 HC	496
3ddb1b75	497	static const struct cpumask *cpu_thread_mask(int cpu)
10ad34bc	498	{
30fc4ca2	499	return &cpu_topology[cpu].thread_mask;
10ad34bc MS	500	}
	501
	502
2dfd7476 VG	503	const struct cpumask *cpu_coregroup_mask(int cpu)
2dfd7476 VG	504	{
30fc4ca2	505	return &cpu_topology[cpu].core_mask;
2dfd7476 VG	506	}
	507
	508	static const struct cpumask *cpu_book_mask(int cpu)
	509	{
30fc4ca2	510	return &cpu_topology[cpu].book_mask;
2dfd7476 VG	511	}
2dfd7476 VG	512
adac0f1e HC	513	static const struct cpumask *cpu_drawer_mask(int cpu)
adac0f1e HC	514	{
30fc4ca2	515	return &cpu_topology[cpu].drawer_mask;
adac0f1e HC	516	}
adac0f1e HC	517
2dfd7476	518	static struct sched_domain_topology_level s390_topology[] = {
10ad34bc	519	{ cpu_thread_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
2dfd7476 VG	520	{ cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
2dfd7476 VG	521	{ cpu_book_mask, SD_INIT_NAME(BOOK) },
adac0f1e	522	{ cpu_drawer_mask, SD_INIT_NAME(DRAWER) },
c0e5ddab	523	{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
2dfd7476 VG	524	{ NULL, },
	525	};
	526
d05d15da HC	527	static void __init alloc_masks(struct sysinfo_15_1_x *info,
	528	struct mask_info *mask, int offset)
	529	{
	530	int i, nr_masks;
	531
	532	nr_masks = info->mag[TOPOLOGY_NR_MAG - offset];
	533	for (i = 0; i < info->mnest - offset; i++)
	534	nr_masks *= info->mag[TOPOLOGY_NR_MAG - offset - 1 - i];
	535	nr_masks = max(nr_masks, 1);
	536	for (i = 0; i < nr_masks; i++) {
eb31d559	537	mask->next = memblock_alloc(sizeof(*mask->next), 8);
8a7f97b9 MR	538	if (!mask->next)
	539	panic("%s: Failed to allocate %zu bytes align=0x%x\n",
	540	__func__, sizeof(*mask->next), 8);
d05d15da HC	541	mask = mask->next;
	542	}
	543	}
	544
8c910580	545	void __init topology_init_early(void)
d05d15da HC	546	{
d05d15da HC	547	struct sysinfo_15_1_x *info;
d05d15da	548
ebb299a5	549	set_sched_topology(s390_topology);
1b25fda0 HC	550	if (topology_mode == TOPOLOGY_MODE_UNINITIALIZED) {
	551	if (MACHINE_HAS_TOPOLOGY)
	552	topology_mode = TOPOLOGY_MODE_HW;
	553	else
	554	topology_mode = TOPOLOGY_MODE_SINGLE;
	555	}
d05d15da	556	if (!MACHINE_HAS_TOPOLOGY)
8c910580	557	goto out;
eb31d559	558	tl_info = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
8a7f97b9 MR	559	if (!tl_info)
	560	panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
	561	__func__, PAGE_SIZE, PAGE_SIZE);
d05d15da HC	562	info = tl_info;
d05d15da HC	563	store_topology(info);
496e59cc HC	564	pr_info("The CPU configuration topology of the machine is: %d %d %d %d %d %d / %d\n",
	565	info->mag[0], info->mag[1], info->mag[2], info->mag[3],
	566	info->mag[4], info->mag[5], info->mnest);
d05d15da HC	567	alloc_masks(info, &socket_info, 1);
d05d15da HC	568	alloc_masks(info, &book_info, 2);
adac0f1e	569	alloc_masks(info, &drawer_info, 3);
8c910580	570	out:
a052096b	571	cpumask_set_cpu(0, &cpu_setup_mask);
8c910580	572	__arch_update_cpu_topology();
1887aa07	573	__arch_update_dedicated_flag(NULL);
d05d15da	574	}
d05d15da	575
1b25fda0 HC	576	static inline int topology_get_mode(int enabled)
	577	{
	578	if (!enabled)
	579	return TOPOLOGY_MODE_SINGLE;
	580	return MACHINE_HAS_TOPOLOGY ? TOPOLOGY_MODE_HW : TOPOLOGY_MODE_PACKAGE;
	581	}
	582
51dce386 HC	583	static inline int topology_is_enabled(void)
	584	{
	585	return topology_mode != TOPOLOGY_MODE_SINGLE;
	586	}
	587
1b25fda0 HC	588	static int __init topology_setup(char *str)
	589	{
	590	bool enabled;
	591	int rc;
	592
	593	rc = kstrtobool(str, &enabled);
	594	if (rc)
	595	return rc;
	596	topology_mode = topology_get_mode(enabled);
	597	return 0;
	598	}
	599	early_param("topology", topology_setup);
	600
51dce386	601	static int topology_ctl_handler(struct ctl_table *ctl, int write,
32927393	602	void buffer, size_t lenp, loff_t *ppos)
51dce386	603	{
196851be	604	int enabled = topology_is_enabled();
51dce386	605	int new_mode;
196851be VG	606	int rc;
	607	struct ctl_table ctl_entry = {
	608	.procname = ctl->procname,
	609	.data = &enabled,
	610	.maxlen = sizeof(int),
eec4844f MC	611	.extra1 = SYSCTL_ZERO,
eec4844f MC	612	.extra2 = SYSCTL_ONE,
196851be VG	613	};
	614
	615	rc = proc_douintvec_minmax(&ctl_entry, write, buffer, lenp, ppos);
	616	if (rc < 0 \|\| !write)
	617	return rc;
51dce386	618
51dce386	619	mutex_lock(&smp_cpu_state_mutex);
196851be	620	new_mode = topology_get_mode(enabled);
51dce386 HC	621	if (topology_mode != new_mode) {
	622	topology_mode = new_mode;
	623	topology_schedule_update();
	624	}
	625	mutex_unlock(&smp_cpu_state_mutex);
	626	topology_flush_work();
196851be VG	627
196851be VG	628	return rc;
51dce386 HC	629	}
	630
	631	static struct ctl_table topology_ctl_table[] = {
	632	{
	633	.procname = "topology",
	634	.mode = 0644,
	635	.proc_handler = topology_ctl_handler,
	636	},
	637	{ },
	638	};
	639
	640	static struct ctl_table topology_dir_table[] = {
	641	{
	642	.procname = "s390",
	643	.maxlen = 0,
	644	.mode = 0555,
	645	.child = topology_ctl_table,
	646	},
	647	{ },
	648	};
	649
83a24e32 HC	650	static int __init topology_init(void)
83a24e32 HC	651	{
5cd79d6a	652	timer_setup(&topology_timer, topology_timer_fn, TIMER_DEFERRABLE);
48e9a6c1 MS	653	if (MACHINE_HAS_TOPOLOGY)
	654	set_topology_timer();
	655	else
83a24e32	656	topology_update_polarization_simple();
51dce386	657	register_sysctl_table(topology_dir_table);
72f31889	658	return device_create_file(cpu_subsys.dev_root, &dev_attr_dispatching);
83a24e32 HC	659	}
83a24e32 HC	660	device_initcall(topology_init);