[fio.git] / os / windows / cpu-affinity.c

#include "os/os.h"

#include <windows.h>

static void print_mask(os_cpu_mask_t *cpumask)
{
	for (int i = 0; i < FIO_CPU_MASK_ROWS; i++)
		dprint(FD_PROCESS, "cpumask[%d]=%" PRIu64 "\n", i, cpumask->row[i]);
}

/* Return the index of the least significant set CPU in cpumask or -1 if no
 * CPUs are set */
int first_set_cpu(os_cpu_mask_t *cpumask)
{
	int cpus_offset, mask_first_cpu, row;

	cpus_offset = 0;
	row = 0;
	mask_first_cpu = -1;
	while (mask_first_cpu < 0 && row < FIO_CPU_MASK_ROWS) {
		int row_first_cpu;

		row_first_cpu = __builtin_ffsll(cpumask->row[row]) - 1;
		dprint(FD_PROCESS, "row_first_cpu=%d cpumask->row[%d]=%" PRIu64 "\n",
		       row_first_cpu, row, cpumask->row[row]);
		if (row_first_cpu > -1) {
			mask_first_cpu = cpus_offset + row_first_cpu;
			dprint(FD_PROCESS, "first set cpu in mask is at index %d\n",
			       mask_first_cpu);
		} else {
			cpus_offset += FIO_CPU_MASK_STRIDE;
			row++;
		}
	}

	return mask_first_cpu;
}

/* Return the index of the most significant set CPU in cpumask or -1 if no
 * CPUs are set */
static int last_set_cpu(os_cpu_mask_t *cpumask)
{
	int cpus_offset, mask_last_cpu, row;

	cpus_offset = (FIO_CPU_MASK_ROWS - 1) * FIO_CPU_MASK_STRIDE;
	row = FIO_CPU_MASK_ROWS - 1;
	mask_last_cpu = -1;
	while (mask_last_cpu < 0 && row >= 0) {
		int row_last_cpu;

		if (cpumask->row[row] == 0)
			row_last_cpu = -1;
		else {
			uint64_t tmp = cpumask->row[row];

			row_last_cpu = 0;
			while (tmp >>= 1)
			    row_last_cpu++;
		}

		dprint(FD_PROCESS, "row_last_cpu=%d cpumask->row[%d]=%" PRIu64 "\n",
		       row_last_cpu, row, cpumask->row[row]);
		if (row_last_cpu > -1) {
			mask_last_cpu = cpus_offset + row_last_cpu;
			dprint(FD_PROCESS, "last set cpu in mask is at index %d\n",
			       mask_last_cpu);
		} else {
			cpus_offset -= FIO_CPU_MASK_STRIDE;
			row--;
		}
	}

	return mask_last_cpu;
}

static int mask_to_group_mask(os_cpu_mask_t *cpumask, int *processor_group, uint64_t *affinity_mask)
{
	WORD online_groups, group, group_size;
	bool found;
	int cpus_offset, search_cpu, last_cpu, bit_offset, row, end;
	uint64_t group_cpumask;

	search_cpu = first_set_cpu(cpumask);
	if (search_cpu < 0) {
		log_info("CPU mask doesn't set any CPUs\n");
		return 1;
	}

	/* Find processor group first set CPU applies to */
	online_groups = GetActiveProcessorGroupCount();
	group = 0;
	found = false;
	cpus_offset = 0;
	group_size = 0;
	while (!found && group < online_groups) {
		group_size = GetActiveProcessorCount(group);
		dprint(FD_PROCESS, "group=%d group_start=%d group_size=%u search_cpu=%d\n",
		       group, cpus_offset, group_size, search_cpu);
		if (cpus_offset + group_size > search_cpu)
			found = true;
		else {
			cpus_offset += group_size;
			group++;
		}
	}

	if (!found) {
		log_err("CPU mask contains processor beyond last active processor index (%d)\n",
			 cpus_offset - 1);
		print_mask(cpumask);
		return 1;
	}

	/* Check all the CPUs in the mask apply to ONLY that processor group */
	last_cpu = last_set_cpu(cpumask);
	if (last_cpu > (cpus_offset + group_size - 1)) {
		log_info("CPU mask cannot bind CPUs (e.g. %d, %d) that are "
			 "in different processor groups\n", search_cpu,
			 last_cpu);
		print_mask(cpumask);
		return 1;
	}

	/* Extract the current processor group mask from the cpumask */
	row = cpus_offset / FIO_CPU_MASK_STRIDE;
	bit_offset = cpus_offset % FIO_CPU_MASK_STRIDE;
	group_cpumask = cpumask->row[row] >> bit_offset;
	end = bit_offset + group_size;
	if (end > FIO_CPU_MASK_STRIDE && (row + 1 < FIO_CPU_MASK_ROWS)) {
		/* Some of the next row needs to be part of the mask */
		int needed, needed_shift, needed_mask_shift;
		uint64_t needed_mask;

		needed = end - FIO_CPU_MASK_STRIDE;
		needed_shift = FIO_CPU_MASK_STRIDE - bit_offset;
		needed_mask_shift = FIO_CPU_MASK_STRIDE - needed;
		needed_mask = (uint64_t)-1 >> needed_mask_shift;
		dprint(FD_PROCESS,
		       "bit_offset=%d end=%d needed=%d needed_shift=%d needed_mask=%" PRIu64 "needed_mask_shift=%d\n",
		       bit_offset, end, needed, needed_shift, needed_mask,
		       needed_mask_shift);
		group_cpumask |= (cpumask->row[row + 1] & needed_mask) << needed_shift;
	}
	group_cpumask &= (uint64_t)-1 >> (FIO_CPU_MASK_STRIDE - group_size);

	/* Return group and mask */
	dprint(FD_PROCESS, "Returning group=%d group_mask=%" PRIu64 "\n",
	       group, group_cpumask);
	*processor_group = group;
	*affinity_mask = group_cpumask;

	return 0;
}

int fio_setaffinity(int pid, os_cpu_mask_t cpumask)
{
	HANDLE handle = NULL;
	int group, ret;
	uint64_t group_mask = 0;
	GROUP_AFFINITY new_group_affinity;

	ret = -1;

	if (mask_to_group_mask(&cpumask, &group, &group_mask) != 0)
		goto err;

	handle = OpenThread(THREAD_QUERY_INFORMATION | THREAD_SET_INFORMATION,
			    TRUE, pid);
	if (handle == NULL) {
		log_err("fio_setaffinity: failed to get handle for pid %d\n", pid);
		goto err;
	}

	/* Set group and mask.
	 * Note: if the GROUP_AFFINITY struct's Reserved members are not
	 * initialised to 0 then SetThreadGroupAffinity will fail with
	 * GetLastError() set to ERROR_INVALID_PARAMETER */
	new_group_affinity.Mask = (KAFFINITY) group_mask;
	new_group_affinity.Group = group;
	new_group_affinity.Reserved[0] = 0;
	new_group_affinity.Reserved[1] = 0;
	new_group_affinity.Reserved[2] = 0;
	if (SetThreadGroupAffinity(handle, &new_group_affinity, NULL) != 0)
		ret = 0;
	else {
		log_err("fio_setaffinity: failed to set thread affinity (pid %d, group %d, mask %" PRIx64 ", GetLastError=%lu)\n",
			pid, group, group_mask, GetLastError());
		goto err;
	}

err:
	if (handle)
		CloseHandle(handle);
	return ret;
}

static void cpu_to_row_offset(int cpu, int *row, int *offset)
{
	*row = cpu / FIO_CPU_MASK_STRIDE;
	*offset = cpu << FIO_CPU_MASK_STRIDE * *row;
}

int fio_cpuset_init(os_cpu_mask_t *mask)
{
	for (int i = 0; i < FIO_CPU_MASK_ROWS; i++)
		mask->row[i] = 0;
	return 0;
}

/*
 * fio_getaffinity() should not be called once a fio_setaffinity() call has
 * been made because fio_setaffinity() may put the process into multiple
 * processor groups
 */
int fio_getaffinity(int pid, os_cpu_mask_t *mask)
{
	int ret;
	int row, offset, end, group, group_size, group_start_cpu;
	DWORD_PTR process_mask, system_mask;
	HANDLE handle;
	PUSHORT current_groups;
	USHORT group_count;
	WORD online_groups;

	ret = -1;
	current_groups = NULL;
	handle = OpenProcess(PROCESS_QUERY_INFORMATION, TRUE, pid);
	if (handle == NULL) {
		log_err("fio_getaffinity: failed to get handle for pid %d\n",
			pid);
		goto err;
	}

	group_count = 16;
	/*
	 * GetProcessGroupAffinity() seems to expect more than the natural
	 * alignment for a USHORT from the area pointed to by current_groups so
	 * arrange for maximum alignment by allocating via malloc()
	 */
	current_groups = malloc(group_count * sizeof(USHORT));
	if (!current_groups) {
		log_err("fio_getaffinity: malloc failed\n");
		goto err;
	}
	if (!GetProcessGroupAffinity(handle, &group_count, current_groups)) {
		log_err("%s: failed to get single group affinity for pid %d (%lu)\n",
			__func__, pid, GetLastError());
		goto err;
	}
	if (group_count > 1) {
		log_err("%s: pid %d is associated with %d process groups\n",
			__func__, pid, group_count);
		goto err;
	}
	if (!GetProcessAffinityMask(handle, &process_mask, &system_mask)) {
		log_err("%s: GetProcessAffinityMask() failed for pid %d\n",
			__func__, pid);
		goto err;
	}

	/* Convert group and group relative mask to full CPU mask */
	online_groups = GetActiveProcessorGroupCount();
	if (online_groups == 0) {
		log_err("fio_getaffinity: error retrieving total processor groups\n");
		goto err;
	}

	group = 0;
	group_start_cpu = 0;
	group_size = 0;
	dprint(FD_PROCESS, "current_groups=%d group_count=%d\n",
	       current_groups[0], group_count);
	while (true) {
		group_size = GetActiveProcessorCount(group);
		if (group_size == 0) {
			log_err("fio_getaffinity: error retrieving size of "
				"processor group %d\n", group);
			goto err;
		} else if (group >= current_groups[0] || group >= online_groups)
			break;
		else {
			group_start_cpu += group_size;
			group++;
		}
	}

	if (group != current_groups[0]) {
		log_err("fio_getaffinity: could not find processor group %d\n",
			current_groups[0]);
		goto err;
	}

	dprint(FD_PROCESS, "group_start_cpu=%d, group size=%u\n",
	       group_start_cpu, group_size);
	if ((group_start_cpu + group_size) >= FIO_MAX_CPUS) {
		log_err("fio_getaffinity failed: current CPU affinity (group "
			"%d, group_start_cpu %d, group_size %d) extends "
			"beyond mask's highest CPU (%d)\n", group,
			group_start_cpu, group_size, FIO_MAX_CPUS);
		goto err;
	}

	fio_cpuset_init(mask);
	cpu_to_row_offset(group_start_cpu, &row, &offset);
	mask->row[row] = process_mask;
	mask->row[row] <<= offset;
	end = offset + group_size;
	if (end > FIO_CPU_MASK_STRIDE) {
		int needed;
		uint64_t needed_mask;

		needed = FIO_CPU_MASK_STRIDE - end;
		needed_mask = (uint64_t)-1 >> (FIO_CPU_MASK_STRIDE - needed);
		row++;
		mask->row[row] = process_mask;
		mask->row[row] >>= needed;
		mask->row[row] &= needed_mask;
	}
	ret = 0;

err:
	if (handle)
		CloseHandle(handle);
	if (current_groups)
		free(current_groups);

	return ret;
}

void fio_cpu_clear(os_cpu_mask_t *mask, int cpu)
{
	int row, offset;
	cpu_to_row_offset(cpu, &row, &offset);

	mask->row[row] &= ~(1ULL << offset);
}

void fio_cpu_set(os_cpu_mask_t *mask, int cpu)
{
	int row, offset;
	cpu_to_row_offset(cpu, &row, &offset);

	mask->row[row] |= 1ULL << offset;
}

int fio_cpu_isset(os_cpu_mask_t *mask, int cpu)
{
	int row, offset;
	cpu_to_row_offset(cpu, &row, &offset);

	return (mask->row[row] & (1ULL << offset)) != 0;
}

int fio_cpu_count(os_cpu_mask_t *mask)
{
	int count = 0;

	for (int i = 0; i < FIO_CPU_MASK_ROWS; i++)
		count += hweight64(mask->row[i]);

	return count;
}

int fio_cpuset_exit(os_cpu_mask_t *mask)
{
	return 0;
}
Commit	Line	Data
a1957954 BVA	1	#include "os/os.h"
	2
	3	#include <windows.h>
	4
a1957954 BVA	5	static void print_mask(os_cpu_mask_t *cpumask)
	6	{
	7	for (int i = 0; i < FIO_CPU_MASK_ROWS; i++)
f5515176	8	dprint(FD_PROCESS, "cpumask[%d]=%" PRIu64 "\n", i, cpumask->row[i]);
a1957954 BVA	9	}
	10
	11	/* Return the index of the least significant set CPU in cpumask or -1 if no
	12	* CPUs are set */
	13	int first_set_cpu(os_cpu_mask_t *cpumask)
	14	{
	15	int cpus_offset, mask_first_cpu, row;
	16
	17	cpus_offset = 0;
	18	row = 0;
	19	mask_first_cpu = -1;
	20	while (mask_first_cpu < 0 && row < FIO_CPU_MASK_ROWS) {
	21	int row_first_cpu;
	22
	23	row_first_cpu = __builtin_ffsll(cpumask->row[row]) - 1;
f5515176	24	dprint(FD_PROCESS, "row_first_cpu=%d cpumask->row[%d]=%" PRIu64 "\n",
a1957954 BVA	25	row_first_cpu, row, cpumask->row[row]);
	26	if (row_first_cpu > -1) {
	27	mask_first_cpu = cpus_offset + row_first_cpu;
	28	dprint(FD_PROCESS, "first set cpu in mask is at index %d\n",
	29	mask_first_cpu);
	30	} else {
	31	cpus_offset += FIO_CPU_MASK_STRIDE;
	32	row++;
	33	}
	34	}
	35
	36	return mask_first_cpu;
	37	}
	38
	39	/* Return the index of the most significant set CPU in cpumask or -1 if no
	40	* CPUs are set */
	41	static int last_set_cpu(os_cpu_mask_t *cpumask)
	42	{
	43	int cpus_offset, mask_last_cpu, row;
	44
	45	cpus_offset = (FIO_CPU_MASK_ROWS - 1) * FIO_CPU_MASK_STRIDE;
	46	row = FIO_CPU_MASK_ROWS - 1;
	47	mask_last_cpu = -1;
	48	while (mask_last_cpu < 0 && row >= 0) {
	49	int row_last_cpu;
	50
	51	if (cpumask->row[row] == 0)
	52	row_last_cpu = -1;
	53	else {
	54	uint64_t tmp = cpumask->row[row];
	55
	56	row_last_cpu = 0;
	57	while (tmp >>= 1)
	58	row_last_cpu++;
	59	}
	60
f5515176	61	dprint(FD_PROCESS, "row_last_cpu=%d cpumask->row[%d]=%" PRIu64 "\n",
a1957954 BVA	62	row_last_cpu, row, cpumask->row[row]);
	63	if (row_last_cpu > -1) {
	64	mask_last_cpu = cpus_offset + row_last_cpu;
	65	dprint(FD_PROCESS, "last set cpu in mask is at index %d\n",
	66	mask_last_cpu);
	67	} else {
	68	cpus_offset -= FIO_CPU_MASK_STRIDE;
	69	row--;
	70	}
	71	}
	72
	73	return mask_last_cpu;
	74	}
	75
	76	static int mask_to_group_mask(os_cpu_mask_t cpumask, int processor_group, uint64_t *affinity_mask)
	77	{
	78	WORD online_groups, group, group_size;
	79	bool found;
	80	int cpus_offset, search_cpu, last_cpu, bit_offset, row, end;
	81	uint64_t group_cpumask;
	82
	83	search_cpu = first_set_cpu(cpumask);
	84	if (search_cpu < 0) {
	85	log_info("CPU mask doesn't set any CPUs\n");
	86	return 1;
	87	}
	88
	89	/* Find processor group first set CPU applies to */
	90	online_groups = GetActiveProcessorGroupCount();
	91	group = 0;
	92	found = false;
	93	cpus_offset = 0;
	94	group_size = 0;
	95	while (!found && group < online_groups) {
	96	group_size = GetActiveProcessorCount(group);
	97	dprint(FD_PROCESS, "group=%d group_start=%d group_size=%u search_cpu=%d\n",
	98	group, cpus_offset, group_size, search_cpu);
	99	if (cpus_offset + group_size > search_cpu)
	100	found = true;
	101	else {
	102	cpus_offset += group_size;
	103	group++;
	104	}
	105	}
	106
	107	if (!found) {
	108	log_err("CPU mask contains processor beyond last active processor index (%d)\n",
	109	cpus_offset - 1);
	110	print_mask(cpumask);
	111	return 1;
	112	}
	113
	114	/* Check all the CPUs in the mask apply to ONLY that processor group */
	115	last_cpu = last_set_cpu(cpumask);
	116	if (last_cpu > (cpus_offset + group_size - 1)) {
	117	log_info("CPU mask cannot bind CPUs (e.g. %d, %d) that are "
	118	"in different processor groups\n", search_cpu,
	119	last_cpu);
	120	print_mask(cpumask);
	121	return 1;
	122	}
	123
	124	/* Extract the current processor group mask from the cpumask */
	125	row = cpus_offset / FIO_CPU_MASK_STRIDE;
126	bit_offset = cpus_offset % FIO_CPU_MASK_STRIDE;
127	group_cpumask = cpumask->row[row] >> bit_offset;
128	end = bit_offset + group_size;
129	if (end > FIO_CPU_MASK_STRIDE && (row + 1 < FIO_CPU_MASK_ROWS)) {
130	/* Some of the next row needs to be part of the mask */
131	int needed, needed_shift, needed_mask_shift;
132	uint64_t needed_mask;
133
134	needed = end - FIO_CPU_MASK_STRIDE;
135	needed_shift = FIO_CPU_MASK_STRIDE - bit_offset;
136	needed_mask_shift = FIO_CPU_MASK_STRIDE - needed;
137	needed_mask = (uint64_t)-1 >> needed_mask_shift;
f5515176 SW	138	dprint(FD_PROCESS,
	139	"bit_offset=%d end=%d needed=%d needed_shift=%d needed_mask=%" PRIu64 "needed_mask_shift=%d\n",
	140	bit_offset, end, needed, needed_shift, needed_mask,
	141	needed_mask_shift);
a1957954 BVA	142	group_cpumask \|= (cpumask->row[row + 1] & needed_mask) << needed_shift;
	143	}
	144	group_cpumask &= (uint64_t)-1 >> (FIO_CPU_MASK_STRIDE - group_size);
	145
	146	/* Return group and mask */
f5515176 SW	147	dprint(FD_PROCESS, "Returning group=%d group_mask=%" PRIu64 "\n",
f5515176 SW	148	group, group_cpumask);
a1957954 BVA	149	*processor_group = group;
	150	*affinity_mask = group_cpumask;
	151
	152	return 0;
	153	}
	154
	155	int fio_setaffinity(int pid, os_cpu_mask_t cpumask)
	156	{
	157	HANDLE handle = NULL;
	158	int group, ret;
	159	uint64_t group_mask = 0;
	160	GROUP_AFFINITY new_group_affinity;
	161
	162	ret = -1;
	163
	164	if (mask_to_group_mask(&cpumask, &group, &group_mask) != 0)
	165	goto err;
	166
	167	handle = OpenThread(THREAD_QUERY_INFORMATION \| THREAD_SET_INFORMATION,
	168	TRUE, pid);
	169	if (handle == NULL) {
	170	log_err("fio_setaffinity: failed to get handle for pid %d\n", pid);
	171	goto err;
	172	}
	173
	174	/* Set group and mask.
	175	* Note: if the GROUP_AFFINITY struct's Reserved members are not
	176	* initialised to 0 then SetThreadGroupAffinity will fail with
	177	* GetLastError() set to ERROR_INVALID_PARAMETER */
	178	new_group_affinity.Mask = (KAFFINITY) group_mask;
	179	new_group_affinity.Group = group;
	180	new_group_affinity.Reserved[0] = 0;
	181	new_group_affinity.Reserved[1] = 0;
	182	new_group_affinity.Reserved[2] = 0;
	183	if (SetThreadGroupAffinity(handle, &new_group_affinity, NULL) != 0)
	184	ret = 0;
	185	else {
4937100f SW	186	log_err("fio_setaffinity: failed to set thread affinity (pid %d, group %d, mask %" PRIx64 ", GetLastError=%lu)\n",
4937100f SW	187	pid, group, group_mask, GetLastError());
a1957954 BVA	188	goto err;
	189	}
	190
	191	err:
	192	if (handle)
	193	CloseHandle(handle);
	194	return ret;
	195	}
	196
	197	static void cpu_to_row_offset(int cpu, int row, int offset)
	198	{
	199	*row = cpu / FIO_CPU_MASK_STRIDE;
	200	offset = cpu << FIO_CPU_MASK_STRIDE *row;
	201	}
	202
	203	int fio_cpuset_init(os_cpu_mask_t *mask)
	204	{
	205	for (int i = 0; i < FIO_CPU_MASK_ROWS; i++)
	206	mask->row[i] = 0;
	207	return 0;
	208	}
	209
	210	/*
	211	* fio_getaffinity() should not be called once a fio_setaffinity() call has
	212	* been made because fio_setaffinity() may put the process into multiple
	213	* processor groups
	214	*/
	215	int fio_getaffinity(int pid, os_cpu_mask_t *mask)
	216	{
	217	int ret;
	218	int row, offset, end, group, group_size, group_start_cpu;
	219	DWORD_PTR process_mask, system_mask;
	220	HANDLE handle;
	221	PUSHORT current_groups;
	222	USHORT group_count;
	223	WORD online_groups;
	224
	225	ret = -1;
	226	current_groups = NULL;
	227	handle = OpenProcess(PROCESS_QUERY_INFORMATION, TRUE, pid);
	228	if (handle == NULL) {
	229	log_err("fio_getaffinity: failed to get handle for pid %d\n",
	230	pid);
	231	goto err;
	232	}
	233
2f56f0a0	234	group_count = 16;
a1957954 BVA	235	/*
	236	* GetProcessGroupAffinity() seems to expect more than the natural
	237	* alignment for a USHORT from the area pointed to by current_groups so
	238	* arrange for maximum alignment by allocating via malloc()
	239	*/
2f56f0a0	240	current_groups = malloc(group_count * sizeof(USHORT));
a1957954 BVA	241	if (!current_groups) {
	242	log_err("fio_getaffinity: malloc failed\n");
	243	goto err;
	244	}
2f56f0a0	245	if (!GetProcessGroupAffinity(handle, &group_count, current_groups)) {
4937100f	246	log_err("%s: failed to get single group affinity for pid %d (%lu)\n",
2f56f0a0 BVA	247	__func__, pid, GetLastError());
	248	goto err;
	249	}
	250	if (group_count > 1) {
	251	log_err("%s: pid %d is associated with %d process groups\n",
	252	__func__, pid, group_count);
	253	goto err;
	254	}
	255	if (!GetProcessAffinityMask(handle, &process_mask, &system_mask)) {
f5515176	256	log_err("%s: GetProcessAffinityMask() failed for pid %d\n",
2f56f0a0	257	__func__, pid);
a1957954 BVA	258	goto err;
a1957954 BVA	259	}
a1957954 BVA	260
	261	/* Convert group and group relative mask to full CPU mask */
	262	online_groups = GetActiveProcessorGroupCount();
	263	if (online_groups == 0) {
	264	log_err("fio_getaffinity: error retrieving total processor groups\n");
	265	goto err;
	266	}
	267
	268	group = 0;
	269	group_start_cpu = 0;
	270	group_size = 0;
	271	dprint(FD_PROCESS, "current_groups=%d group_count=%d\n",
	272	current_groups[0], group_count);
	273	while (true) {
	274	group_size = GetActiveProcessorCount(group);
	275	if (group_size == 0) {
	276	log_err("fio_getaffinity: error retrieving size of "
	277	"processor group %d\n", group);
	278	goto err;
	279	} else if (group >= current_groups[0] \|\| group >= online_groups)
	280	break;
	281	else {
	282	group_start_cpu += group_size;
	283	group++;
	284	}
	285	}
	286
	287	if (group != current_groups[0]) {
	288	log_err("fio_getaffinity: could not find processor group %d\n",
	289	current_groups[0]);
	290	goto err;
	291	}
	292
	293	dprint(FD_PROCESS, "group_start_cpu=%d, group size=%u\n",
	294	group_start_cpu, group_size);
	295	if ((group_start_cpu + group_size) >= FIO_MAX_CPUS) {
	296	log_err("fio_getaffinity failed: current CPU affinity (group "
	297	"%d, group_start_cpu %d, group_size %d) extends "
	298	"beyond mask's highest CPU (%d)\n", group,
	299	group_start_cpu, group_size, FIO_MAX_CPUS);
	300	goto err;
	301	}
	302
	303	fio_cpuset_init(mask);
	304	cpu_to_row_offset(group_start_cpu, &row, &offset);
	305	mask->row[row] = process_mask;
	306	mask->row[row] <<= offset;
	307	end = offset + group_size;
	308	if (end > FIO_CPU_MASK_STRIDE) {
	309	int needed;
	310	uint64_t needed_mask;
	311
	312	needed = FIO_CPU_MASK_STRIDE - end;
	313	needed_mask = (uint64_t)-1 >> (FIO_CPU_MASK_STRIDE - needed);
	314	row++;
	315	mask->row[row] = process_mask;
	316	mask->row[row] >>= needed;
	317	mask->row[row] &= needed_mask;
	318	}
	319	ret = 0;
	320
	321	err:
	322	if (handle)
	323	CloseHandle(handle);
324	if (current_groups)
325	free(current_groups);
326
327	return ret;
328	}
329
330	void fio_cpu_clear(os_cpu_mask_t *mask, int cpu)
331	{
332	int row, offset;
333	cpu_to_row_offset(cpu, &row, &offset);
334
335	mask->row[row] &= ~(1ULL << offset);
336	}
337
338	void fio_cpu_set(os_cpu_mask_t *mask, int cpu)
339	{
340	int row, offset;
341	cpu_to_row_offset(cpu, &row, &offset);
342
343	mask->row[row] \|= 1ULL << offset;
344	}
345
346	int fio_cpu_isset(os_cpu_mask_t *mask, int cpu)
347	{
348	int row, offset;
349	cpu_to_row_offset(cpu, &row, &offset);
350
351	return (mask->row[row] & (1ULL << offset)) != 0;
352	}
353
354	int fio_cpu_count(os_cpu_mask_t *mask)
355	{
356	int count = 0;
357
358	for (int i = 0; i < FIO_CPU_MASK_ROWS; i++)
359	count += hweight64(mask->row[i]);
360
361	return count;
362	}
363
364	int fio_cpuset_exit(os_cpu_mask_t *mask)
365	{
366	return 0;
367	}