[linux-2.6-block.git] / tools / perf / arch / x86 / util / intel-bts.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * intel-bts.c: Intel Processor Trace support
 * Copyright (c) 2013-2015, Intel Corporation.
 */

#include <errno.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/bitops.h>
#include <linux/log2.h>

#include "../../util/cpumap.h"
#include "../../util/evsel.h"
#include "../../util/evlist.h"
#include "../../util/session.h"
#include "../../util/util.h"
#include "../../util/pmu.h"
#include "../../util/debug.h"
#include "../../util/tsc.h"
#include "../../util/auxtrace.h"
#include "../../util/intel-bts.h"

#define KiB(x) ((x) * 1024)
#define MiB(x) ((x) * 1024 * 1024)
#define KiB_MASK(x) (KiB(x) - 1)
#define MiB_MASK(x) (MiB(x) - 1)

struct intel_bts_snapshot_ref {
	void	*ref_buf;
	size_t	ref_offset;
	bool	wrapped;
};

struct intel_bts_recording {
	struct auxtrace_record		itr;
	struct perf_pmu			*intel_bts_pmu;
	struct perf_evlist		*evlist;
	bool				snapshot_mode;
	size_t				snapshot_size;
	int				snapshot_ref_cnt;
	struct intel_bts_snapshot_ref	*snapshot_refs;
};

struct branch {
	u64 from;
	u64 to;
	u64 misc;
};

static size_t
intel_bts_info_priv_size(struct auxtrace_record *itr __maybe_unused,
			 struct perf_evlist *evlist __maybe_unused)
{
	return INTEL_BTS_AUXTRACE_PRIV_SIZE;
}

static int intel_bts_info_fill(struct auxtrace_record *itr,
			       struct perf_session *session,
			       struct auxtrace_info_event *auxtrace_info,
			       size_t priv_size)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);
	struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu;
	struct perf_event_mmap_page *pc;
	struct perf_tsc_conversion tc = { .time_mult = 0, };
	bool cap_user_time_zero = false;
	int err;

	if (priv_size != INTEL_BTS_AUXTRACE_PRIV_SIZE)
		return -EINVAL;

	if (!session->evlist->nr_mmaps)
		return -EINVAL;

	pc = session->evlist->mmap[0].base;
	if (pc) {
		err = perf_read_tsc_conversion(pc, &tc);
		if (err) {
			if (err != -EOPNOTSUPP)
				return err;
		} else {
			cap_user_time_zero = tc.time_mult != 0;
		}
		if (!cap_user_time_zero)
			ui__warning("Intel BTS: TSC not available\n");
	}

	auxtrace_info->type = PERF_AUXTRACE_INTEL_BTS;
	auxtrace_info->priv[INTEL_BTS_PMU_TYPE] = intel_bts_pmu->type;
	auxtrace_info->priv[INTEL_BTS_TIME_SHIFT] = tc.time_shift;
	auxtrace_info->priv[INTEL_BTS_TIME_MULT] = tc.time_mult;
	auxtrace_info->priv[INTEL_BTS_TIME_ZERO] = tc.time_zero;
	auxtrace_info->priv[INTEL_BTS_CAP_USER_TIME_ZERO] = cap_user_time_zero;
	auxtrace_info->priv[INTEL_BTS_SNAPSHOT_MODE] = btsr->snapshot_mode;

	return 0;
}

static int intel_bts_recording_options(struct auxtrace_record *itr,
				       struct perf_evlist *evlist,
				       struct record_opts *opts)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);
	struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu;
	struct perf_evsel *evsel, *intel_bts_evsel = NULL;
	const struct cpu_map *cpus = evlist->cpus;
	bool privileged = geteuid() == 0 || perf_event_paranoid() < 0;

	btsr->evlist = evlist;
	btsr->snapshot_mode = opts->auxtrace_snapshot_mode;

	evlist__for_each_entry(evlist, evsel) {
		if (evsel->attr.type == intel_bts_pmu->type) {
			if (intel_bts_evsel) {
				pr_err("There may be only one " INTEL_BTS_PMU_NAME " event\n");
				return -EINVAL;
			}
			evsel->attr.freq = 0;
			evsel->attr.sample_period = 1;
			intel_bts_evsel = evsel;
			opts->full_auxtrace = true;
		}
	}

	if (opts->auxtrace_snapshot_mode && !opts->full_auxtrace) {
		pr_err("Snapshot mode (-S option) requires " INTEL_BTS_PMU_NAME " PMU event (-e " INTEL_BTS_PMU_NAME ")\n");
		return -EINVAL;
	}

	if (!opts->full_auxtrace)
		return 0;

	if (opts->full_auxtrace && !cpu_map__empty(cpus)) {
		pr_err(INTEL_BTS_PMU_NAME " does not support per-cpu recording\n");
		return -EINVAL;
	}

	/* Set default sizes for snapshot mode */
	if (opts->auxtrace_snapshot_mode) {
		if (!opts->auxtrace_snapshot_size && !opts->auxtrace_mmap_pages) {
			if (privileged) {
				opts->auxtrace_mmap_pages = MiB(4) / page_size;
			} else {
				opts->auxtrace_mmap_pages = KiB(128) / page_size;
				if (opts->mmap_pages == UINT_MAX)
					opts->mmap_pages = KiB(256) / page_size;
			}
		} else if (!opts->auxtrace_mmap_pages && !privileged &&
			   opts->mmap_pages == UINT_MAX) {
			opts->mmap_pages = KiB(256) / page_size;
		}
		if (!opts->auxtrace_snapshot_size)
			opts->auxtrace_snapshot_size =
				opts->auxtrace_mmap_pages * (size_t)page_size;
		if (!opts->auxtrace_mmap_pages) {
			size_t sz = opts->auxtrace_snapshot_size;

			sz = round_up(sz, page_size) / page_size;
			opts->auxtrace_mmap_pages = roundup_pow_of_two(sz);
		}
		if (opts->auxtrace_snapshot_size >
				opts->auxtrace_mmap_pages * (size_t)page_size) {
			pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n",
			       opts->auxtrace_snapshot_size,
			       opts->auxtrace_mmap_pages * (size_t)page_size);
			return -EINVAL;
		}
		if (!opts->auxtrace_snapshot_size || !opts->auxtrace_mmap_pages) {
			pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n");
			return -EINVAL;
		}
		pr_debug2("Intel BTS snapshot size: %zu\n",
			  opts->auxtrace_snapshot_size);
	}

	/* Set default sizes for full trace mode */
	if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) {
		if (privileged) {
			opts->auxtrace_mmap_pages = MiB(4) / page_size;
		} else {
			opts->auxtrace_mmap_pages = KiB(128) / page_size;
			if (opts->mmap_pages == UINT_MAX)
				opts->mmap_pages = KiB(256) / page_size;
		}
	}

	/* Validate auxtrace_mmap_pages */
	if (opts->auxtrace_mmap_pages) {
		size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size;
		size_t min_sz;

		if (opts->auxtrace_snapshot_mode)
			min_sz = KiB(4);
		else
			min_sz = KiB(8);

		if (sz < min_sz || !is_power_of_2(sz)) {
			pr_err("Invalid mmap size for Intel BTS: must be at least %zuKiB and a power of 2\n",
			       min_sz / 1024);
			return -EINVAL;
		}
	}

	if (intel_bts_evsel) {
		/*
		 * To obtain the auxtrace buffer file descriptor, the auxtrace event
		 * must come first.
		 */
		perf_evlist__to_front(evlist, intel_bts_evsel);
		/*
		 * In the case of per-cpu mmaps, we need the CPU on the
		 * AUX event.
		 */
		if (!cpu_map__empty(cpus))
			perf_evsel__set_sample_bit(intel_bts_evsel, CPU);
	}

	/* Add dummy event to keep tracking */
	if (opts->full_auxtrace) {
		struct perf_evsel *tracking_evsel;
		int err;

		err = parse_events(evlist, "dummy:u", NULL);
		if (err)
			return err;

		tracking_evsel = perf_evlist__last(evlist);

		perf_evlist__set_tracking_event(evlist, tracking_evsel);

		tracking_evsel->attr.freq = 0;
		tracking_evsel->attr.sample_period = 1;
	}

	return 0;
}

static int intel_bts_parse_snapshot_options(struct auxtrace_record *itr,
					    struct record_opts *opts,
					    const char *str)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);
	unsigned long long snapshot_size = 0;
	char *endptr;

	if (str) {
		snapshot_size = strtoull(str, &endptr, 0);
		if (*endptr || snapshot_size > SIZE_MAX)
			return -1;
	}

	opts->auxtrace_snapshot_mode = true;
	opts->auxtrace_snapshot_size = snapshot_size;

	btsr->snapshot_size = snapshot_size;

	return 0;
}

static u64 intel_bts_reference(struct auxtrace_record *itr __maybe_unused)
{
	return rdtsc();
}

static int intel_bts_alloc_snapshot_refs(struct intel_bts_recording *btsr,
					 int idx)
{
	const size_t sz = sizeof(struct intel_bts_snapshot_ref);
	int cnt = btsr->snapshot_ref_cnt, new_cnt = cnt * 2;
	struct intel_bts_snapshot_ref *refs;

	if (!new_cnt)
		new_cnt = 16;

	while (new_cnt <= idx)
		new_cnt *= 2;

	refs = calloc(new_cnt, sz);
	if (!refs)
		return -ENOMEM;

	memcpy(refs, btsr->snapshot_refs, cnt * sz);

	btsr->snapshot_refs = refs;
	btsr->snapshot_ref_cnt = new_cnt;

	return 0;
}

static void intel_bts_free_snapshot_refs(struct intel_bts_recording *btsr)
{
	int i;

	for (i = 0; i < btsr->snapshot_ref_cnt; i++)
		zfree(&btsr->snapshot_refs[i].ref_buf);
	zfree(&btsr->snapshot_refs);
}

static void intel_bts_recording_free(struct auxtrace_record *itr)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);

	intel_bts_free_snapshot_refs(btsr);
	free(btsr);
}

static int intel_bts_snapshot_start(struct auxtrace_record *itr)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);
	struct perf_evsel *evsel;

	evlist__for_each_entry(btsr->evlist, evsel) {
		if (evsel->attr.type == btsr->intel_bts_pmu->type)
			return perf_evsel__disable(evsel);
	}
	return -EINVAL;
}

static int intel_bts_snapshot_finish(struct auxtrace_record *itr)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);
	struct perf_evsel *evsel;

	evlist__for_each_entry(btsr->evlist, evsel) {
		if (evsel->attr.type == btsr->intel_bts_pmu->type)
			return perf_evsel__enable(evsel);
	}
	return -EINVAL;
}

static bool intel_bts_first_wrap(u64 *data, size_t buf_size)
{
	int i, a, b;

	b = buf_size >> 3;
	a = b - 512;
	if (a < 0)
		a = 0;

	for (i = a; i < b; i++) {
		if (data[i])
			return true;
	}

	return false;
}

static int intel_bts_find_snapshot(struct auxtrace_record *itr, int idx,
				   struct auxtrace_mmap *mm, unsigned char *data,
				   u64 *head, u64 *old)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);
	bool wrapped;
	int err;

	pr_debug3("%s: mmap index %d old head %zu new head %zu\n",
		  __func__, idx, (size_t)*old, (size_t)*head);

	if (idx >= btsr->snapshot_ref_cnt) {
		err = intel_bts_alloc_snapshot_refs(btsr, idx);
		if (err)
			goto out_err;
	}

	wrapped = btsr->snapshot_refs[idx].wrapped;
	if (!wrapped && intel_bts_first_wrap((u64 *)data, mm->len)) {
		btsr->snapshot_refs[idx].wrapped = true;
		wrapped = true;
	}

	/*
	 * In full trace mode 'head' continually increases.  However in snapshot
	 * mode 'head' is an offset within the buffer.  Here 'old' and 'head'
	 * are adjusted to match the full trace case which expects that 'old' is
	 * always less than 'head'.
	 */
	if (wrapped) {
		*old = *head;
		*head += mm->len;
	} else {
		if (mm->mask)
			*old &= mm->mask;
		else
			*old %= mm->len;
		if (*old > *head)
			*head += mm->len;
	}

	pr_debug3("%s: wrap-around %sdetected, adjusted old head %zu adjusted new head %zu\n",
		  __func__, wrapped ? "" : "not ", (size_t)*old, (size_t)*head);

	return 0;

out_err:
	pr_err("%s: failed, error %d\n", __func__, err);
	return err;
}

static int intel_bts_read_finish(struct auxtrace_record *itr, int idx)
{
	struct intel_bts_recording *btsr =
			container_of(itr, struct intel_bts_recording, itr);
	struct perf_evsel *evsel;

	evlist__for_each_entry(btsr->evlist, evsel) {
		if (evsel->attr.type == btsr->intel_bts_pmu->type)
			return perf_evlist__enable_event_idx(btsr->evlist,
							     evsel, idx);
	}
	return -EINVAL;
}

struct auxtrace_record *intel_bts_recording_init(int *err)
{
	struct perf_pmu *intel_bts_pmu = perf_pmu__find(INTEL_BTS_PMU_NAME);
	struct intel_bts_recording *btsr;

	if (!intel_bts_pmu)
		return NULL;

	if (setenv("JITDUMP_USE_ARCH_TIMESTAMP", "1", 1)) {
		*err = -errno;
		return NULL;
	}

	btsr = zalloc(sizeof(struct intel_bts_recording));
	if (!btsr) {
		*err = -ENOMEM;
		return NULL;
	}

	btsr->intel_bts_pmu = intel_bts_pmu;
	btsr->itr.recording_options = intel_bts_recording_options;
	btsr->itr.info_priv_size = intel_bts_info_priv_size;
	btsr->itr.info_fill = intel_bts_info_fill;
	btsr->itr.free = intel_bts_recording_free;
	btsr->itr.snapshot_start = intel_bts_snapshot_start;
	btsr->itr.snapshot_finish = intel_bts_snapshot_finish;
	btsr->itr.find_snapshot = intel_bts_find_snapshot;
	btsr->itr.parse_snapshot_options = intel_bts_parse_snapshot_options;
	btsr->itr.reference = intel_bts_reference;
	btsr->itr.read_finish = intel_bts_read_finish;
	btsr->itr.alignment = sizeof(struct branch);
	return &btsr->itr;
}
Commit	Line	Data
2025cf9e	1	// SPDX-License-Identifier: GPL-2.0-only
d0170af7 AH	2	/*
	3	* intel-bts.c: Intel Processor Trace support
	4	* Copyright (c) 2013-2015, Intel Corporation.
d0170af7 AH	5	*/
d0170af7 AH	6
a43783ae	7	#include <errno.h>
d0170af7 AH	8	#include <linux/kernel.h>
	9	#include <linux/types.h>
	10	#include <linux/bitops.h>
	11	#include <linux/log2.h>
	12
	13	#include "../../util/cpumap.h"
	14	#include "../../util/evsel.h"
	15	#include "../../util/evlist.h"
	16	#include "../../util/session.h"
	17	#include "../../util/util.h"
	18	#include "../../util/pmu.h"
	19	#include "../../util/debug.h"
	20	#include "../../util/tsc.h"
	21	#include "../../util/auxtrace.h"
	22	#include "../../util/intel-bts.h"
	23
	24	#define KiB(x) ((x) * 1024)
	25	#define MiB(x) ((x) * 1024 * 1024)
	26	#define KiB_MASK(x) (KiB(x) - 1)
	27	#define MiB_MASK(x) (MiB(x) - 1)
	28
d0170af7 AH	29	struct intel_bts_snapshot_ref {
	30	void *ref_buf;
	31	size_t ref_offset;
	32	bool wrapped;
	33	};
	34
	35	struct intel_bts_recording {
	36	struct auxtrace_record itr;
	37	struct perf_pmu *intel_bts_pmu;
	38	struct perf_evlist *evlist;
	39	bool snapshot_mode;
	40	size_t snapshot_size;
	41	int snapshot_ref_cnt;
	42	struct intel_bts_snapshot_ref *snapshot_refs;
	43	};
	44
	45	struct branch {
	46	u64 from;
	47	u64 to;
	48	u64 misc;
	49	};
	50
14a05e13 MP	51	static size_t
	52	intel_bts_info_priv_size(struct auxtrace_record *itr __maybe_unused,
	53	struct perf_evlist *evlist __maybe_unused)
d0170af7 AH	54	{
	55	return INTEL_BTS_AUXTRACE_PRIV_SIZE;
	56	}
	57
	58	static int intel_bts_info_fill(struct auxtrace_record *itr,
	59	struct perf_session *session,
	60	struct auxtrace_info_event *auxtrace_info,
	61	size_t priv_size)
	62	{
	63	struct intel_bts_recording *btsr =
	64	container_of(itr, struct intel_bts_recording, itr);
	65	struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu;
	66	struct perf_event_mmap_page *pc;
	67	struct perf_tsc_conversion tc = { .time_mult = 0, };
	68	bool cap_user_time_zero = false;
	69	int err;
	70
	71	if (priv_size != INTEL_BTS_AUXTRACE_PRIV_SIZE)
	72	return -EINVAL;
	73
	74	if (!session->evlist->nr_mmaps)
	75	return -EINVAL;
	76
	77	pc = session->evlist->mmap[0].base;
	78	if (pc) {
	79	err = perf_read_tsc_conversion(pc, &tc);
	80	if (err) {
	81	if (err != -EOPNOTSUPP)
	82	return err;
	83	} else {
	84	cap_user_time_zero = tc.time_mult != 0;
	85	}
	86	if (!cap_user_time_zero)
	87	ui__warning("Intel BTS: TSC not available\n");
	88	}
	89
	90	auxtrace_info->type = PERF_AUXTRACE_INTEL_BTS;
	91	auxtrace_info->priv[INTEL_BTS_PMU_TYPE] = intel_bts_pmu->type;
	92	auxtrace_info->priv[INTEL_BTS_TIME_SHIFT] = tc.time_shift;
	93	auxtrace_info->priv[INTEL_BTS_TIME_MULT] = tc.time_mult;
	94	auxtrace_info->priv[INTEL_BTS_TIME_ZERO] = tc.time_zero;
	95	auxtrace_info->priv[INTEL_BTS_CAP_USER_TIME_ZERO] = cap_user_time_zero;
	96	auxtrace_info->priv[INTEL_BTS_SNAPSHOT_MODE] = btsr->snapshot_mode;
	97
	98	return 0;
	99	}
	100
	101	static int intel_bts_recording_options(struct auxtrace_record *itr,
	102	struct perf_evlist *evlist,
	103	struct record_opts *opts)
	104	{
	105	struct intel_bts_recording *btsr =
	106	container_of(itr, struct intel_bts_recording, itr);
	107	struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu;
	108	struct perf_evsel evsel, intel_bts_evsel = NULL;
	109	const struct cpu_map *cpus = evlist->cpus;
	110	bool privileged = geteuid() == 0 \|\| perf_event_paranoid() < 0;
	111
	112	btsr->evlist = evlist;
	113	btsr->snapshot_mode = opts->auxtrace_snapshot_mode;
	114
e5cadb93	115	evlist__for_each_entry(evlist, evsel) {
d0170af7 AH	116	if (evsel->attr.type == intel_bts_pmu->type) {
	117	if (intel_bts_evsel) {
	118	pr_err("There may be only one " INTEL_BTS_PMU_NAME " event\n");
	119	return -EINVAL;
	120	}
	121	evsel->attr.freq = 0;
	122	evsel->attr.sample_period = 1;
	123	intel_bts_evsel = evsel;
	124	opts->full_auxtrace = true;
	125	}
	126	}
	127
	128	if (opts->auxtrace_snapshot_mode && !opts->full_auxtrace) {
	129	pr_err("Snapshot mode (-S option) requires " INTEL_BTS_PMU_NAME " PMU event (-e " INTEL_BTS_PMU_NAME ")\n");
	130	return -EINVAL;
	131	}
	132
	133	if (!opts->full_auxtrace)
	134	return 0;
	135
	136	if (opts->full_auxtrace && !cpu_map__empty(cpus)) {
	137	pr_err(INTEL_BTS_PMU_NAME " does not support per-cpu recording\n");
	138	return -EINVAL;
	139	}
	140
	141	/* Set default sizes for snapshot mode */
	142	if (opts->auxtrace_snapshot_mode) {
	143	if (!opts->auxtrace_snapshot_size && !opts->auxtrace_mmap_pages) {
	144	if (privileged) {
	145	opts->auxtrace_mmap_pages = MiB(4) / page_size;
	146	} else {
	147	opts->auxtrace_mmap_pages = KiB(128) / page_size;
	148	if (opts->mmap_pages == UINT_MAX)
	149	opts->mmap_pages = KiB(256) / page_size;
	150	}
	151	} else if (!opts->auxtrace_mmap_pages && !privileged &&
	152	opts->mmap_pages == UINT_MAX) {
	153	opts->mmap_pages = KiB(256) / page_size;
	154	}
	155	if (!opts->auxtrace_snapshot_size)
	156	opts->auxtrace_snapshot_size =
	157	opts->auxtrace_mmap_pages * (size_t)page_size;
	158	if (!opts->auxtrace_mmap_pages) {
	159	size_t sz = opts->auxtrace_snapshot_size;
	160
	161	sz = round_up(sz, page_size) / page_size;
	162	opts->auxtrace_mmap_pages = roundup_pow_of_two(sz);
	163	}
	164	if (opts->auxtrace_snapshot_size >
	165	opts->auxtrace_mmap_pages * (size_t)page_size) {
	166	pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n",
	167	opts->auxtrace_snapshot_size,
	168	opts->auxtrace_mmap_pages * (size_t)page_size);
	169	return -EINVAL;
	170	}
	171	if (!opts->auxtrace_snapshot_size \|\| !opts->auxtrace_mmap_pages) {
	172	pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n");
	173	return -EINVAL;
	174	}
	175	pr_debug2("Intel BTS snapshot size: %zu\n",
	176	opts->auxtrace_snapshot_size);
	177	}
	178
	179	/* Set default sizes for full trace mode */
180	if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) {
181	if (privileged) {
182	opts->auxtrace_mmap_pages = MiB(4) / page_size;
183	} else {
184	opts->auxtrace_mmap_pages = KiB(128) / page_size;
185	if (opts->mmap_pages == UINT_MAX)
186	opts->mmap_pages = KiB(256) / page_size;
187	}
188	}
189
190	/* Validate auxtrace_mmap_pages */
191	if (opts->auxtrace_mmap_pages) {
192	size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size;
193	size_t min_sz;
194
195	if (opts->auxtrace_snapshot_mode)
196	min_sz = KiB(4);
197	else
198	min_sz = KiB(8);
199
200	if (sz < min_sz \|\| !is_power_of_2(sz)) {
201	pr_err("Invalid mmap size for Intel BTS: must be at least %zuKiB and a power of 2\n",
202	min_sz / 1024);
203	return -EINVAL;
204	}
205	}
206
207	if (intel_bts_evsel) {
208	/*
209	* To obtain the auxtrace buffer file descriptor, the auxtrace event
210	* must come first.
211	*/
212	perf_evlist__to_front(evlist, intel_bts_evsel);
213	/*
214	* In the case of per-cpu mmaps, we need the CPU on the
215	* AUX event.
216	*/
217	if (!cpu_map__empty(cpus))
218	perf_evsel__set_sample_bit(intel_bts_evsel, CPU);
219	}
220
221	/* Add dummy event to keep tracking */
222	if (opts->full_auxtrace) {
223	struct perf_evsel *tracking_evsel;
224	int err;
225
226	err = parse_events(evlist, "dummy:u", NULL);
227	if (err)
228	return err;
229
230	tracking_evsel = perf_evlist__last(evlist);
231
232	perf_evlist__set_tracking_event(evlist, tracking_evsel);
233
234	tracking_evsel->attr.freq = 0;
235	tracking_evsel->attr.sample_period = 1;
236	}
237
238	return 0;
239	}
240
241	static int intel_bts_parse_snapshot_options(struct auxtrace_record *itr,
242	struct record_opts *opts,
243	const char *str)
244	{
245	struct intel_bts_recording *btsr =
246	container_of(itr, struct intel_bts_recording, itr);
247	unsigned long long snapshot_size = 0;
248	char *endptr;
249
250	if (str) {
251	snapshot_size = strtoull(str, &endptr, 0);
252	if (*endptr \|\| snapshot_size > SIZE_MAX)
253	return -1;
254	}
255
256	opts->auxtrace_snapshot_mode = true;
257	opts->auxtrace_snapshot_size = snapshot_size;
258
259	btsr->snapshot_size = snapshot_size;
260
261	return 0;
262	}
263
264	static u64 intel_bts_reference(struct auxtrace_record *itr __maybe_unused)
265	{
266	return rdtsc();
267	}
268
269	static int intel_bts_alloc_snapshot_refs(struct intel_bts_recording *btsr,
270	int idx)
271	{
272	const size_t sz = sizeof(struct intel_bts_snapshot_ref);
273	int cnt = btsr->snapshot_ref_cnt, new_cnt = cnt * 2;
274	struct intel_bts_snapshot_ref *refs;
275
276	if (!new_cnt)
277	new_cnt = 16;
278
279	while (new_cnt <= idx)
280	new_cnt *= 2;
281
282	refs = calloc(new_cnt, sz);
283	if (!refs)
284	return -ENOMEM;
285
286	memcpy(refs, btsr->snapshot_refs, cnt * sz);
287
288	btsr->snapshot_refs = refs;
289	btsr->snapshot_ref_cnt = new_cnt;
290
291	return 0;
292	}
293
294	static void intel_bts_free_snapshot_refs(struct intel_bts_recording *btsr)
295	{
296	int i;
297
298	for (i = 0; i < btsr->snapshot_ref_cnt; i++)
299	zfree(&btsr->snapshot_refs[i].ref_buf);
300	zfree(&btsr->snapshot_refs);
301	}
302
303	static void intel_bts_recording_free(struct auxtrace_record *itr)
304	{
305	struct intel_bts_recording *btsr =
306	container_of(itr, struct intel_bts_recording, itr);
307
308	intel_bts_free_snapshot_refs(btsr);
309	free(btsr);
310	}
311
312	static int intel_bts_snapshot_start(struct auxtrace_record *itr)
313	{
314	struct intel_bts_recording *btsr =
315	container_of(itr, struct intel_bts_recording, itr);
316	struct perf_evsel *evsel;
317
e5cadb93	318	evlist__for_each_entry(btsr->evlist, evsel) {
d0170af7	319	if (evsel->attr.type == btsr->intel_bts_pmu->type)
d2190a80	320	return perf_evsel__disable(evsel);
d0170af7 AH	321	}
	322	return -EINVAL;
	323	}
	324
	325	static int intel_bts_snapshot_finish(struct auxtrace_record *itr)
	326	{
	327	struct intel_bts_recording *btsr =
	328	container_of(itr, struct intel_bts_recording, itr);
	329	struct perf_evsel *evsel;
	330
e5cadb93	331	evlist__for_each_entry(btsr->evlist, evsel) {
d0170af7	332	if (evsel->attr.type == btsr->intel_bts_pmu->type)
d2190a80	333	return perf_evsel__enable(evsel);
d0170af7 AH	334	}
	335	return -EINVAL;
	336	}
	337
	338	static bool intel_bts_first_wrap(u64 *data, size_t buf_size)
	339	{
	340	int i, a, b;
	341
	342	b = buf_size >> 3;
	343	a = b - 512;
	344	if (a < 0)
	345	a = 0;
	346
	347	for (i = a; i < b; i++) {
	348	if (data[i])
	349	return true;
	350	}
	351
	352	return false;
	353	}
	354
	355	static int intel_bts_find_snapshot(struct auxtrace_record *itr, int idx,
	356	struct auxtrace_mmap mm, unsigned char data,
	357	u64 head, u64 old)
	358	{
	359	struct intel_bts_recording *btsr =
	360	container_of(itr, struct intel_bts_recording, itr);
	361	bool wrapped;
	362	int err;
	363
	364	pr_debug3("%s: mmap index %d old head %zu new head %zu\n",
	365	__func__, idx, (size_t)old, (size_t)head);
	366
	367	if (idx >= btsr->snapshot_ref_cnt) {
	368	err = intel_bts_alloc_snapshot_refs(btsr, idx);
	369	if (err)
	370	goto out_err;
	371	}
	372
	373	wrapped = btsr->snapshot_refs[idx].wrapped;
	374	if (!wrapped && intel_bts_first_wrap((u64 *)data, mm->len)) {
	375	btsr->snapshot_refs[idx].wrapped = true;
	376	wrapped = true;
	377	}
	378
	379	/*
	380	* In full trace mode 'head' continually increases. However in snapshot
	381	* mode 'head' is an offset within the buffer. Here 'old' and 'head'
	382	* are adjusted to match the full trace case which expects that 'old' is
	383	* always less than 'head'.
	384	*/
	385	if (wrapped) {
	386	old = head;
	387	*head += mm->len;
	388	} else {
	389	if (mm->mask)
	390	*old &= mm->mask;
	391	else
	392	*old %= mm->len;
	393	if (old > head)
	394	*head += mm->len;
	395	}
	396
	397	pr_debug3("%s: wrap-around %sdetected, adjusted old head %zu adjusted new head %zu\n",
398	__func__, wrapped ? "" : "not ", (size_t)old, (size_t)head);
399
400	return 0;
401
402	out_err:
403	pr_err("%s: failed, error %d\n", __func__, err);
404	return err;
405	}
406
407	static int intel_bts_read_finish(struct auxtrace_record *itr, int idx)
408	{
409	struct intel_bts_recording *btsr =
410	container_of(itr, struct intel_bts_recording, itr);
411	struct perf_evsel *evsel;
412
e5cadb93	413	evlist__for_each_entry(btsr->evlist, evsel) {
d0170af7 AH	414	if (evsel->attr.type == btsr->intel_bts_pmu->type)
	415	return perf_evlist__enable_event_idx(btsr->evlist,
	416	evsel, idx);
	417	}
	418	return -EINVAL;
	419	}
	420
	421	struct auxtrace_record intel_bts_recording_init(int err)
	422	{
	423	struct perf_pmu *intel_bts_pmu = perf_pmu__find(INTEL_BTS_PMU_NAME);
	424	struct intel_bts_recording *btsr;
	425
	426	if (!intel_bts_pmu)
	427	return NULL;
	428
bd0c7a54 AH	429	if (setenv("JITDUMP_USE_ARCH_TIMESTAMP", "1", 1)) {
	430	*err = -errno;
	431	return NULL;
	432	}
	433
d0170af7 AH	434	btsr = zalloc(sizeof(struct intel_bts_recording));
	435	if (!btsr) {
	436	*err = -ENOMEM;
	437	return NULL;
	438	}
	439
	440	btsr->intel_bts_pmu = intel_bts_pmu;
	441	btsr->itr.recording_options = intel_bts_recording_options;
	442	btsr->itr.info_priv_size = intel_bts_info_priv_size;
	443	btsr->itr.info_fill = intel_bts_info_fill;
	444	btsr->itr.free = intel_bts_recording_free;
	445	btsr->itr.snapshot_start = intel_bts_snapshot_start;
	446	btsr->itr.snapshot_finish = intel_bts_snapshot_finish;
	447	btsr->itr.find_snapshot = intel_bts_find_snapshot;
	448	btsr->itr.parse_snapshot_options = intel_bts_parse_snapshot_options;
	449	btsr->itr.reference = intel_bts_reference;
	450	btsr->itr.read_finish = intel_bts_read_finish;
	451	btsr->itr.alignment = sizeof(struct branch);
	452	return &btsr->itr;
	453	}