[linux-2.6-block.git] / tools / perf / util / mem-events.c

#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <api/fs/fs.h>
#include "mem-events.h"
#include "debug.h"
#include "symbol.h"
#include "sort.h"

unsigned int perf_mem_events__loads_ldlat = 30;

#define E(t, n, s) { .tag = t, .name = n, .sysfs_name = s }

struct perf_mem_event perf_mem_events[PERF_MEM_EVENTS__MAX] = {
	E("ldlat-loads",	"cpu/mem-loads,ldlat=%u/P",	"mem-loads"),
	E("ldlat-stores",	"cpu/mem-stores/P",		"mem-stores"),
};
#undef E

#undef E

static char mem_loads_name[100];
static bool mem_loads_name__init;

char *perf_mem_events__name(int i)
{
	if (i == PERF_MEM_EVENTS__LOAD) {
		if (!mem_loads_name__init) {
			mem_loads_name__init = true;
			scnprintf(mem_loads_name, sizeof(mem_loads_name),
				  perf_mem_events[i].name,
				  perf_mem_events__loads_ldlat);
		}
		return mem_loads_name;
	}

	return (char *)perf_mem_events[i].name;
}

int perf_mem_events__parse(const char *str)
{
	char *tok, *saveptr = NULL;
	bool found = false;
	char *buf;
	int j;

	/* We need buffer that we know we can write to. */
	buf = malloc(strlen(str) + 1);
	if (!buf)
		return -ENOMEM;

	strcpy(buf, str);

	tok = strtok_r((char *)buf, ",", &saveptr);

	while (tok) {
		for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) {
			struct perf_mem_event *e = &perf_mem_events[j];

			if (strstr(e->tag, tok))
				e->record = found = true;
		}

		tok = strtok_r(NULL, ",", &saveptr);
	}

	free(buf);

	if (found)
		return 0;

	pr_err("failed: event '%s' not found, use '-e list' to get list of available events\n", str);
	return -1;
}

int perf_mem_events__init(void)
{
	const char *mnt = sysfs__mount();
	bool found = false;
	int j;

	if (!mnt)
		return -ENOENT;

	for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) {
		char path[PATH_MAX];
		struct perf_mem_event *e = &perf_mem_events[j];
		struct stat st;

		scnprintf(path, PATH_MAX, "%s/devices/cpu/events/%s",
			  mnt, e->sysfs_name);

		if (!stat(path, &st))
			e->supported = found = true;
	}

	return found ? 0 : -ENOENT;
}

static const char * const tlb_access[] = {
	"N/A",
	"HIT",
	"MISS",
	"L1",
	"L2",
	"Walker",
	"Fault",
};

int perf_mem__tlb_scnprintf(char *out, size_t sz, struct mem_info *mem_info)
{
	size_t l = 0, i;
	u64 m = PERF_MEM_TLB_NA;
	u64 hit, miss;

	sz -= 1; /* -1 for null termination */
	out[0] = '\0';

	if (mem_info)
		m = mem_info->data_src.mem_dtlb;

	hit = m & PERF_MEM_TLB_HIT;
	miss = m & PERF_MEM_TLB_MISS;

	/* already taken care of */
	m &= ~(PERF_MEM_TLB_HIT|PERF_MEM_TLB_MISS);

	for (i = 0; m && i < ARRAY_SIZE(tlb_access); i++, m >>= 1) {
		if (!(m & 0x1))
			continue;
		if (l) {
			strcat(out, " or ");
			l += 4;
		}
		l += scnprintf(out + l, sz - l, tlb_access[i]);
	}
	if (*out == '\0')
		l += scnprintf(out, sz - l, "N/A");
	if (hit)
		l += scnprintf(out + l, sz - l, " hit");
	if (miss)
		l += scnprintf(out + l, sz - l, " miss");

	return l;
}

static const char * const mem_lvl[] = {
	"N/A",
	"HIT",
	"MISS",
	"L1",
	"LFB",
	"L2",
	"L3",
	"Local RAM",
	"Remote RAM (1 hop)",
	"Remote RAM (2 hops)",
	"Remote Cache (1 hop)",
	"Remote Cache (2 hops)",
	"I/O",
	"Uncached",
};

int perf_mem__lvl_scnprintf(char *out, size_t sz, struct mem_info *mem_info)
{
	size_t i, l = 0;
	u64 m =  PERF_MEM_LVL_NA;
	u64 hit, miss;

	if (mem_info)
		m  = mem_info->data_src.mem_lvl;

	sz -= 1; /* -1 for null termination */
	out[0] = '\0';

	hit = m & PERF_MEM_LVL_HIT;
	miss = m & PERF_MEM_LVL_MISS;

	/* already taken care of */
	m &= ~(PERF_MEM_LVL_HIT|PERF_MEM_LVL_MISS);

	for (i = 0; m && i < ARRAY_SIZE(mem_lvl); i++, m >>= 1) {
		if (!(m & 0x1))
			continue;
		if (l) {
			strcat(out, " or ");
			l += 4;
		}
		l += scnprintf(out + l, sz - l, mem_lvl[i]);
	}
	if (*out == '\0')
		l += scnprintf(out, sz - l, "N/A");
	if (hit)
		l += scnprintf(out + l, sz - l, " hit");
	if (miss)
		l += scnprintf(out + l, sz - l, " miss");

	return l;
}

static const char * const snoop_access[] = {
	"N/A",
	"None",
	"Miss",
	"Hit",
	"HitM",
};

int perf_mem__snp_scnprintf(char *out, size_t sz, struct mem_info *mem_info)
{
	size_t i, l = 0;
	u64 m = PERF_MEM_SNOOP_NA;

	sz -= 1; /* -1 for null termination */
	out[0] = '\0';

	if (mem_info)
		m = mem_info->data_src.mem_snoop;

	for (i = 0; m && i < ARRAY_SIZE(snoop_access); i++, m >>= 1) {
		if (!(m & 0x1))
			continue;
		if (l) {
			strcat(out, " or ");
			l += 4;
		}
		l += scnprintf(out + l, sz - l, snoop_access[i]);
	}

	if (*out == '\0')
		l += scnprintf(out, sz - l, "N/A");

	return l;
}

int perf_mem__lck_scnprintf(char *out, size_t sz, struct mem_info *mem_info)
{
	u64 mask = PERF_MEM_LOCK_NA;
	int l;

	if (mem_info)
		mask = mem_info->data_src.mem_lock;

	if (mask & PERF_MEM_LOCK_NA)
		l = scnprintf(out, sz, "N/A");
	else if (mask & PERF_MEM_LOCK_LOCKED)
		l = scnprintf(out, sz, "Yes");
	else
		l = scnprintf(out, sz, "No");

	return l;
}

int perf_script__meminfo_scnprintf(char *out, size_t sz, struct mem_info *mem_info)
{
	int i = 0;

	i += perf_mem__lvl_scnprintf(out, sz, mem_info);
	i += scnprintf(out + i, sz - i, "|SNP ");
	i += perf_mem__snp_scnprintf(out + i, sz - i, mem_info);
	i += scnprintf(out + i, sz - i, "|TLB ");
	i += perf_mem__tlb_scnprintf(out + i, sz - i, mem_info);
	i += scnprintf(out + i, sz - i, "|LCK ");
	i += perf_mem__lck_scnprintf(out + i, sz - i, mem_info);

	return i;
}

int c2c_decode_stats(struct c2c_stats *stats, struct mem_info *mi)
{
	union perf_mem_data_src *data_src = &mi->data_src;
	u64 daddr  = mi->daddr.addr;
	u64 op     = data_src->mem_op;
	u64 lvl    = data_src->mem_lvl;
	u64 snoop  = data_src->mem_snoop;
	u64 lock   = data_src->mem_lock;
	int err = 0;

#define HITM_INC(__f)		\
do {				\
	stats->__f++;		\
	stats->tot_hitm++;	\
} while (0)

#define P(a, b) PERF_MEM_##a##_##b

	stats->nr_entries++;

	if (lock & P(LOCK, LOCKED)) stats->locks++;

	if (op & P(OP, LOAD)) {
		/* load */
		stats->load++;

		if (!daddr) {
			stats->ld_noadrs++;
			return -1;
		}

		if (lvl & P(LVL, HIT)) {
			if (lvl & P(LVL, UNC)) stats->ld_uncache++;
			if (lvl & P(LVL, IO))  stats->ld_io++;
			if (lvl & P(LVL, LFB)) stats->ld_fbhit++;
			if (lvl & P(LVL, L1 )) stats->ld_l1hit++;
			if (lvl & P(LVL, L2 )) stats->ld_l2hit++;
			if (lvl & P(LVL, L3 )) {
				if (snoop & P(SNOOP, HITM))
					HITM_INC(lcl_hitm);
				else
					stats->ld_llchit++;
			}

			if (lvl & P(LVL, LOC_RAM)) {
				stats->lcl_dram++;
				if (snoop & P(SNOOP, HIT))
					stats->ld_shared++;
				else
					stats->ld_excl++;
			}

			if ((lvl & P(LVL, REM_RAM1)) ||
			    (lvl & P(LVL, REM_RAM2))) {
				stats->rmt_dram++;
				if (snoop & P(SNOOP, HIT))
					stats->ld_shared++;
				else
					stats->ld_excl++;
			}
		}

		if ((lvl & P(LVL, REM_CCE1)) ||
		    (lvl & P(LVL, REM_CCE2))) {
			if (snoop & P(SNOOP, HIT))
				stats->rmt_hit++;
			else if (snoop & P(SNOOP, HITM))
				HITM_INC(rmt_hitm);
		}

		if ((lvl & P(LVL, MISS)))
			stats->ld_miss++;

	} else if (op & P(OP, STORE)) {
		/* store */
		stats->store++;

		if (!daddr) {
			stats->st_noadrs++;
			return -1;
		}

		if (lvl & P(LVL, HIT)) {
			if (lvl & P(LVL, UNC)) stats->st_uncache++;
			if (lvl & P(LVL, L1 )) stats->st_l1hit++;
		}
		if (lvl & P(LVL, MISS))
			if (lvl & P(LVL, L1)) stats->st_l1miss++;
	} else {
		/* unparsable data_src? */
		stats->noparse++;
		return -1;
	}

	if (!mi->daddr.map || !mi->iaddr.map) {
		stats->nomap++;
		return -1;
	}

#undef P
#undef HITM_INC
	return err;
}

void c2c_add_stats(struct c2c_stats *stats, struct c2c_stats *add)
{
	stats->nr_entries	+= add->nr_entries;

	stats->locks		+= add->locks;
	stats->store		+= add->store;
	stats->st_uncache	+= add->st_uncache;
	stats->st_noadrs	+= add->st_noadrs;
	stats->st_l1hit		+= add->st_l1hit;
	stats->st_l1miss	+= add->st_l1miss;
	stats->load		+= add->load;
	stats->ld_excl		+= add->ld_excl;
	stats->ld_shared	+= add->ld_shared;
	stats->ld_uncache	+= add->ld_uncache;
	stats->ld_io		+= add->ld_io;
	stats->ld_miss		+= add->ld_miss;
	stats->ld_noadrs	+= add->ld_noadrs;
	stats->ld_fbhit		+= add->ld_fbhit;
	stats->ld_l1hit		+= add->ld_l1hit;
	stats->ld_l2hit		+= add->ld_l2hit;
	stats->ld_llchit	+= add->ld_llchit;
	stats->lcl_hitm		+= add->lcl_hitm;
	stats->rmt_hitm		+= add->rmt_hitm;
	stats->tot_hitm		+= add->tot_hitm;
	stats->rmt_hit		+= add->rmt_hit;
	stats->lcl_dram		+= add->lcl_dram;
	stats->rmt_dram		+= add->rmt_dram;
	stats->nomap		+= add->nomap;
	stats->noparse		+= add->noparse;
}
Commit	Line	Data
ce1e22b0 JO	1	#include <stddef.h>
	2	#include <stdlib.h>
	3	#include <string.h>
	4	#include <errno.h>
54fbad54 JO	5	#include <sys/types.h>
	6	#include <sys/stat.h>
	7	#include <unistd.h>
	8	#include <api/fs/fs.h>
acbe613e	9	#include "mem-events.h"
ce1e22b0	10	#include "debug.h"
0c877d75	11	#include "symbol.h"
aadddd68	12	#include "sort.h"
acbe613e	13
b0d745b3 JO	14	unsigned int perf_mem_events__loads_ldlat = 30;
b0d745b3 JO	15
54fbad54	16	#define E(t, n, s) { .tag = t, .name = n, .sysfs_name = s }
acbe613e JO	17
acbe613e JO	18	struct perf_mem_event perf_mem_events[PERF_MEM_EVENTS__MAX] = {
b0d745b3	19	E("ldlat-loads", "cpu/mem-loads,ldlat=%u/P", "mem-loads"),
54fbad54	20	E("ldlat-stores", "cpu/mem-stores/P", "mem-stores"),
acbe613e	21	};
54fbad54	22	#undef E
acbe613e JO	23
acbe613e JO	24	#undef E
ce1e22b0	25
b0d745b3 JO	26	static char mem_loads_name[100];
	27	static bool mem_loads_name__init;
	28
2ba7ac58 JO	29	char *perf_mem_events__name(int i)
2ba7ac58 JO	30	{
b0d745b3 JO	31	if (i == PERF_MEM_EVENTS__LOAD) {
	32	if (!mem_loads_name__init) {
	33	mem_loads_name__init = true;
	34	scnprintf(mem_loads_name, sizeof(mem_loads_name),
	35	perf_mem_events[i].name,
	36	perf_mem_events__loads_ldlat);
	37	}
	38	return mem_loads_name;
	39	}
	40
2ba7ac58 JO	41	return (char *)perf_mem_events[i].name;
	42	}
	43
ce1e22b0 JO	44	int perf_mem_events__parse(const char *str)
	45	{
	46	char tok, saveptr = NULL;
	47	bool found = false;
	48	char *buf;
	49	int j;
	50
	51	/* We need buffer that we know we can write to. */
	52	buf = malloc(strlen(str) + 1);
	53	if (!buf)
	54	return -ENOMEM;
	55
	56	strcpy(buf, str);
	57
	58	tok = strtok_r((char *)buf, ",", &saveptr);
	59
	60	while (tok) {
	61	for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) {
	62	struct perf_mem_event *e = &perf_mem_events[j];
	63
	64	if (strstr(e->tag, tok))
	65	e->record = found = true;
	66	}
	67
	68	tok = strtok_r(NULL, ",", &saveptr);
	69	}
	70
	71	free(buf);
	72
	73	if (found)
	74	return 0;
	75
	76	pr_err("failed: event '%s' not found, use '-e list' to get list of available events\n", str);
	77	return -1;
	78	}
54fbad54 JO	79
	80	int perf_mem_events__init(void)
	81	{
	82	const char *mnt = sysfs__mount();
	83	bool found = false;
	84	int j;
	85
	86	if (!mnt)
	87	return -ENOENT;
	88
	89	for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) {
	90	char path[PATH_MAX];
	91	struct perf_mem_event *e = &perf_mem_events[j];
	92	struct stat st;
	93
	94	scnprintf(path, PATH_MAX, "%s/devices/cpu/events/%s",
	95	mnt, e->sysfs_name);
	96
	97	if (!stat(path, &st))
	98	e->supported = found = true;
	99	}
	100
	101	return found ? 0 : -ENOENT;
	102	}
0c877d75 JO	103
	104	static const char * const tlb_access[] = {
	105	"N/A",
	106	"HIT",
	107	"MISS",
	108	"L1",
	109	"L2",
	110	"Walker",
	111	"Fault",
	112	};
	113
b1a5fbea	114	int perf_mem__tlb_scnprintf(char out, size_t sz, struct mem_info mem_info)
0c877d75 JO	115	{
	116	size_t l = 0, i;
	117	u64 m = PERF_MEM_TLB_NA;
	118	u64 hit, miss;
	119
	120	sz -= 1; /* -1 for null termination */
	121	out[0] = '\0';
	122
	123	if (mem_info)
	124	m = mem_info->data_src.mem_dtlb;
	125
	126	hit = m & PERF_MEM_TLB_HIT;
	127	miss = m & PERF_MEM_TLB_MISS;
	128
	129	/* already taken care of */
	130	m &= ~(PERF_MEM_TLB_HIT\|PERF_MEM_TLB_MISS);
	131
	132	for (i = 0; m && i < ARRAY_SIZE(tlb_access); i++, m >>= 1) {
	133	if (!(m & 0x1))
	134	continue;
	135	if (l) {
	136	strcat(out, " or ");
	137	l += 4;
	138	}
b1a5fbea	139	l += scnprintf(out + l, sz - l, tlb_access[i]);
0c877d75 JO	140	}
0c877d75 JO	141	if (*out == '\0')
b1a5fbea	142	l += scnprintf(out, sz - l, "N/A");
0c877d75	143	if (hit)
b1a5fbea	144	l += scnprintf(out + l, sz - l, " hit");
0c877d75	145	if (miss)
b1a5fbea JO	146	l += scnprintf(out + l, sz - l, " miss");
	147
	148	return l;
0c877d75	149	}
071e9a1e JO	150
	151	static const char * const mem_lvl[] = {
	152	"N/A",
	153	"HIT",
	154	"MISS",
	155	"L1",
	156	"LFB",
	157	"L2",
	158	"L3",
	159	"Local RAM",
	160	"Remote RAM (1 hop)",
	161	"Remote RAM (2 hops)",
	162	"Remote Cache (1 hop)",
	163	"Remote Cache (2 hops)",
	164	"I/O",
	165	"Uncached",
	166	};
	167
96907563	168	int perf_mem__lvl_scnprintf(char out, size_t sz, struct mem_info mem_info)
071e9a1e JO	169	{
	170	size_t i, l = 0;
	171	u64 m = PERF_MEM_LVL_NA;
	172	u64 hit, miss;
	173
	174	if (mem_info)
	175	m = mem_info->data_src.mem_lvl;
	176
	177	sz -= 1; /* -1 for null termination */
	178	out[0] = '\0';
	179
	180	hit = m & PERF_MEM_LVL_HIT;
	181	miss = m & PERF_MEM_LVL_MISS;
	182
	183	/* already taken care of */
	184	m &= ~(PERF_MEM_LVL_HIT\|PERF_MEM_LVL_MISS);
	185
	186	for (i = 0; m && i < ARRAY_SIZE(mem_lvl); i++, m >>= 1) {
	187	if (!(m & 0x1))
	188	continue;
	189	if (l) {
	190	strcat(out, " or ");
	191	l += 4;
	192	}
96907563	193	l += scnprintf(out + l, sz - l, mem_lvl[i]);
071e9a1e JO	194	}
071e9a1e JO	195	if (*out == '\0')
96907563	196	l += scnprintf(out, sz - l, "N/A");
071e9a1e	197	if (hit)
96907563	198	l += scnprintf(out + l, sz - l, " hit");
071e9a1e	199	if (miss)
96907563 JO	200	l += scnprintf(out + l, sz - l, " miss");
	201
	202	return l;
071e9a1e	203	}
2c07af13 JO	204
	205	static const char * const snoop_access[] = {
	206	"N/A",
	207	"None",
	208	"Miss",
	209	"Hit",
	210	"HitM",
	211	};
	212
149d7507	213	int perf_mem__snp_scnprintf(char out, size_t sz, struct mem_info mem_info)
2c07af13 JO	214	{
	215	size_t i, l = 0;
	216	u64 m = PERF_MEM_SNOOP_NA;
	217
	218	sz -= 1; /* -1 for null termination */
	219	out[0] = '\0';
	220
	221	if (mem_info)
	222	m = mem_info->data_src.mem_snoop;
	223
	224	for (i = 0; m && i < ARRAY_SIZE(snoop_access); i++, m >>= 1) {
	225	if (!(m & 0x1))
	226	continue;
	227	if (l) {
	228	strcat(out, " or ");
	229	l += 4;
	230	}
149d7507	231	l += scnprintf(out + l, sz - l, snoop_access[i]);
2c07af13 JO	232	}
	233
	234	if (*out == '\0')
149d7507 JO	235	l += scnprintf(out, sz - l, "N/A");
	236
	237	return l;
2c07af13	238	}
69a77275	239
8b0819c8	240	int perf_mem__lck_scnprintf(char out, size_t sz, struct mem_info mem_info)
69a77275 JO	241	{
69a77275 JO	242	u64 mask = PERF_MEM_LOCK_NA;
8b0819c8	243	int l;
69a77275 JO	244
	245	if (mem_info)
	246	mask = mem_info->data_src.mem_lock;
	247
	248	if (mask & PERF_MEM_LOCK_NA)
8b0819c8	249	l = scnprintf(out, sz, "N/A");
69a77275	250	else if (mask & PERF_MEM_LOCK_LOCKED)
8b0819c8	251	l = scnprintf(out, sz, "Yes");
69a77275	252	else
8b0819c8 JO	253	l = scnprintf(out, sz, "No");
	254
	255	return l;
69a77275	256	}
c19ac912 JO	257
	258	int perf_script__meminfo_scnprintf(char out, size_t sz, struct mem_info mem_info)
	259	{
	260	int i = 0;
	261
	262	i += perf_mem__lvl_scnprintf(out, sz, mem_info);
	263	i += scnprintf(out + i, sz - i, "\|SNP ");
	264	i += perf_mem__snp_scnprintf(out + i, sz - i, mem_info);
	265	i += scnprintf(out + i, sz - i, "\|TLB ");
	266	i += perf_mem__tlb_scnprintf(out + i, sz - i, mem_info);
	267	i += scnprintf(out + i, sz - i, "\|LCK ");
	268	i += perf_mem__lck_scnprintf(out + i, sz - i, mem_info);
	269
	270	return i;
	271	}
aadddd68 JO	272
	273	int c2c_decode_stats(struct c2c_stats stats, struct mem_info mi)
	274	{
	275	union perf_mem_data_src *data_src = &mi->data_src;
	276	u64 daddr = mi->daddr.addr;
	277	u64 op = data_src->mem_op;
	278	u64 lvl = data_src->mem_lvl;
	279	u64 snoop = data_src->mem_snoop;
	280	u64 lock = data_src->mem_lock;
	281	int err = 0;
	282
dba8ab93 JO	283	#define HITM_INC(__f) \
	284	do { \
	285	stats->__f++; \
	286	stats->tot_hitm++; \
	287	} while (0)
	288
aadddd68 JO	289	#define P(a, b) PERF_MEM_##a##_##b
	290
	291	stats->nr_entries++;
	292
	293	if (lock & P(LOCK, LOCKED)) stats->locks++;
	294
	295	if (op & P(OP, LOAD)) {
	296	/* load */
	297	stats->load++;
	298
	299	if (!daddr) {
	300	stats->ld_noadrs++;
	301	return -1;
	302	}
	303
	304	if (lvl & P(LVL, HIT)) {
	305	if (lvl & P(LVL, UNC)) stats->ld_uncache++;
	306	if (lvl & P(LVL, IO)) stats->ld_io++;
	307	if (lvl & P(LVL, LFB)) stats->ld_fbhit++;
	308	if (lvl & P(LVL, L1 )) stats->ld_l1hit++;
	309	if (lvl & P(LVL, L2 )) stats->ld_l2hit++;
	310	if (lvl & P(LVL, L3 )) {
	311	if (snoop & P(SNOOP, HITM))
dba8ab93	312	HITM_INC(lcl_hitm);
aadddd68 JO	313	else
	314	stats->ld_llchit++;
	315	}
	316
	317	if (lvl & P(LVL, LOC_RAM)) {
	318	stats->lcl_dram++;
	319	if (snoop & P(SNOOP, HIT))
	320	stats->ld_shared++;
	321	else
	322	stats->ld_excl++;
	323	}
	324
	325	if ((lvl & P(LVL, REM_RAM1)) \|\|
	326	(lvl & P(LVL, REM_RAM2))) {
	327	stats->rmt_dram++;
	328	if (snoop & P(SNOOP, HIT))
	329	stats->ld_shared++;
	330	else
	331	stats->ld_excl++;
	332	}
	333	}
	334
	335	if ((lvl & P(LVL, REM_CCE1)) \|\|
	336	(lvl & P(LVL, REM_CCE2))) {
	337	if (snoop & P(SNOOP, HIT))
	338	stats->rmt_hit++;
	339	else if (snoop & P(SNOOP, HITM))
dba8ab93	340	HITM_INC(rmt_hitm);
aadddd68 JO	341	}
	342
	343	if ((lvl & P(LVL, MISS)))
	344	stats->ld_miss++;
	345
	346	} else if (op & P(OP, STORE)) {
	347	/* store */
	348	stats->store++;
	349
	350	if (!daddr) {
	351	stats->st_noadrs++;
	352	return -1;
	353	}
	354
	355	if (lvl & P(LVL, HIT)) {
	356	if (lvl & P(LVL, UNC)) stats->st_uncache++;
	357	if (lvl & P(LVL, L1 )) stats->st_l1hit++;
	358	}
	359	if (lvl & P(LVL, MISS))
	360	if (lvl & P(LVL, L1)) stats->st_l1miss++;
	361	} else {
	362	/* unparsable data_src? */
	363	stats->noparse++;
	364	return -1;
	365	}
	366
	367	if (!mi->daddr.map \|\| !mi->iaddr.map) {
	368	stats->nomap++;
	369	return -1;
	370	}
	371
	372	#undef P
dba8ab93	373	#undef HITM_INC
aadddd68 JO	374	return err;
aadddd68 JO	375	}
0a9a24cc JO	376
	377	void c2c_add_stats(struct c2c_stats stats, struct c2c_stats add)
	378	{
	379	stats->nr_entries += add->nr_entries;
	380
	381	stats->locks += add->locks;
	382	stats->store += add->store;
	383	stats->st_uncache += add->st_uncache;
	384	stats->st_noadrs += add->st_noadrs;
	385	stats->st_l1hit += add->st_l1hit;
	386	stats->st_l1miss += add->st_l1miss;
	387	stats->load += add->load;
	388	stats->ld_excl += add->ld_excl;
	389	stats->ld_shared += add->ld_shared;
	390	stats->ld_uncache += add->ld_uncache;
	391	stats->ld_io += add->ld_io;
	392	stats->ld_miss += add->ld_miss;
	393	stats->ld_noadrs += add->ld_noadrs;
	394	stats->ld_fbhit += add->ld_fbhit;
	395	stats->ld_l1hit += add->ld_l1hit;
	396	stats->ld_l2hit += add->ld_l2hit;
	397	stats->ld_llchit += add->ld_llchit;
	398	stats->lcl_hitm += add->lcl_hitm;
	399	stats->rmt_hitm += add->rmt_hitm;
dba8ab93	400	stats->tot_hitm += add->tot_hitm;
0a9a24cc JO	401	stats->rmt_hit += add->rmt_hit;
	402	stats->lcl_dram += add->lcl_dram;
	403	stats->rmt_dram += add->rmt_dram;
	404	stats->nomap += add->nomap;
	405	stats->noparse += add->noparse;
	406	}